diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java index 61c15dfa8d5..9ce01c8b7b8 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java @@ -25,7 +25,7 @@ public final class Msg { /** * IMPORTANT: Please update the following comment after you add a new code - * Last used code value: 2139 + * Last used code value: 2140 */ private Msg() {} diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/3951-change-missing-field-search-part1.yml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/3951-change-missing-field-search-part1.yml new file mode 100644 index 00000000000..562bf39b31e --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/3951-change-missing-field-search-part1.yml @@ -0,0 +1,8 @@ +--- +type: fix +issue: 3951 +title: "There are now 2 different methods of Missing Fields search. + One that works if Enable Missing Fields Search is enabled, + and one that works if it is not. + These 2 are not compatible together. + " diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java index bfb405d23e6..9fa3d938236 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java @@ -8,10 +8,8 @@ import ca.uhn.fhir.interceptor.api.IInterceptorService; import ca.uhn.fhir.interceptor.executor.InterceptorService; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.api.dao.IDao; import ca.uhn.fhir.jpa.api.model.ExpungeOptions; import ca.uhn.fhir.jpa.api.svc.IIdHelperService; -import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; import ca.uhn.fhir.jpa.batch.BatchJobsConfig; import ca.uhn.fhir.jpa.batch.config.BatchConstants; import ca.uhn.fhir.jpa.binary.interceptor.BinaryStorageInterceptor; @@ -73,11 +71,10 @@ import ca.uhn.fhir.jpa.search.ISynchronousSearchSvc; import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider; import ca.uhn.fhir.jpa.search.PersistedJpaBundleProviderFactory; import ca.uhn.fhir.jpa.search.PersistedJpaSearchFirstPageBundleProvider; -import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.search.SearchStrategyFactory; import ca.uhn.fhir.jpa.search.SynchronousSearchSvcImpl; import ca.uhn.fhir.jpa.search.builder.QueryStack; -import ca.uhn.fhir.jpa.search.builder.SearchBuilder; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchTask; import ca.uhn.fhir.jpa.search.builder.predicate.ComboNonUniqueSearchParameterPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.ComboUniqueSearchParameterPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.CoordsPredicateBuilder; @@ -116,9 +113,7 @@ import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamProvider; import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc; import ca.uhn.fhir.jpa.sp.SearchParamPresenceSvcImpl; import ca.uhn.fhir.jpa.term.TermConceptMappingSvcImpl; -import ca.uhn.fhir.jpa.term.TermDeferredStorageSvcImpl; import ca.uhn.fhir.jpa.term.api.ITermConceptMappingSvc; -import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; import ca.uhn.fhir.jpa.term.config.TermCodeSystemConfig; import ca.uhn.fhir.jpa.util.MemoryCacheService; import ca.uhn.fhir.jpa.validation.ResourceLoaderImpl; @@ -133,11 +128,9 @@ import ca.uhn.fhir.rest.server.interceptor.consent.IConsentContextServices; import ca.uhn.fhir.rest.server.interceptor.partition.RequestTenantPartitionInterceptor; import ca.uhn.fhir.util.ThreadPoolUtil; import org.hl7.fhir.common.hapi.validation.support.UnknownCodeSystemWarningValidationSupport; -import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.utilities.graphql.IGraphQLStorageServices; import org.hl7.fhir.utilities.npm.PackageClient; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @@ -184,7 +177,8 @@ import java.util.Date; SearchParamConfig.class, ValidationSupportConfig.class, Batch2SupportConfig.class, - JpaBulkExportConfig.class + JpaBulkExportConfig.class, + SearchConfig.class }) public class JpaConfig { public static final String JPA_VALIDATION_SUPPORT_CHAIN = "myJpaValidationSupportChain"; @@ -499,7 +493,7 @@ public class JpaConfig { @Bean(name = PERSISTED_JPA_SEARCH_FIRST_PAGE_BUNDLE_PROVIDER) @Scope("prototype") - public PersistedJpaSearchFirstPageBundleProvider newPersistedJpaSearchFirstPageBundleProvider(RequestDetails theRequest, Search theSearch, SearchCoordinatorSvcImpl.SearchTask theSearchTask, ISearchBuilder theSearchBuilder) { + public PersistedJpaSearchFirstPageBundleProvider newPersistedJpaSearchFirstPageBundleProvider(RequestDetails theRequest, Search theSearch, SearchTask theSearchTask, ISearchBuilder theSearchBuilder) { return new PersistedJpaSearchFirstPageBundleProvider(theSearch, theSearchTask, theSearchBuilder, theRequest); } @@ -618,12 +612,6 @@ public class JpaConfig { return new SearchQueryExecutor(theGeneratedSql, theMaxResultsToFetch); } - @Bean(name = ISearchBuilder.SEARCH_BUILDER_BEAN_NAME) - @Scope("prototype") - public ISearchBuilder newSearchBuilder(IDao theDao, String theResourceName, Class theResourceType, DaoConfig theDaoConfig) { - return new SearchBuilder(theDao, theResourceName, theResourceType); - } - @Bean(name = HISTORY_BUILDER) @Scope("prototype") public HistoryBuilder newPersistedJpaSearchFirstPageBundleProvider(@Nullable String theResourceType, @Nullable Long theResourceId, @Nullable Date theRangeStartInclusive, @Nullable Date theRangeEndInclusive) { @@ -641,11 +629,6 @@ public class JpaConfig { return new IdHelperService(); } - @Bean - public ISearchCoordinatorSvc searchCoordinatorSvc() { - return new SearchCoordinatorSvcImpl(); - } - @Bean public SearchStrategyFactory searchStrategyFactory(@Autowired(required = false) IFulltextSearchSvc theFulltextSvc) { return new SearchStrategyFactory(myDaoConfig, theFulltextSvc); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/SearchConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/SearchConfig.java new file mode 100644 index 00000000000..18dd158e0b8 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/SearchConfig.java @@ -0,0 +1,174 @@ +package ca.uhn.fhir.jpa.config; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.api.dao.IDao; +import ca.uhn.fhir.jpa.api.svc.IIdHelperService; +import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; +import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; +import ca.uhn.fhir.jpa.dao.ISearchBuilder; +import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; +import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao; +import ca.uhn.fhir.jpa.dao.data.IResourceTagDao; +import ca.uhn.fhir.jpa.model.config.PartitionSettings; +import ca.uhn.fhir.jpa.model.entity.ModelConfig; +import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; +import ca.uhn.fhir.jpa.search.ExceptionService; +import ca.uhn.fhir.jpa.search.ISynchronousSearchSvc; +import ca.uhn.fhir.jpa.search.PersistedJpaBundleProviderFactory; +import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.search.SearchStrategyFactory; +import ca.uhn.fhir.jpa.search.builder.SearchBuilder; +import ca.uhn.fhir.jpa.search.builder.sql.SqlObjectFactory; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchContinuationTask; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchTask; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchTaskParameters; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; +import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc; +import ca.uhn.fhir.rest.server.IPagingProvider; +import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +public class SearchConfig { + public static final String SEARCH_TASK = "searchTask"; + public static final String CONTINUE_TASK = "continueTask"; + + @Autowired + private DaoConfig myDaoConfig; + @Autowired + private HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory; + @Autowired + private SqlObjectFactory mySqlBuilderFactory; + @Autowired + private HibernatePropertiesProvider myDialectProvider; + @Autowired + private ModelConfig myModelConfig; + @Autowired + private ISearchParamRegistry mySearchParamRegistry; + @Autowired + private PartitionSettings myPartitionSettings; + @Autowired + protected IInterceptorBroadcaster myInterceptorBroadcaster; + @Autowired + protected IResourceTagDao myResourceTagDao; + @Autowired + private DaoRegistry myDaoRegistry; + @Autowired + private IResourceSearchViewDao myResourceSearchViewDao; + @Autowired + private FhirContext myContext; + @Autowired + private IIdHelperService myIdHelperService; + @Autowired + private PlatformTransactionManager myManagedTxManager; + @Autowired + private SearchStrategyFactory mySearchStrategyFactory; + @Autowired + private SearchBuilderFactory mySearchBuilderFactory; + @Autowired + private ISearchResultCacheSvc mySearchResultCacheSvc; + @Autowired + private ISearchCacheSvc mySearchCacheSvc; + @Autowired + private IPagingProvider myPagingProvider; + @Autowired + private BeanFactory myBeanFactory; + @Autowired + private ISynchronousSearchSvc mySynchronousSearchSvc; + @Autowired + private PersistedJpaBundleProviderFactory myPersistedJpaBundleProviderFactory; + @Autowired + private IRequestPartitionHelperSvc myRequestPartitionHelperService; + + @Bean + public ISearchCoordinatorSvc searchCoordinatorSvc() { + return new SearchCoordinatorSvcImpl( + myContext, + myDaoConfig, + myInterceptorBroadcaster, + myManagedTxManager, + mySearchCacheSvc, + mySearchResultCacheSvc, + myDaoRegistry, + mySearchBuilderFactory, + mySynchronousSearchSvc, + myPersistedJpaBundleProviderFactory, + myRequestPartitionHelperService, + mySearchParamRegistry, + mySearchStrategyFactory, + exceptionService(), + myBeanFactory + ); + } + + @Bean + public ExceptionService exceptionService() { + return new ExceptionService(myContext); + } + + @Bean(name = ISearchBuilder.SEARCH_BUILDER_BEAN_NAME) + @Scope("prototype") + public ISearchBuilder newSearchBuilder(IDao theDao, String theResourceName, Class theResourceType, DaoConfig theDaoConfig) { + return new SearchBuilder(theDao, + theResourceName, + myDaoConfig, + myEntityManagerFactory, + mySqlBuilderFactory, + myDialectProvider, + myModelConfig, + mySearchParamRegistry, + myPartitionSettings, + myInterceptorBroadcaster, + myResourceTagDao, + myDaoRegistry, + myResourceSearchViewDao, + myContext, + myIdHelperService, + theResourceType + ); + } + + @Bean(name = SEARCH_TASK) + @Scope("prototype") + public SearchTask createSearchTask(SearchTaskParameters theParams) { + return new SearchTask(theParams, + myManagedTxManager, + myContext, + mySearchStrategyFactory, + myInterceptorBroadcaster, + mySearchBuilderFactory, + mySearchResultCacheSvc, + myDaoConfig, + mySearchCacheSvc, + myPagingProvider + ); + } + + + @Bean(name = CONTINUE_TASK) + @Scope("prototype") + public SearchContinuationTask createSearchContinuationTask(SearchTaskParameters theParams) { + return new SearchContinuationTask(theParams, + myManagedTxManager, + myContext, + mySearchStrategyFactory, + myInterceptorBroadcaster, + mySearchBuilderFactory, + mySearchResultCacheSvc, + myDaoConfig, + mySearchCacheSvc, + myPagingProvider, + exceptionService() // singleton + ); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java index 1f49f28a197..683c18499ab 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java @@ -1537,19 +1537,7 @@ public abstract class BaseHapiFhirResourceDao extends B if (theParams.getSearchContainedMode() != SearchContainedModeEnum.FALSE && !myModelConfig.isIndexOnContainedResources()) { throw new MethodNotAllowedException(Msg.code(984) + "Searching with _contained mode enabled is not enabled on this server"); } - - if (getConfig().getIndexMissingFields() == DaoConfig.IndexEnabledEnum.DISABLED) { - for (List> nextAnds : theParams.values()) { - for (List nextOrs : nextAnds) { - for (IQueryParameterType next : nextOrs) { - if (next.getMissing() != null) { - throw new MethodNotAllowedException(Msg.code(985) + ":missing modifier is disabled on this server"); - } - } - } - } - } - + translateListSearchParams(theParams); notifySearchInterceptors(theParams, theRequest); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/HistoryBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/HistoryBuilder.java index 16683ddaf16..a22d0655d62 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/HistoryBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/HistoryBuilder.java @@ -53,7 +53,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static ca.uhn.fhir.jpa.search.builder.SearchBuilder.toPredicateArray; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toPredicateArray; /** diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java index 9d8f6da70f1..7ee7bed978c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java @@ -2,8 +2,8 @@ package ca.uhn.fhir.jpa.entity; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; -import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.rest.param.DateRangeParam; import ca.uhn.fhir.rest.server.util.ICachedSearchDetails; @@ -228,7 +228,7 @@ public class Search implements ICachedSearchDetails, Serializable { public void setFailureMessage(String theFailureMessage) { myFailureMessage = left(theFailureMessage, FAILURE_MESSAGE_LENGTH); - if (System.getProperty(SearchCoordinatorSvcImpl.UNIT_TEST_CAPTURE_STACK) != null) { + if (System.getProperty(QueryParameterUtils.UNIT_TEST_CAPTURE_STACK) != null) { myFailureMessage = theFailureMessage; } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java index 3f5786eddbd..d1bbd6dd244 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java @@ -103,7 +103,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import static ca.uhn.fhir.jpa.search.builder.SearchBuilder.toPredicateArray; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toPredicateArray; import static ca.uhn.fhir.util.StringUtil.toUtf8String; import static org.apache.commons.lang3.StringUtils.defaultString; import static org.apache.commons.lang3.StringUtils.isNotBlank; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/ExceptionService.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/ExceptionService.java new file mode 100644 index 00000000000..e079fa6ab62 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/ExceptionService.java @@ -0,0 +1,24 @@ +package ca.uhn.fhir.jpa.search; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; +import ca.uhn.fhir.rest.server.method.PageMethodBinding; + +import javax.annotation.Nonnull; + +public class ExceptionService { + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchCoordinatorSvcImpl.class); + + private final FhirContext myContext; + + public ExceptionService(FhirContext theContext) { + myContext = theContext; + } + + @Nonnull + public ResourceGoneException newUnknownSearchException(String theUuid) { + ourLog.trace("Client requested unknown paging ID[{}]", theUuid); + String msg = myContext.getLocalizer().getMessage(PageMethodBinding.class, "unknownSearchId", theUuid); + return new ResourceGoneException(msg); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java index 54473d55c3e..2cc36d1585a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java @@ -41,6 +41,7 @@ import ca.uhn.fhir.jpa.partition.RequestPartitionHelperSvc; import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.SearchCacheStatusEnum; import ca.uhn.fhir.jpa.util.MemoryCacheService; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.model.primitive.InstantDt; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails; @@ -384,7 +385,7 @@ public class PersistedJpaBundleProvider implements IBundleProvider { @Override public Integer size() { ensureSearchEntityLoaded(); - SearchCoordinatorSvcImpl.verifySearchHasntFailedOrThrowInternalErrorException(mySearchEntity); + QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(mySearchEntity); Integer size = mySearchEntity.getTotalCount(); if (size != null) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProviderFactory.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProviderFactory.java index dd22695a82e..0e77506db02 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProviderFactory.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProviderFactory.java @@ -23,6 +23,7 @@ package ca.uhn.fhir.jpa.search; import ca.uhn.fhir.jpa.config.JpaConfig; import ca.uhn.fhir.jpa.dao.ISearchBuilder; import ca.uhn.fhir.jpa.entity.Search; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchTask; import ca.uhn.fhir.rest.api.server.RequestDetails; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -42,7 +43,7 @@ public class PersistedJpaBundleProviderFactory { return (PersistedJpaBundleProvider) retVal; } - public PersistedJpaSearchFirstPageBundleProvider newInstanceFirstPage(RequestDetails theRequestDetails, Search theSearch, SearchCoordinatorSvcImpl.SearchTask theTask, ISearchBuilder theSearchBuilder) { + public PersistedJpaSearchFirstPageBundleProvider newInstanceFirstPage(RequestDetails theRequestDetails, Search theSearch, SearchTask theTask, ISearchBuilder theSearchBuilder) { return (PersistedJpaSearchFirstPageBundleProvider) myApplicationContext.getBean(JpaConfig.PERSISTED_JPA_SEARCH_FIRST_PAGE_BUNDLE_PROVIDER, theRequestDetails, theSearch, theTask, theSearchBuilder); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java index c6b25771310..ce8fc6c12e9 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java @@ -22,9 +22,10 @@ package ca.uhn.fhir.jpa.search; import ca.uhn.fhir.jpa.dao.ISearchBuilder; import ca.uhn.fhir.jpa.entity.Search; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; -import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl.SearchTask; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchTask; import ca.uhn.fhir.model.api.IResource; import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum; import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum; @@ -61,7 +62,7 @@ public class PersistedJpaSearchFirstPageBundleProvider extends PersistedJpaBundl @Nonnull @Override public List getResources(int theFromIndex, int theToIndex) { - SearchCoordinatorSvcImpl.verifySearchHasntFailedOrThrowInternalErrorException(mySearch); + QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(mySearch); mySearchTask.awaitInitialSync(); @@ -121,7 +122,7 @@ public class PersistedJpaSearchFirstPageBundleProvider extends PersistedJpaBundl Integer size = mySearchTask.awaitInitialSync(); ourLog.trace("size() - Finished waiting for local sync"); - SearchCoordinatorSvcImpl.verifySearchHasntFailedOrThrowInternalErrorException(mySearch); + QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(mySearch); if (size != null) { return size; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index a30e0b591be..a1f26f11fbb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -31,147 +31,144 @@ import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IDao; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; +import ca.uhn.fhir.jpa.config.SearchConfig; import ca.uhn.fhir.jpa.dao.BaseStorageDao; -import ca.uhn.fhir.jpa.dao.IResultIterator; import ca.uhn.fhir.jpa.dao.ISearchBuilder; import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; import ca.uhn.fhir.jpa.dao.search.ResourceNotFoundInIndexException; import ca.uhn.fhir.jpa.entity.Search; -import ca.uhn.fhir.jpa.entity.SearchInclude; -import ca.uhn.fhir.jpa.entity.SearchTypeEnum; -import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails; -import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; +import ca.uhn.fhir.jpa.search.builder.StorageInterceptorHooksFacade; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchContinuationTask; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchTask; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchTaskParameters; import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.search.cache.SearchCacheStatusEnum; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; import ca.uhn.fhir.rest.api.SearchTotalModeEnum; import ca.uhn.fhir.rest.api.server.IBundleProvider; -import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; -import ca.uhn.fhir.rest.server.IPagingProvider; -import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; -import ca.uhn.fhir.rest.server.method.PageMethodBinding; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; -import ca.uhn.fhir.rest.server.util.ICachedSearchDetails; import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; import ca.uhn.fhir.util.AsyncUtil; import ca.uhn.fhir.util.StopWatch; import ca.uhn.fhir.util.UrlUtil; -import co.elastic.apm.api.ElasticApm; -import co.elastic.apm.api.Span; -import co.elastic.apm.api.Transaction; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang3.Validate; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.time.DateUtils; import org.hl7.fhir.instance.model.api.IBaseResource; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.BeanFactory; import org.springframework.data.domain.AbstractPageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; -import org.springframework.orm.jpa.JpaDialect; -import org.springframework.orm.jpa.JpaTransactionManager; -import org.springframework.orm.jpa.vendor.HibernateJpaDialect; import org.springframework.stereotype.Component; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionDefinition; -import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Date; -import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.stream.Collectors; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.DEFAULT_SYNC_SIZE; import static ca.uhn.fhir.jpa.util.SearchParameterMapCalculator.isWantCount; -import static ca.uhn.fhir.jpa.util.SearchParameterMapCalculator.isWantOnlyCount; -import static java.util.Objects.nonNull; -import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; @Component("mySearchCoordinatorSvc") public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { - public static final int DEFAULT_SYNC_SIZE = 250; - public static final String UNIT_TEST_CAPTURE_STACK = "unit_test_capture_stack"; private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchCoordinatorSvcImpl.class); - private final ConcurrentHashMap myIdToSearchTask = new ConcurrentHashMap<>(); - @Autowired - private FhirContext myContext; - @Autowired - private DaoConfig myDaoConfig; + + private final FhirContext myContext; + private final DaoConfig myDaoConfig; + private final IInterceptorBroadcaster myInterceptorBroadcaster; + private final PlatformTransactionManager myManagedTxManager; + private final ISearchCacheSvc mySearchCacheSvc; + private final ISearchResultCacheSvc mySearchResultCacheSvc; + private final DaoRegistry myDaoRegistry; + private final SearchBuilderFactory mySearchBuilderFactory; + private final ISynchronousSearchSvc mySynchronousSearchSvc; + private final PersistedJpaBundleProviderFactory myPersistedJpaBundleProviderFactory; + private final IRequestPartitionHelperSvc myRequestPartitionHelperService; + private final ISearchParamRegistry mySearchParamRegistry; + private final SearchStrategyFactory mySearchStrategyFactory; + private final ExceptionService myExceptionSvc; + private final BeanFactory myBeanFactory; private Integer myLoadingThrottleForUnitTests = null; private long myMaxMillisToWaitForRemoteResults = DateUtils.MILLIS_PER_MINUTE; private boolean myNeverUseLocalSearchForUnitTests; - @Autowired - private IInterceptorBroadcaster myInterceptorBroadcaster; - @Autowired - private PlatformTransactionManager myManagedTxManager; - @Autowired - private ISearchCacheSvc mySearchCacheSvc; - @Autowired - private ISearchResultCacheSvc mySearchResultCacheSvc; - @Autowired - private DaoRegistry myDaoRegistry; - @Autowired - private IPagingProvider myPagingProvider; - @Autowired - private SearchBuilderFactory mySearchBuilderFactory; - - @Autowired - private ISynchronousSearchSvc mySynchronousSearchSvc; private int mySyncSize = DEFAULT_SYNC_SIZE; - /** - * Set in {@link #start()} - */ - private boolean myCustomIsolationSupported; - @Autowired - private PersistedJpaBundleProviderFactory myPersistedJpaBundleProviderFactory; - @Autowired - private IRequestPartitionHelperSvc myRequestPartitionHelperService; - @Autowired - private ISearchParamRegistry mySearchParamRegistry; - @Autowired - private SearchStrategyFactory mySearchStrategyFactory; + + private final ConcurrentHashMap myIdToSearchTask = new ConcurrentHashMap<>(); + + private final Consumer myOnRemoveSearchTask = (theId) -> { + myIdToSearchTask.remove(theId); + }; + + private final StorageInterceptorHooksFacade myStorageInterceptorHooks; /** * Constructor */ - @Autowired - public SearchCoordinatorSvcImpl() { + public SearchCoordinatorSvcImpl( + FhirContext theContext, + DaoConfig theDaoConfig, + IInterceptorBroadcaster theInterceptorBroadcaster, + PlatformTransactionManager theManagedTxManager, + ISearchCacheSvc theSearchCacheSvc, + ISearchResultCacheSvc theSearchResultCacheSvc, + DaoRegistry theDaoRegistry, + SearchBuilderFactory theSearchBuilderFactory, + ISynchronousSearchSvc theSynchronousSearchSvc, + PersistedJpaBundleProviderFactory thePersistedJpaBundleProviderFactory, + IRequestPartitionHelperSvc theRequestPartitionHelperService, + ISearchParamRegistry theSearchParamRegistry, + SearchStrategyFactory theSearchStrategyFactory, + ExceptionService theExceptionSvc, + BeanFactory theBeanFactory + ) { super(); + myContext = theContext; + myDaoConfig = theDaoConfig; + myInterceptorBroadcaster = theInterceptorBroadcaster; + myManagedTxManager = theManagedTxManager; + mySearchCacheSvc = theSearchCacheSvc; + mySearchResultCacheSvc = theSearchResultCacheSvc; + myDaoRegistry = theDaoRegistry; + mySearchBuilderFactory = theSearchBuilderFactory; + mySynchronousSearchSvc = theSynchronousSearchSvc; + myPersistedJpaBundleProviderFactory = thePersistedJpaBundleProviderFactory; + myRequestPartitionHelperService = theRequestPartitionHelperService; + mySearchParamRegistry = theSearchParamRegistry; + mySearchStrategyFactory = theSearchStrategyFactory; + myExceptionSvc = theExceptionSvc; + myBeanFactory = theBeanFactory; + + myStorageInterceptorHooks = new StorageInterceptorHooksFacade(myInterceptorBroadcaster); } @VisibleForTesting @@ -180,22 +177,18 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } @VisibleForTesting - public void setSearchCacheServicesForUnitTest(ISearchCacheSvc theSearchCacheSvc, ISearchResultCacheSvc theSearchResultCacheSvc) { - mySearchCacheSvc = theSearchCacheSvc; - mySearchResultCacheSvc = theSearchResultCacheSvc; + public void setLoadingThrottleForUnitTests(Integer theLoadingThrottleForUnitTests) { + myLoadingThrottleForUnitTests = theLoadingThrottleForUnitTests; } - @PostConstruct - public void start() { - if (myManagedTxManager instanceof JpaTransactionManager) { - JpaDialect jpaDialect = ((JpaTransactionManager) myManagedTxManager).getJpaDialect(); - if (jpaDialect instanceof HibernateJpaDialect) { - myCustomIsolationSupported = true; - } - } - if (myCustomIsolationSupported == false) { - ourLog.warn("JPA dialect does not support transaction isolation! This can have an impact on search performance."); - } + @VisibleForTesting + public void setNeverUseLocalSearchForUnitTests(boolean theNeverUseLocalSearchForUnitTests) { + myNeverUseLocalSearchForUnitTests = theNeverUseLocalSearchForUnitTests; + } + + @VisibleForTesting + public void setSyncSizeForUnitTests(int theSyncSize) { + mySyncSize = theSyncSize; } @Override @@ -213,29 +206,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { myMaxMillisToWaitForRemoteResults = theMaxMillisToWaitForRemoteResults; } - /** - * facade over raw hook intererface - */ - public class StorageInterceptorHooks { - /** - * Interceptor call: STORAGE_PRESEARCH_REGISTERED - * - * @param theRequestDetails - * @param theParams - * @param search - */ - private void callStoragePresearchRegistered(RequestDetails theRequestDetails, SearchParameterMap theParams, Search search) { - HookParams params = new HookParams() - .add(ICachedSearchDetails.class, search) - .add(RequestDetails.class, theRequestDetails) - .addIfMatchesType(ServletRequestDetails.class, theRequestDetails) - .add(SearchParameterMap.class, theParams); - CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PRESEARCH_REGISTERED, params); - } - //private IInterceptorBroadcaster myInterceptorBroadcaster; - } - private StorageInterceptorHooks myStorageInterceptorHooks = new StorageInterceptorHooks(); - /** * This method is called by the HTTP client processing thread in order to * fetch resources. @@ -279,9 +249,9 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { search = mySearchCacheSvc .fetchByUuid(theUuid) - .orElseThrow(() -> newResourceGoneException(theUuid)); + .orElseThrow(() -> myExceptionSvc.newUnknownSearchException(theUuid)); - verifySearchHasntFailedOrThrowInternalErrorException(search); + QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(search); if (search.getStatus() == SearchStatusEnum.FINISHED) { ourLog.trace("Search entity marked as finished with {} results", search.getNumFound()); break; @@ -308,7 +278,20 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { SearchParameterMap params = search.getSearchParameterMap().orElseThrow(() -> new IllegalStateException("No map in PASSCOMPLET search")); IFhirResourceDao resourceDao = myDaoRegistry.getResourceDao(resourceType); RequestPartitionId requestPartitionId = myRequestPartitionHelperService.determineReadPartitionForRequestForSearchType(theRequestDetails, resourceType, params, null); - SearchContinuationTask task = new SearchContinuationTask(search, resourceDao, params, resourceType, theRequestDetails, requestPartitionId); + + SearchTaskParameters parameters = new SearchTaskParameters( + search, + resourceDao, + params, + resourceType, + theRequestDetails, + requestPartitionId, + myOnRemoveSearchTask, + mySyncSize + ); + parameters.setLoadingThrottleForUnitTests(myLoadingThrottleForUnitTests); + SearchContinuationTask task = (SearchContinuationTask) myBeanFactory.getBean(SearchConfig.CONTINUE_TASK, + parameters); myIdToSearchTask.put(search.getUuid(), task); task.call(); } @@ -321,7 +304,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { List pids = mySearchResultCacheSvc.fetchResultPids(search, theFrom, theTo); if (pids == null) { - throw newResourceGoneException(theUuid); + throw myExceptionSvc.newUnknownSearchException(theUuid); } ourLog.trace("Fetched {} results", pids.size()); @@ -329,13 +312,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { return pids; } - @Nonnull - private ResourceGoneException newResourceGoneException(String theUuid) { - ourLog.trace("Client requested unknown paging ID[{}]", theUuid); - String msg = myContext.getLocalizer().getMessage(PageMethodBinding.class, "unknownSearchId", theUuid); - return new ResourceGoneException(msg); - } - @Override public IBundleProvider registerSearch(final IFhirResourceDao theCallingDao, final SearchParameterMap theParams, String theResourceType, CacheControlDirective theCacheControlDirective, RequestDetails theRequestDetails, RequestPartitionId theRequestPartitionId) { final String searchUuid = UUID.randomUUID().toString(); @@ -344,7 +320,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { ourLog.debug("Registering new search {}", searchUuid); Search search = new Search(); - populateSearchEntity(theParams, theResourceType, searchUuid, queryString, search, theRequestPartitionId); + QueryParameterUtils.populateSearchEntity(theParams, theResourceType, searchUuid, queryString, search, theRequestPartitionId); myStorageInterceptorHooks.callStoragePresearchRegistered(theRequestDetails, theParams, search); @@ -474,7 +450,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { if (searchParameterMap.isPresent() && searchParameterMap.get().getSearchTotalMode() == SearchTotalModeEnum.ACCURATE) { for (int i = 0; i < 10; i++) { if (search.isPresent()) { - verifySearchHasntFailedOrThrowInternalErrorException(search.get()); + QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(search.get()); if (search.get().getTotalCount() != null) { return Optional.of(search.get().getTotalCount()); } @@ -490,7 +466,19 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { @Nonnull private PersistedJpaSearchFirstPageBundleProvider submitSearch(IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequestDetails, String theSearchUuid, ISearchBuilder theSb, String theQueryString, RequestPartitionId theRequestPartitionId, Search theSearch) { StopWatch w = new StopWatch(); - SearchTask task = new SearchTask(theSearch, theCallingDao, theParams, theResourceType, theRequestDetails, theRequestPartitionId); + + SearchTaskParameters stp = new SearchTaskParameters( + theSearch, + theCallingDao, + theParams, + theResourceType, + theRequestDetails, + theRequestPartitionId, + myOnRemoveSearchTask, + mySyncSize + ); + stp.setLoadingThrottleForUnitTests(myLoadingThrottleForUnitTests); + SearchTask task = (SearchTask) myBeanFactory.getBean(SearchConfig.SEARCH_TASK, stp); myIdToSearchTask.put(theSearch.getUuid(), task); task.call(); @@ -562,89 +550,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { return loadSynchronousUpTo; } - @VisibleForTesting - void setContextForUnitTest(FhirContext theCtx) { - myContext = theCtx; - } - - @VisibleForTesting - void setDaoConfigForUnitTest(DaoConfig theDaoConfig) { - myDaoConfig = theDaoConfig; - } - - @VisibleForTesting - public void setLoadingThrottleForUnitTests(Integer theLoadingThrottleForUnitTests) { - myLoadingThrottleForUnitTests = theLoadingThrottleForUnitTests; - } - - @VisibleForTesting - public void setNeverUseLocalSearchForUnitTests(boolean theNeverUseLocalSearchForUnitTests) { - myNeverUseLocalSearchForUnitTests = theNeverUseLocalSearchForUnitTests; - } - - @VisibleForTesting - public void setSyncSizeForUnitTests(int theSyncSize) { - mySyncSize = theSyncSize; - } - - @VisibleForTesting - void setTransactionManagerForUnitTest(PlatformTransactionManager theTxManager) { - myManagedTxManager = theTxManager; - } - - @VisibleForTesting - void setDaoRegistryForUnitTest(DaoRegistry theDaoRegistry) { - myDaoRegistry = theDaoRegistry; - } - - @VisibleForTesting - void setInterceptorBroadcasterForUnitTest(IInterceptorBroadcaster theInterceptorBroadcaster) { - myInterceptorBroadcaster = theInterceptorBroadcaster; - } - - @VisibleForTesting - public void setSearchBuilderFactoryForUnitTest(SearchBuilderFactory theSearchBuilderFactory) { - mySearchBuilderFactory = theSearchBuilderFactory; - } - - @VisibleForTesting - public void setPersistedJpaBundleProviderFactoryForUnitTest(PersistedJpaBundleProviderFactory thePersistedJpaBundleProviderFactory) { - myPersistedJpaBundleProviderFactory = thePersistedJpaBundleProviderFactory; - } - - @VisibleForTesting - public void setRequestPartitionHelperService(IRequestPartitionHelperSvc theRequestPartitionHelperService) { - myRequestPartitionHelperService = theRequestPartitionHelperService; - } - - @VisibleForTesting - public void setSynchronousSearchSvc(ISynchronousSearchSvc theSynchronousSearchSvc) { - mySynchronousSearchSvc = theSynchronousSearchSvc; - } - - public static void populateSearchEntity(SearchParameterMap theParams, String theResourceType, String theSearchUuid, String theQueryString, Search theSearch, RequestPartitionId theRequestPartitionId) { - theSearch.setDeleted(false); - theSearch.setUuid(theSearchUuid); - theSearch.setCreated(new Date()); - theSearch.setTotalCount(null); - theSearch.setNumFound(0); - theSearch.setPreferredPageSize(theParams.getCount()); - theSearch.setSearchType(theParams.getEverythingMode() != null ? SearchTypeEnum.EVERYTHING : SearchTypeEnum.SEARCH); - theSearch.setLastUpdated(theParams.getLastUpdated()); - theSearch.setResourceType(theResourceType); - theSearch.setStatus(SearchStatusEnum.LOADING); - theSearch.setSearchQueryString(theQueryString, theRequestPartitionId); - - if (theParams.hasIncludes()) { - for (Include next : theParams.getIncludes()) { - theSearch.addInclude(new SearchInclude(theSearch, next.getValue(), false, next.isRecurse())); - } - } - - for (Include next : theParams.getRevIncludes()) { - theSearch.addInclude(new SearchInclude(theSearch, next.getValue(), true, next.isRecurse())); - } - } /** * Creates a {@link Pageable} using a start and end index @@ -696,596 +601,4 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { return page; } - static void verifySearchHasntFailedOrThrowInternalErrorException(Search theSearch) { - if (theSearch.getStatus() == SearchStatusEnum.FAILED) { - Integer status = theSearch.getFailureCode(); - status = defaultIfNull(status, 500); - - String message = theSearch.getFailureMessage(); - throw BaseServerResponseException.newInstance(status, message); - } - } - - /** - * A search task is a Callable task that runs in - * a thread pool to handle an individual search. One instance - * is created for any requested search and runs from the - * beginning to the end of the search. - *

- * Understand: - * This class executes in its own thread separate from the - * web server client thread that made the request. We do that - * so that we can return to the client as soon as possible, - * but keep the search going in the background (and have - * the next page of results ready to go when the client asks). - */ - public class SearchTask implements Callable { - private final SearchParameterMap myParams; - private final IDao myCallingDao; - private final String myResourceType; - private final ArrayList mySyncedPids = new ArrayList<>(); - private final CountDownLatch myInitialCollectionLatch = new CountDownLatch(1); - private final CountDownLatch myCompletionLatch; - private final ArrayList myUnsyncedPids = new ArrayList<>(); - private final RequestDetails myRequest; - private final RequestPartitionId myRequestPartitionId; - private final SearchRuntimeDetails mySearchRuntimeDetails; - private final Transaction myParentTransaction; - private Search mySearch; - private boolean myAbortRequested; - private int myCountSavedTotal = 0; - private int myCountSavedThisPass = 0; - private int myCountBlockedThisPass = 0; - private boolean myAdditionalPrefetchThresholdsRemaining; - private List myPreviouslyAddedResourcePids; - private Integer myMaxResultsToFetch; - - /** - * Constructor - */ - protected SearchTask(Search theSearch, IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequest, RequestPartitionId theRequestPartitionId) { - mySearch = theSearch; - myCallingDao = theCallingDao; - myParams = theParams; - myResourceType = theResourceType; - myCompletionLatch = new CountDownLatch(1); - mySearchRuntimeDetails = new SearchRuntimeDetails(theRequest, mySearch.getUuid()); - mySearchRuntimeDetails.setQueryString(theParams.toNormalizedQueryString(theCallingDao.getContext())); - myRequestPartitionId = theRequestPartitionId; - myRequest = theRequest; - myParentTransaction = ElasticApm.currentTransaction(); - } - - /** - * This method is called by the server HTTP thread, and - * will block until at least one page of results have been - * fetched from the DB, and will never block after that. - */ - Integer awaitInitialSync() { - ourLog.trace("Awaiting initial sync"); - do { - ourLog.trace("Search {} aborted: {}", getSearch().getUuid(), !isNotAborted()); - if (AsyncUtil.awaitLatchAndThrowInternalErrorExceptionOnInterrupt(getInitialCollectionLatch(), 250L, TimeUnit.MILLISECONDS)) { - break; - } - } while (getSearch().getStatus() == SearchStatusEnum.LOADING); - ourLog.trace("Initial sync completed"); - - return getSearch().getTotalCount(); - } - - protected Search getSearch() { - return mySearch; - } - - CountDownLatch getInitialCollectionLatch() { - return myInitialCollectionLatch; - } - - void setPreviouslyAddedResourcePids(List thePreviouslyAddedResourcePids) { - myPreviouslyAddedResourcePids = thePreviouslyAddedResourcePids; - myCountSavedTotal = myPreviouslyAddedResourcePids.size(); - } - - private ISearchBuilder newSearchBuilder() { - Class resourceTypeClass = myContext.getResourceDefinition(myResourceType).getImplementingClass(); - return mySearchBuilderFactory.newSearchBuilder(myCallingDao, myResourceType, resourceTypeClass); - } - - @Nonnull - List getResourcePids(int theFromIndex, int theToIndex) { - ourLog.debug("Requesting search PIDs from {}-{}", theFromIndex, theToIndex); - - boolean keepWaiting; - do { - synchronized (mySyncedPids) { - ourLog.trace("Search status is {}", mySearch.getStatus()); - boolean haveEnoughResults = mySyncedPids.size() >= theToIndex; - if (!haveEnoughResults) { - switch (mySearch.getStatus()) { - case LOADING: - keepWaiting = true; - break; - case PASSCMPLET: - /* - * If we get here, it means that the user requested resources that crossed the - * current pre-fetch boundary. For example, if the prefetch threshold is 50 and the - * user has requested resources 0-60, then they would get 0-50 back but the search - * coordinator would then stop searching.SearchCoordinatorSvcImplTest - */ - keepWaiting = false; - break; - case FAILED: - case FINISHED: - case GONE: - default: - keepWaiting = false; - break; - } - } else { - keepWaiting = false; - } - } - - if (keepWaiting) { - ourLog.info("Waiting as we only have {} results - Search status: {}", mySyncedPids.size(), mySearch.getStatus()); - AsyncUtil.sleep(500L); - } - } while (keepWaiting); - - ourLog.debug("Proceeding, as we have {} results", mySyncedPids.size()); - - ArrayList retVal = new ArrayList<>(); - synchronized (mySyncedPids) { - verifySearchHasntFailedOrThrowInternalErrorException(mySearch); - - int toIndex = theToIndex; - if (mySyncedPids.size() < toIndex) { - toIndex = mySyncedPids.size(); - } - for (int i = theFromIndex; i < toIndex; i++) { - retVal.add(mySyncedPids.get(i)); - } - } - - ourLog.trace("Done syncing results - Wanted {}-{} and returning {} of {}", theFromIndex, theToIndex, retVal.size(), mySyncedPids.size()); - - return retVal; - } - - void saveSearch() { - TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); - txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); - txTemplate.execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theArg0) { - doSaveSearch(); - } - - }); - } - - private void saveUnsynced(final IResultIterator theResultIter) { - TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); - txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); - txTemplate.execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theArg0) { - if (mySearch.getId() == null) { - doSaveSearch(); - } - - ArrayList unsyncedPids = myUnsyncedPids; - int countBlocked = 0; - - // Interceptor call: STORAGE_PREACCESS_RESOURCES - // This can be used to remove results from the search result details before - // the user has a chance to know that they were in the results - if (mySearchRuntimeDetails.getRequestDetails() != null && unsyncedPids.isEmpty() == false) { - JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(unsyncedPids, () -> newSearchBuilder()); - HookParams params = new HookParams() - .add(IPreResourceAccessDetails.class, accessDetails) - .add(RequestDetails.class, mySearchRuntimeDetails.getRequestDetails()) - .addIfMatchesType(ServletRequestDetails.class, mySearchRuntimeDetails.getRequestDetails()); - CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params); - - for (int i = unsyncedPids.size() - 1; i >= 0; i--) { - if (accessDetails.isDontReturnResourceAtIndex(i)) { - unsyncedPids.remove(i); - myCountBlockedThisPass++; - myCountSavedTotal++; - countBlocked++; - } - } - } - - // Actually store the results in the query cache storage - myCountSavedTotal += unsyncedPids.size(); - myCountSavedThisPass += unsyncedPids.size(); - mySearchResultCacheSvc.storeResults(mySearch, mySyncedPids, unsyncedPids); - - synchronized (mySyncedPids) { - int numSyncedThisPass = unsyncedPids.size(); - ourLog.trace("Syncing {} search results - Have more: {}", numSyncedThisPass, theResultIter.hasNext()); - mySyncedPids.addAll(unsyncedPids); - unsyncedPids.clear(); - - if (theResultIter.hasNext() == false) { - int skippedCount = theResultIter.getSkippedCount(); - int nonSkippedCount = theResultIter.getNonSkippedCount(); - int totalFetched = skippedCount + myCountSavedThisPass + myCountBlockedThisPass; - ourLog.trace("MaxToFetch[{}] SkippedCount[{}] CountSavedThisPass[{}] CountSavedThisTotal[{}] AdditionalPrefetchRemaining[{}]", myMaxResultsToFetch, skippedCount, myCountSavedThisPass, myCountSavedTotal, myAdditionalPrefetchThresholdsRemaining); - - if (nonSkippedCount == 0 || (myMaxResultsToFetch != null && totalFetched < myMaxResultsToFetch)) { - ourLog.trace("Setting search status to FINISHED"); - mySearch.setStatus(SearchStatusEnum.FINISHED); - mySearch.setTotalCount(myCountSavedTotal - countBlocked); - } else if (myAdditionalPrefetchThresholdsRemaining) { - ourLog.trace("Setting search status to PASSCMPLET"); - mySearch.setStatus(SearchStatusEnum.PASSCMPLET); - mySearch.setSearchParameterMap(myParams); - } else { - ourLog.trace("Setting search status to FINISHED"); - mySearch.setStatus(SearchStatusEnum.FINISHED); - mySearch.setTotalCount(myCountSavedTotal - countBlocked); - } - } - } - - mySearch.setNumFound(myCountSavedTotal); - mySearch.setNumBlocked(mySearch.getNumBlocked() + countBlocked); - - int numSynced; - synchronized (mySyncedPids) { - numSynced = mySyncedPids.size(); - } - - if (myDaoConfig.getCountSearchResultsUpTo() == null || - myDaoConfig.getCountSearchResultsUpTo() <= 0 || - myDaoConfig.getCountSearchResultsUpTo() <= numSynced) { - myInitialCollectionLatch.countDown(); - } - - doSaveSearch(); - - ourLog.trace("saveUnsynced() - pre-commit"); - } - }); - ourLog.trace("saveUnsynced() - post-commit"); - - } - - boolean isNotAborted() { - return myAbortRequested == false; - } - - void markComplete() { - myCompletionLatch.countDown(); - } - - CountDownLatch getCompletionLatch() { - return myCompletionLatch; - } - - /** - * Request that the task abort as soon as possible - */ - void requestImmediateAbort() { - myAbortRequested = true; - } - - /** - * This is the method which actually performs the search. - * It is called automatically by the thread pool. - */ - @Override - public Void call() { - StopWatch sw = new StopWatch(); - Span span = myParentTransaction.startSpan("db", "query", "search"); - span.setName("FHIR Database Search"); - try { - // Create an initial search in the DB and give it an ID - saveSearch(); - - TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); - txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); - - if (myCustomIsolationSupported) { - txTemplate.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED); - } - - txTemplate.execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theStatus) { - doSearch(); - } - }); - - mySearchRuntimeDetails.setSearchStatus(mySearch.getStatus()); - if (mySearch.getStatus() == SearchStatusEnum.FINISHED) { - HookParams params = new HookParams() - .add(RequestDetails.class, myRequest) - .addIfMatchesType(ServletRequestDetails.class, myRequest) - .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); - CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_COMPLETE, params); - } else { - HookParams params = new HookParams() - .add(RequestDetails.class, myRequest) - .addIfMatchesType(ServletRequestDetails.class, myRequest) - .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); - CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_PASS_COMPLETE, params); - } - - ourLog.trace("Have completed search for [{}{}] and found {} resources in {}ms - Status is {}", mySearch.getResourceType(), mySearch.getSearchQueryString(), mySyncedPids.size(), sw.getMillis(), mySearch.getStatus()); - - } catch (Throwable t) { - - /* - * Don't print a stack trace for client errors (i.e. requests that - * aren't valid because the client screwed up).. that's just noise - * in the logs and who needs that. - */ - boolean logged = false; - if (t instanceof BaseServerResponseException) { - BaseServerResponseException exception = (BaseServerResponseException) t; - if (exception.getStatusCode() >= 400 && exception.getStatusCode() < 500) { - logged = true; - ourLog.warn("Failed during search due to invalid request: {}", t.toString()); - } - } - - if (!logged) { - ourLog.error("Failed during search loading after {}ms", sw.getMillis(), t); - } - myUnsyncedPids.clear(); - Throwable rootCause = ExceptionUtils.getRootCause(t); - rootCause = defaultIfNull(rootCause, t); - - String failureMessage = rootCause.getMessage(); - - int failureCode = InternalErrorException.STATUS_CODE; - if (t instanceof BaseServerResponseException) { - failureCode = ((BaseServerResponseException) t).getStatusCode(); - } - - if (System.getProperty(UNIT_TEST_CAPTURE_STACK) != null) { - failureMessage += "\nStack\n" + ExceptionUtils.getStackTrace(rootCause); - } - - mySearch.setFailureMessage(failureMessage); - mySearch.setFailureCode(failureCode); - mySearch.setStatus(SearchStatusEnum.FAILED); - - mySearchRuntimeDetails.setSearchStatus(mySearch.getStatus()); - HookParams params = new HookParams() - .add(RequestDetails.class, myRequest) - .addIfMatchesType(ServletRequestDetails.class, myRequest) - .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); - CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FAILED, params); - - saveSearch(); - span.captureException(t); - } finally { - - myIdToSearchTask.remove(mySearch.getUuid()); - myInitialCollectionLatch.countDown(); - markComplete(); - span.end(); - - } - return null; - } - - private void doSaveSearch() { - Search newSearch = mySearchCacheSvc.save(mySearch); - - // mySearchDao.save is not supposed to return null, but in unit tests - // it can if the mock search dao isn't set up to handle that - if (newSearch != null) { - mySearch = newSearch; - } - } - - /** - * This method actually creates the database query to perform the - * search, and starts it. - */ - private void doSearch() { - /* - * If the user has explicitly requested a _count, perform a - * - * SELECT COUNT(*) .... - * - * before doing anything else. - */ - boolean myParamWantOnlyCount = isWantOnlyCount(myParams); - boolean myParamOrDefaultWantCount = nonNull(myParams.getSearchTotalMode()) ? isWantCount(myParams) : isWantCount(myDaoConfig.getDefaultTotalMode()); - - if (myParamWantOnlyCount || myParamOrDefaultWantCount) { - ourLog.trace("Performing count"); - ISearchBuilder sb = newSearchBuilder(); - - /* - * createCountQuery - * NB: (see createQuery below) - * Because FulltextSearchSvcImpl will (internally) - * mutate the myParams (searchmap), - * (specifically removing the _content and _text filters) - * we will have to clone those parameters here so that - * the "correct" params are used in createQuery below - */ - Long count = sb.createCountQuery(myParams.clone(), mySearch.getUuid(), myRequest, myRequestPartitionId); - - ourLog.trace("Got count {}", count); - - TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); - txTemplate.execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theArg0) { - mySearch.setTotalCount(count.intValue()); - if (myParamWantOnlyCount) { - mySearch.setStatus(SearchStatusEnum.FINISHED); - } - doSaveSearch(); - } - }); - if (myParamWantOnlyCount) { - return; - } - } - - ourLog.trace("Done count"); - ISearchBuilder sb = newSearchBuilder(); - - /* - * Figure out how many results we're actually going to fetch from the - * database in this pass. This calculation takes into consideration the - * "pre-fetch thresholds" specified in DaoConfig#getSearchPreFetchThresholds() - * as well as the value of the _count parameter. - */ - int currentlyLoaded = defaultIfNull(mySearch.getNumFound(), 0); - int minWanted = 0; - if (myParams.getCount() != null) { - minWanted = myParams.getCount(); - minWanted = Math.min(minWanted, myPagingProvider.getMaximumPageSize()); - minWanted += currentlyLoaded; - } - - for (Iterator iter = myDaoConfig.getSearchPreFetchThresholds().iterator(); iter.hasNext(); ) { - int next = iter.next(); - if (next != -1 && next <= currentlyLoaded) { - continue; - } - - if (next == -1) { - sb.setMaxResultsToFetch(null); - } else { - myMaxResultsToFetch = Math.max(next, minWanted); - sb.setMaxResultsToFetch(myMaxResultsToFetch); - } - - if (iter.hasNext()) { - myAdditionalPrefetchThresholdsRemaining = true; - } - - // If we get here's we've found an appropriate threshold - break; - } - - /* - * Provide any PID we loaded in previous search passes to the - * SearchBuilder so that we don't get duplicates coming from running - * the same query again. - * - * We could possibly accomplish this in a different way by using sorted - * results in our SQL query and specifying an offset. I don't actually - * know if that would be faster or not. At some point should test this - * idea. - */ - if (myPreviouslyAddedResourcePids != null) { - sb.setPreviouslyAddedResourcePids(myPreviouslyAddedResourcePids); - mySyncedPids.addAll(myPreviouslyAddedResourcePids); - } - - /* - * createQuery - * Construct the SQL query we'll be sending to the database - * - * NB: (See createCountQuery above) - * We will pass the original myParams here (not a copy) - * because we actually _want_ the mutation of the myParams to happen. - * Specifically because SearchBuilder itself will _expect_ - * not to have these parameters when dumping back - * to our DB. - * - * This is an odd implementation behaviour, but the change - * for this will require a lot more handling at higher levels - */ - try (IResultIterator resultIterator = sb.createQuery(myParams, mySearchRuntimeDetails, myRequest, myRequestPartitionId)) { - assert (resultIterator != null); - - /* - * The following loop actually loads the PIDs of the resources - * matching the search off of the disk and into memory. After - * every X results, we commit to the HFJ_SEARCH table. - */ - int syncSize = mySyncSize; - while (resultIterator.hasNext()) { - myUnsyncedPids.add(resultIterator.next()); - - boolean shouldSync = myUnsyncedPids.size() >= syncSize; - - if (myDaoConfig.getCountSearchResultsUpTo() != null && - myDaoConfig.getCountSearchResultsUpTo() > 0 && - myDaoConfig.getCountSearchResultsUpTo() < myUnsyncedPids.size()) { - shouldSync = false; - } - - if (myUnsyncedPids.size() > 50000) { - shouldSync = true; - } - - // If no abort was requested, bail out - Validate.isTrue(isNotAborted(), "Abort has been requested"); - - if (shouldSync) { - saveUnsynced(resultIterator); - } - - if (myLoadingThrottleForUnitTests != null) { - AsyncUtil.sleep(myLoadingThrottleForUnitTests); - } - - } - - // If no abort was requested, bail out - Validate.isTrue(isNotAborted(), "Abort has been requested"); - - saveUnsynced(resultIterator); - - } catch (IOException e) { - ourLog.error("IO failure during database access", e); - throw new InternalErrorException(Msg.code(1166) + e); - } - } - } - - public class SearchContinuationTask extends SearchTask { - - public SearchContinuationTask(Search theSearch, IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequest, RequestPartitionId theRequestPartitionId) { - super(theSearch, theCallingDao, theParams, theResourceType, theRequest, theRequestPartitionId); - } - - @Override - public Void call() { - try { - TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); - txTemplate.afterPropertiesSet(); - txTemplate.execute(t -> { - List previouslyAddedResourcePids = mySearchResultCacheSvc.fetchAllResultPids(getSearch()); - if (previouslyAddedResourcePids == null) { - throw newResourceGoneException(getSearch().getUuid()); - } - - ourLog.trace("Have {} previously added IDs in search: {}", previouslyAddedResourcePids.size(), getSearch().getUuid()); - setPreviouslyAddedResourcePids(previouslyAddedResourcePids); - return null; - }); - } catch (Throwable e) { - ourLog.error("Failure processing search", e); - getSearch().setFailureMessage(e.getMessage()); - getSearch().setStatus(SearchStatusEnum.FAILED); - if (e instanceof BaseServerResponseException) { - getSearch().setFailureCode(((BaseServerResponseException) e).getStatusCode()); - } - - saveSearch(); - return null; - } - - return super.call(); - } - - } - } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/QueryStack.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/QueryStack.java index fd62be16bc7..217c852fcbd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/QueryStack.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/QueryStack.java @@ -33,13 +33,20 @@ import ca.uhn.fhir.jpa.model.entity.ModelConfig; import ca.uhn.fhir.jpa.model.entity.NormalizedQuantitySearchLevel; import ca.uhn.fhir.jpa.model.entity.TagTypeEnum; import ca.uhn.fhir.jpa.model.util.UcumServiceUtil; +import ca.uhn.fhir.jpa.search.builder.models.MissingParameterQueryParams; +import ca.uhn.fhir.jpa.search.builder.models.MissingQueryParameterPredicateParams; +import ca.uhn.fhir.jpa.search.builder.models.PredicateBuilderCacheKey; +import ca.uhn.fhir.jpa.search.builder.models.PredicateBuilderCacheLookupResult; +import ca.uhn.fhir.jpa.search.builder.models.PredicateBuilderTypeEnum; import ca.uhn.fhir.jpa.search.builder.predicate.BaseJoiningPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.BaseQuantityPredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.predicate.BaseSearchParamPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.ComboNonUniqueSearchParameterPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.ComboUniqueSearchParameterPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.CoordsPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.DatePredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.ForcedIdPredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.predicate.ICanMakeMissingParamPredicate; import ca.uhn.fhir.jpa.search.builder.predicate.NumberPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.ResourceIdPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.ResourceLinkPredicateBuilder; @@ -50,11 +57,13 @@ import ca.uhn.fhir.jpa.search.builder.predicate.StringPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.TagPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.TokenPredicateBuilder; import ca.uhn.fhir.jpa.search.builder.predicate.UriPredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.sql.PredicateBuilderFactory; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.extractor.BaseSearchParamExtractor; import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil; import ca.uhn.fhir.jpa.searchparam.util.SourceParam; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.model.api.IQueryParameterAnd; import ca.uhn.fhir.model.api.IQueryParameterOr; import ca.uhn.fhir.model.api.IQueryParameterType; @@ -68,7 +77,6 @@ import ca.uhn.fhir.rest.param.CompositeParam; import ca.uhn.fhir.rest.param.DateParam; import ca.uhn.fhir.rest.param.HasParam; import ca.uhn.fhir.rest.param.NumberParam; -import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.QuantityParam; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.StringParam; @@ -94,22 +102,15 @@ import com.healthmarketscience.sqlbuilder.SetOperationQuery; import com.healthmarketscience.sqlbuilder.Subquery; import com.healthmarketscience.sqlbuilder.UnionQuery; import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn; -import org.apache.commons.collections4.BidiMap; -import org.apache.commons.collections4.bidimap.DualHashBidiMap; -import org.apache.commons.collections4.bidimap.UnmodifiableBidiMap; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.tuple.Triple; import org.hl7.fhir.instance.model.api.IAnyResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.math.BigDecimal; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -121,29 +122,19 @@ import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; -import static org.apache.commons.lang3.StringUtils.isBlank; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.fromOperation; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.getChainedPart; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.getParamNameWithPrefix; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toAndPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toEqualToOrInPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toOperation; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toOrPredicate; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.split; public class QueryStack { private static final Logger ourLog = LoggerFactory.getLogger(QueryStack.class); - private static final BidiMap ourCompareOperationToParamPrefix; - - static { - DualHashBidiMap compareOperationToParamPrefix = new DualHashBidiMap<>(); - compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.ap, ParamPrefixEnum.APPROXIMATE); - compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.eq, ParamPrefixEnum.EQUAL); - compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.gt, ParamPrefixEnum.GREATERTHAN); - compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.ge, ParamPrefixEnum.GREATERTHAN_OR_EQUALS); - compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.lt, ParamPrefixEnum.LESSTHAN); - compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.le, ParamPrefixEnum.LESSTHAN_OR_EQUALS); - compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.ne, ParamPrefixEnum.NOT_EQUAL); - compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.eb, ParamPrefixEnum.ENDS_BEFORE); - compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.sa, ParamPrefixEnum.STARTS_AFTER); - ourCompareOperationToParamPrefix = UnmodifiableBidiMap.unmodifiableBidiMap(compareOperationToParamPrefix); - } private final ModelConfig myModelConfig; private final FhirContext myFhirContext; @@ -355,25 +346,194 @@ public class QueryStack { } + + private Condition createMissingParameterQuery( + MissingParameterQueryParams theParams + ) { + if (theParams.getParamType() == RestSearchParameterTypeEnum.COMPOSITE) { + ourLog.error("Cannot create missing parameter query for a composite parameter."); + return null; + } else if (theParams.getParamType() == RestSearchParameterTypeEnum.REFERENCE) { + if (isEligibleForContainedResourceSearch(theParams.getQueryParameterTypes())) { + ourLog.error("Cannot construct missing query parameter search for ContainedResource REFERENCE search."); + return null; + } + } + + // TODO - Change this when we have HFJ_SPIDX_MISSING table + /** + * How we search depends on if the + * {@link DaoConfig#getIndexMissingFields()} property + * is Enabled or Disabled. + * + * If it is, we will use the SP_MISSING values set into the various + * SP_INDX_X tables and search on those ("old" search). + * + * If it is not set, however, we will try and construct a query that + * looks for missing SearchParameters in the SP_IDX_* tables ("new" search). + * + * You cannot mix and match, however (SP_MISSING is not in HASH_IDENTITY information). + * So setting (or unsetting) the IndexMissingFields + * property should always be followed up with a /$reindex call. + * + * --- + * + * Current limitations: + * Checking if a row exists ("new" search) for a given missing field in an SP_INDX_* table + * (ie, :missing=true) is slow when there are many resources in the table. (Defaults to + * a table scan, since HASH_IDENTITY isn't part of the index). + * + * However, the "old" search method was slow for the reverse: when looking for resources + * that do not have a missing field (:missing=false) for much the same reason. + */ + SearchQueryBuilder sqlBuilder = theParams.getSqlBuilder(); + if (myDaoConfig.getIndexMissingFields() == DaoConfig.IndexEnabledEnum.DISABLED) { + // new search + return createMissingPredicateForUnindexedMissingFields(theParams, sqlBuilder); + } else { + // old search + return createMissingPredicateForIndexedMissingFields(theParams, sqlBuilder); + } + } + + /** + * Old way of searching. + * Missing values must be indexed! + */ + private Condition createMissingPredicateForIndexedMissingFields(MissingParameterQueryParams theParams, SearchQueryBuilder sqlBuilder) { + PredicateBuilderTypeEnum predicateType = null; + Supplier supplier = null; + switch (theParams.getParamType()) { + case STRING: + predicateType = PredicateBuilderTypeEnum.STRING; + supplier = () -> sqlBuilder.addStringPredicateBuilder(theParams.getSourceJoinColumn()); + break; + case NUMBER: + predicateType = PredicateBuilderTypeEnum.NUMBER; + supplier = () -> sqlBuilder.addNumberPredicateBuilder(theParams.getSourceJoinColumn()); + break; + case DATE: + predicateType = PredicateBuilderTypeEnum.DATE; + supplier = () -> sqlBuilder.addDatePredicateBuilder(theParams.getSourceJoinColumn()); + break; + case TOKEN: + predicateType = PredicateBuilderTypeEnum.TOKEN; + supplier = () -> sqlBuilder.addTokenPredicateBuilder(theParams.getSourceJoinColumn()); + break; + case QUANTITY: + predicateType = PredicateBuilderTypeEnum.QUANTITY; + supplier = () -> sqlBuilder.addQuantityPredicateBuilder(theParams.getSourceJoinColumn()); + break; + case REFERENCE: + case URI: + // we expect these values, but the pattern is slightly different; + // see below + break; + case HAS: + case SPECIAL: + predicateType = PredicateBuilderTypeEnum.COORDS; + supplier = () -> sqlBuilder.addCoordsPredicateBuilder(theParams.getSourceJoinColumn()); + break; + case COMPOSITE: + default: + break; + } + + if (supplier != null) { + BaseSearchParamPredicateBuilder join = (BaseSearchParamPredicateBuilder) createOrReusePredicateBuilder( + predicateType, + theParams.getSourceJoinColumn(), + theParams.getParamName(), + supplier + ).getResult(); + + return join.createPredicateParamMissingForNonReference( + theParams.getResourceType(), + theParams.getParamName(), + theParams.isMissing(), + theParams.getRequestPartitionId() + ); + } else { + if (theParams.getParamType() == RestSearchParameterTypeEnum.REFERENCE) { + SearchParamPresentPredicateBuilder join = sqlBuilder.addSearchParamPresentPredicateBuilder(theParams.getSourceJoinColumn()); + return join.createPredicateParamMissingForReference( + theParams.getResourceType(), + theParams.getParamName(), + theParams.isMissing(), + theParams.getRequestPartitionId() + ); + } else if (theParams.getParamType() == RestSearchParameterTypeEnum.URI) { + UriPredicateBuilder join = sqlBuilder.addUriPredicateBuilder(theParams.getSourceJoinColumn()); + return join.createPredicateParamMissingForNonReference( + theParams.getResourceType(), + theParams.getParamName(), + theParams.isMissing(), + theParams.getRequestPartitionId() + ); + } else { + // we don't expect to see this + ourLog.error("Invalid param type " + theParams.getParamType().name()); + return null; + } + } + } + + /** + * New way of searching for missing fields. + * Missing values must not indexed! + */ + private Condition createMissingPredicateForUnindexedMissingFields(MissingParameterQueryParams theParams, SearchQueryBuilder sqlBuilder) { + ResourceTablePredicateBuilder table = sqlBuilder.getOrCreateResourceTablePredicateBuilder(); + + ICanMakeMissingParamPredicate innerQuery = PredicateBuilderFactory.createPredicateBuilderForParamType( + theParams.getParamType(), + theParams.getSqlBuilder(), + this + ); + + return innerQuery.createPredicateParamMissingValue( + new MissingQueryParameterPredicateParams( + table, + theParams.isMissing(), + theParams.getParamName(), + theParams.getRequestPartitionId() + ) + ); + } + public Condition createPredicateCoords(@Nullable DbColumn theSourceJoinColumn, String theResourceName, + String theSpnamePrefix, RuntimeSearchParam theSearchParam, List theList, - RequestPartitionId theRequestPartitionId) { + RequestPartitionId theRequestPartitionId, + SearchQueryBuilder theSqlBuilder) { + Boolean isMissing = theList.get(0).getMissing(); + if (isMissing != null) { + String paramName = getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); - CoordsPredicateBuilder predicateBuilder = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.COORDS, theSourceJoinColumn, theSearchParam.getName(), () -> mySqlBuilder.addCoordsPredicateBuilder(theSourceJoinColumn)).getResult(); + return createMissingParameterQuery( + new MissingParameterQueryParams( + theSqlBuilder, + theSearchParam.getParamType(), + theList, + paramName, + theResourceName, + theSourceJoinColumn, + theRequestPartitionId + ) + ); + } else { + CoordsPredicateBuilder predicateBuilder = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.COORDS, theSourceJoinColumn, theSearchParam.getName(), () -> mySqlBuilder.addCoordsPredicateBuilder(theSourceJoinColumn)).getResult(); - if (theList.get(0).getMissing() != null) { - return predicateBuilder.createPredicateParamMissingForNonReference(theResourceName, theSearchParam.getName(), theList.get(0).getMissing(), theRequestPartitionId); + List codePredicates = new ArrayList<>(); + for (IQueryParameterType nextOr : theList) { + Condition singleCode = predicateBuilder.createPredicateCoords(mySearchParameters, nextOr, theResourceName, theSearchParam, predicateBuilder, theRequestPartitionId); + codePredicates.add(singleCode); + } + + return predicateBuilder.combineWithRequestPartitionIdPredicate(theRequestPartitionId, ComboCondition.or(codePredicates.toArray(new Condition[0]))); } - - List codePredicates = new ArrayList<>(); - for (IQueryParameterType nextOr : theList) { - Condition singleCode = predicateBuilder.createPredicateCoords(mySearchParameters, nextOr, theResourceName, theSearchParam, predicateBuilder, theRequestPartitionId); - codePredicates.add(singleCode); - } - - return predicateBuilder.combineWithRequestPartitionIdPredicate(theRequestPartitionId, ComboCondition.or(codePredicates.toArray(new Condition[0]))); } public Condition createPredicateDate(@Nullable DbColumn theSourceJoinColumn, String theResourceName, @@ -384,34 +544,42 @@ public class QueryStack { public Condition createPredicateDate(@Nullable DbColumn theSourceJoinColumn, String theResourceName, String theSpnamePrefix, RuntimeSearchParam theSearchParam, List theList, SearchFilterParser.CompareOperation theOperation, RequestPartitionId theRequestPartitionId, SearchQueryBuilder theSqlBuilder) { - String paramName = getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); - PredicateBuilderCacheLookupResult predicateBuilderLookupResult = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.DATE, theSourceJoinColumn, paramName, () -> theSqlBuilder.addDatePredicateBuilder(theSourceJoinColumn)); - DatePredicateBuilder predicateBuilder = predicateBuilderLookupResult.getResult(); - boolean cacheHit = predicateBuilderLookupResult.isCacheHit(); + Boolean isMissing = theList.get(0).getMissing(); + if (isMissing != null) { + return createMissingParameterQuery( + new MissingParameterQueryParams( + theSqlBuilder, + theSearchParam.getParamType(), + theList, + paramName, + theResourceName, + theSourceJoinColumn, + theRequestPartitionId + ) + ); + } else { + PredicateBuilderCacheLookupResult predicateBuilderLookupResult = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.DATE, theSourceJoinColumn, paramName, () -> theSqlBuilder.addDatePredicateBuilder(theSourceJoinColumn)); + DatePredicateBuilder predicateBuilder = predicateBuilderLookupResult.getResult(); + boolean cacheHit = predicateBuilderLookupResult.isCacheHit(); - if (theList.get(0).getMissing() != null) { - Boolean missing = theList.get(0).getMissing(); - return predicateBuilder.createPredicateParamMissingForNonReference(theResourceName, paramName, missing, theRequestPartitionId); + List codePredicates = new ArrayList<>(); + + for (IQueryParameterType nextOr : theList) { + Condition p = predicateBuilder.createPredicateDateWithoutIdentityPredicate(nextOr, theOperation); + codePredicates.add(p); + } + + Condition predicate = toOrPredicate(codePredicates); + + if (!cacheHit) { + predicate = predicateBuilder.combineWithHashIdentityPredicate(theResourceName, paramName, predicate); + predicate = predicateBuilder.combineWithRequestPartitionIdPredicate(theRequestPartitionId, predicate); + } + + return predicate; } - - List codePredicates = new ArrayList<>(); - - for (IQueryParameterType nextOr : theList) { - Condition p = predicateBuilder.createPredicateDateWithoutIdentityPredicate(nextOr, theOperation); - codePredicates.add(p); - } - - Condition predicate = toOrPredicate(codePredicates); - - if (!cacheHit) { - predicate = predicateBuilder.combineWithHashIdentityPredicate(theResourceName, paramName, predicate); - predicate = predicateBuilder.combineWithRequestPartitionIdPredicate(theRequestPartitionId, predicate); - } - - return predicate; - } private Condition createPredicateFilter(QueryStack theQueryStack3, SearchFilterParser.BaseFilter theFilter, String theResourceName, RequestDetails theRequest, RequestPartitionId theRequestPartitionId) { @@ -597,39 +765,50 @@ public class QueryStack { String paramName = getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); - NumberPredicateBuilder join = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.NUMBER, theSourceJoinColumn, paramName, () -> theSqlBuilder.addNumberPredicateBuilder(theSourceJoinColumn)).getResult(); + Boolean isMissing = theList.get(0).getMissing(); + if (isMissing != null) { + return createMissingParameterQuery( + new MissingParameterQueryParams( + theSqlBuilder, + theSearchParam.getParamType(), + theList, + paramName, + theResourceName, + theSourceJoinColumn, + theRequestPartitionId + ) + ); + } else { + NumberPredicateBuilder join = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.NUMBER, theSourceJoinColumn, paramName, () -> theSqlBuilder.addNumberPredicateBuilder(theSourceJoinColumn)).getResult(); - if (theList.get(0).getMissing() != null) { - return join.createPredicateParamMissingForNonReference(theResourceName, paramName, theList.get(0).getMissing(), theRequestPartitionId); - } + List codePredicates = new ArrayList<>(); + for (IQueryParameterType nextOr : theList) { - List codePredicates = new ArrayList<>(); - for (IQueryParameterType nextOr : theList) { + if (nextOr instanceof NumberParam) { + NumberParam param = (NumberParam) nextOr; - if (nextOr instanceof NumberParam) { - NumberParam param = (NumberParam) nextOr; + BigDecimal value = param.getValue(); + if (value == null) { + continue; + } - BigDecimal value = param.getValue(); - if (value == null) { - continue; + SearchFilterParser.CompareOperation operation = theOperation; + if (operation == null) { + operation = toOperation(param.getPrefix()); + } + + + Condition predicate = join.createPredicateNumeric(theResourceName, paramName, operation, value, theRequestPartitionId, nextOr); + codePredicates.add(predicate); + + } else { + throw new IllegalArgumentException(Msg.code(1211) + "Invalid token type: " + nextOr.getClass()); } - SearchFilterParser.CompareOperation operation = theOperation; - if (operation == null) { - operation = toOperation(param.getPrefix()); - } - - - Condition predicate = join.createPredicateNumeric(theResourceName, paramName, operation, value, theRequestPartitionId, nextOr); - codePredicates.add(predicate); - - } else { - throw new IllegalArgumentException(Msg.code(1211) + "Invalid token type: " + nextOr.getClass()); } + return join.combineWithRequestPartitionIdPredicate(theRequestPartitionId, ComboCondition.or(codePredicates.toArray(new Condition[0]))); } - - return join.combineWithRequestPartitionIdPredicate(theRequestPartitionId, ComboCondition.or(codePredicates.toArray(new Condition[0]))); } public Condition createPredicateQuantity(@Nullable DbColumn theSourceJoinColumn, String theResourceName, @@ -644,42 +823,52 @@ public class QueryStack { String paramName = getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); - if (theList.get(0).getMissing() != null) { - BaseQuantityPredicateBuilder join = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.QUANTITY, theSourceJoinColumn, theSearchParam.getName(), () -> theSqlBuilder.addQuantityPredicateBuilder(theSourceJoinColumn)).getResult(); - return join.createPredicateParamMissingForNonReference(theResourceName, paramName, theList.get(0).getMissing(), theRequestPartitionId); - } - - List quantityParams = theList - .stream() - .map(t -> QuantityParam.toQuantityParam(t)) - .collect(Collectors.toList()); - - BaseQuantityPredicateBuilder join = null; - boolean normalizedSearchEnabled = myModelConfig.getNormalizedQuantitySearchLevel().equals(NormalizedQuantitySearchLevel.NORMALIZED_QUANTITY_SEARCH_SUPPORTED); - if (normalizedSearchEnabled) { - List normalizedQuantityParams = quantityParams + Boolean isMissing = theList.get(0).getMissing(); + if (isMissing != null) { + return createMissingParameterQuery( + new MissingParameterQueryParams( + theSqlBuilder, + theSearchParam.getParamType(), + theList, + paramName, + theResourceName, + theSourceJoinColumn, + theRequestPartitionId + ) + ); + } else { + List quantityParams = theList .stream() - .map(t -> UcumServiceUtil.toCanonicalQuantityOrNull(t)) - .filter(t -> t != null) + .map(t -> QuantityParam.toQuantityParam(t)) .collect(Collectors.toList()); - if (normalizedQuantityParams.size() == quantityParams.size()) { - join = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.QUANTITY, theSourceJoinColumn, paramName, () -> theSqlBuilder.addQuantityNormalizedPredicateBuilder(theSourceJoinColumn)).getResult(); - quantityParams = normalizedQuantityParams; + BaseQuantityPredicateBuilder join = null; + boolean normalizedSearchEnabled = myModelConfig.getNormalizedQuantitySearchLevel().equals(NormalizedQuantitySearchLevel.NORMALIZED_QUANTITY_SEARCH_SUPPORTED); + if (normalizedSearchEnabled) { + List normalizedQuantityParams = quantityParams + .stream() + .map(t -> UcumServiceUtil.toCanonicalQuantityOrNull(t)) + .filter(t -> t != null) + .collect(Collectors.toList()); + + if (normalizedQuantityParams.size() == quantityParams.size()) { + join = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.QUANTITY, theSourceJoinColumn, paramName, () -> theSqlBuilder.addQuantityNormalizedPredicateBuilder(theSourceJoinColumn)).getResult(); + quantityParams = normalizedQuantityParams; + } } - } - if (join == null) { - join = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.QUANTITY, theSourceJoinColumn, paramName, () -> theSqlBuilder.addQuantityPredicateBuilder(theSourceJoinColumn)).getResult(); - } + if (join == null) { + join = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.QUANTITY, theSourceJoinColumn, paramName, () -> theSqlBuilder.addQuantityPredicateBuilder(theSourceJoinColumn)).getResult(); + } - List codePredicates = new ArrayList<>(); - for (QuantityParam nextOr : quantityParams) { - Condition singleCode = join.createPredicateQuantity(nextOr, theResourceName, paramName, null, join, theOperation, theRequestPartitionId); - codePredicates.add(singleCode); - } + List codePredicates = new ArrayList<>(); + for (QuantityParam nextOr : quantityParams) { + Condition singleCode = join.createPredicateQuantity(nextOr, theResourceName, paramName, null, join, theOperation, theRequestPartitionId); + codePredicates.add(singleCode); + } - return join.combineWithRequestPartitionIdPredicate(theRequestPartitionId, ComboCondition.or(codePredicates.toArray(new Condition[0]))); + return join.combineWithRequestPartitionIdPredicate(theRequestPartitionId, ComboCondition.or(codePredicates.toArray(new Condition[0]))); + } } public Condition createPredicateReference(@Nullable DbColumn theSourceJoinColumn, @@ -708,14 +897,23 @@ public class QueryStack { throw new InvalidRequestException(Msg.code(1212) + "Invalid operator specified for reference predicate. Supported operators for reference predicate are \"eq\" and \"ne\"."); } - if (theList.get(0).getMissing() != null) { - SearchParamPresentPredicateBuilder join = theSqlBuilder.addSearchParamPresentPredicateBuilder(theSourceJoinColumn); - return join.createPredicateParamMissingForReference(theResourceName, theParamName, theList.get(0).getMissing(), theRequestPartitionId); - + Boolean isMissing = theList.get(0).getMissing(); + if (isMissing != null) { + return createMissingParameterQuery( + new MissingParameterQueryParams( + theSqlBuilder, + RestSearchParameterTypeEnum.REFERENCE, + theList, + theParamName, + theResourceName, + theSourceJoinColumn, + theRequestPartitionId + ) + ); + } else { + ResourceLinkPredicateBuilder predicateBuilder = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.REFERENCE, theSourceJoinColumn, theParamName, () -> theSqlBuilder.addReferencePredicateBuilder(this, theSourceJoinColumn)).getResult(); + return predicateBuilder.createPredicate(theRequest, theResourceName, theParamName, theQualifiers, theList, theOperation, theRequestPartitionId); } - - ResourceLinkPredicateBuilder predicateBuilder = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.REFERENCE, theSourceJoinColumn, theParamName, () -> theSqlBuilder.addReferencePredicateBuilder(this, theSourceJoinColumn)).getResult(); - return predicateBuilder.createPredicate(theRequest, theResourceName, theParamName, theQualifiers, theList, theOperation, theRequestPartitionId); } private class ChainElement { @@ -1184,15 +1382,25 @@ public class QueryStack { String theSpnamePrefix, RuntimeSearchParam theSearchParam, List theList, SearchFilterParser.CompareOperation theOperation, RequestPartitionId theRequestPartitionId, SearchQueryBuilder theSqlBuilder) { - + Boolean isMissing = theList.get(0).getMissing(); String paramName = getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); - StringPredicateBuilder join = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.STRING, theSourceJoinColumn, paramName, () -> theSqlBuilder.addStringPredicateBuilder(theSourceJoinColumn)).getResult(); - - if (theList.get(0).getMissing() != null) { - return join.createPredicateParamMissingForNonReference(theResourceName, paramName, theList.get(0).getMissing(), theRequestPartitionId); + if (isMissing != null) { + return createMissingParameterQuery( + new MissingParameterQueryParams( + theSqlBuilder, + theSearchParam.getParamType(), + theList, + paramName, + theResourceName, + theSourceJoinColumn, + theRequestPartitionId + ) + ); } + StringPredicateBuilder join = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.STRING, theSourceJoinColumn, paramName, () -> theSqlBuilder.addStringPredicateBuilder(theSourceJoinColumn)).getResult(); + List codePredicates = new ArrayList<>(); for (IQueryParameterType nextOr : theList) { Condition singleCode = join.createPredicateString(nextOr, theResourceName, theSpnamePrefix, theSearchParam, join, theOperation); @@ -1367,13 +1575,23 @@ public class QueryStack { } } else { - - TokenPredicateBuilder tokenJoin = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.TOKEN, theSourceJoinColumn, paramName, () -> theSqlBuilder.addTokenPredicateBuilder(theSourceJoinColumn)).getResult(); - - if (theList.get(0).getMissing() != null) { - return tokenJoin.createPredicateParamMissingForNonReference(theResourceName, paramName, theList.get(0).getMissing(), theRequestPartitionId); + Boolean isMissing = theList.get(0).getMissing(); + if (isMissing != null) { + return createMissingParameterQuery( + new MissingParameterQueryParams( + theSqlBuilder, + theSearchParam.getParamType(), + theList, + paramName, + theResourceName, + theSourceJoinColumn, + theRequestPartitionId + ) + ); } + TokenPredicateBuilder tokenJoin = createOrReusePredicateBuilder(PredicateBuilderTypeEnum.TOKEN, theSourceJoinColumn, paramName, () -> theSqlBuilder.addTokenPredicateBuilder(theSourceJoinColumn)).getResult(); + predicate = tokenJoin.createPredicateToken(tokens, theResourceName, theSpnamePrefix, theSearchParam, theOperation, theRequestPartitionId); join = tokenJoin; } @@ -1395,14 +1613,25 @@ public class QueryStack { String paramName = getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); - UriPredicateBuilder join = theSqlBuilder.addUriPredicateBuilder(theSourceJoinColumn); + Boolean isMissing = theList.get(0).getMissing(); + if (isMissing != null) { + return createMissingParameterQuery( + new MissingParameterQueryParams( + theSqlBuilder, + theSearchParam.getParamType(), + theList, + paramName, + theResourceName, + theSourceJoinColumn, + theRequestPartitionId + ) + ); + } else { + UriPredicateBuilder join = theSqlBuilder.addUriPredicateBuilder(theSourceJoinColumn); - if (theList.get(0).getMissing() != null) { - return join.createPredicateParamMissingForNonReference(theResourceName, paramName, theList.get(0).getMissing(), theRequestPartitionId); + Condition predicate = join.addPredicate(theList, paramName, theOperation, theRequestDetails); + return join.combineWithRequestPartitionIdPredicate(theRequestPartitionId, predicate); } - - Condition predicate = join.addPredicate(theList, paramName, theOperation, theRequestDetails); - return join.combineWithRequestPartitionIdPredicate(theRequestPartitionId, predicate); } public QueryStack newChildQueryFactoryWithFullBuilderReuse() { @@ -1437,9 +1666,7 @@ public class QueryStack { default: return createPredicateSearchParameter(theSourceJoinColumn, theResourceName, theParamName, theAndOrParams, theRequest, theRequestPartitionId); - } - } @Nullable @@ -1494,7 +1721,7 @@ public class QueryStack { case TOKEN: for (List nextAnd : theAndOrParams) { if ("Location.position".equals(nextParamDef.getPath())) { - andPredicates.add(createPredicateCoords(theSourceJoinColumn, theResourceName, nextParamDef, nextAnd, theRequestPartitionId)); + andPredicates.add(createPredicateCoords(theSourceJoinColumn, theResourceName, null, nextParamDef, nextAnd, theRequestPartitionId, mySqlBuilder)); } else { andPredicates.add(createPredicateToken(theSourceJoinColumn, theResourceName, null, nextParamDef, nextAnd, null, theRequestPartitionId)); } @@ -1519,7 +1746,7 @@ public class QueryStack { case SPECIAL: for (List nextAnd : theAndOrParams) { if ("Location.position".equals(nextParamDef.getPath())) { - andPredicates.add(createPredicateCoords(theSourceJoinColumn, theResourceName, nextParamDef, nextAnd, theRequestPartitionId)); + andPredicates.add(createPredicateCoords(theSourceJoinColumn, theResourceName, null, nextParamDef, nextAnd, theRequestPartitionId, mySqlBuilder)); } } break; @@ -1629,150 +1856,4 @@ public class QueryStack { return qp; } - private enum PredicateBuilderTypeEnum { - DATE, COORDS, NUMBER, QUANTITY, REFERENCE, SOURCE, STRING, TOKEN, TAG - } - - private static class PredicateBuilderCacheLookupResult { - private final boolean myCacheHit; - private final T myResult; - - private PredicateBuilderCacheLookupResult(boolean theCacheHit, T theResult) { - myCacheHit = theCacheHit; - myResult = theResult; - } - - public boolean isCacheHit() { - return myCacheHit; - } - - public T getResult() { - return myResult; - } - } - - private static class PredicateBuilderCacheKey { - private final DbColumn myDbColumn; - private final PredicateBuilderTypeEnum myType; - private final String myParamName; - private final int myHashCode; - - private PredicateBuilderCacheKey(DbColumn theDbColumn, PredicateBuilderTypeEnum theType, String theParamName) { - myDbColumn = theDbColumn; - myType = theType; - myParamName = theParamName; - myHashCode = new HashCodeBuilder().append(myDbColumn).append(myType).append(myParamName).toHashCode(); - } - - @Override - public boolean equals(Object theO) { - if (this == theO) { - return true; - } - - if (theO == null || getClass() != theO.getClass()) { - return false; - } - - PredicateBuilderCacheKey that = (PredicateBuilderCacheKey) theO; - - return new EqualsBuilder() - .append(myDbColumn, that.myDbColumn) - .append(myType, that.myType) - .append(myParamName, that.myParamName) - .isEquals(); - } - - @Override - public int hashCode() { - return myHashCode; - } - } - - @Nullable - public static Condition toAndPredicate(List theAndPredicates) { - List andPredicates = theAndPredicates.stream().filter(t -> t != null).collect(Collectors.toList()); - if (andPredicates.size() == 0) { - return null; - } else if (andPredicates.size() == 1) { - return andPredicates.get(0); - } else { - return ComboCondition.and(andPredicates.toArray(new Condition[0])); - } - } - - @Nullable - public static Condition toOrPredicate(List theOrPredicates) { - List orPredicates = theOrPredicates.stream().filter(t -> t != null).collect(Collectors.toList()); - if (orPredicates.size() == 0) { - return null; - } else if (orPredicates.size() == 1) { - return orPredicates.get(0); - } else { - return ComboCondition.or(orPredicates.toArray(new Condition[0])); - } - } - - @Nullable - public static Condition toOrPredicate(Condition... theOrPredicates) { - return toOrPredicate(Arrays.asList(theOrPredicates)); - } - - @Nullable - public static Condition toAndPredicate(Condition... theAndPredicates) { - return toAndPredicate(Arrays.asList(theAndPredicates)); - } - - @Nonnull - public static Condition toEqualToOrInPredicate(DbColumn theColumn, List theValuePlaceholders, boolean theInverse) { - if (theInverse) { - return toNotEqualToOrNotInPredicate(theColumn, theValuePlaceholders); - } else { - return toEqualToOrInPredicate(theColumn, theValuePlaceholders); - } - } - - @Nonnull - public static Condition toEqualToOrInPredicate(DbColumn theColumn, List theValuePlaceholders) { - if (theValuePlaceholders.size() == 1) { - return BinaryCondition.equalTo(theColumn, theValuePlaceholders.get(0)); - } - return new InCondition(theColumn, theValuePlaceholders); - } - - @Nonnull - public static Condition toNotEqualToOrNotInPredicate(DbColumn theColumn, List theValuePlaceholders) { - if (theValuePlaceholders.size() == 1) { - return BinaryCondition.notEqualTo(theColumn, theValuePlaceholders.get(0)); - } - return new InCondition(theColumn, theValuePlaceholders).setNegate(true); - } - - public static SearchFilterParser.CompareOperation toOperation(ParamPrefixEnum thePrefix) { - SearchFilterParser.CompareOperation retVal = null; - if (thePrefix != null && ourCompareOperationToParamPrefix.containsValue(thePrefix)) { - retVal = ourCompareOperationToParamPrefix.getKey(thePrefix); - } - return defaultIfNull(retVal, SearchFilterParser.CompareOperation.eq); - } - - public static ParamPrefixEnum fromOperation(SearchFilterParser.CompareOperation thePrefix) { - ParamPrefixEnum retVal = null; - if (thePrefix != null && ourCompareOperationToParamPrefix.containsKey(thePrefix)) { - retVal = ourCompareOperationToParamPrefix.get(thePrefix); - } - return defaultIfNull(retVal, ParamPrefixEnum.EQUAL); - } - - private static String getChainedPart(String parameter) { - return parameter.substring(parameter.indexOf(".") + 1); - } - - public static String getParamNameWithPrefix(String theSpnamePrefix, String theParamName) { - - if (isBlank(theSpnamePrefix)) - return theParamName; - - return theSpnamePrefix + "." + theParamName; - } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 97d5b0cefd2..cc52e42c07a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -49,7 +49,6 @@ import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails; import ca.uhn.fhir.jpa.model.config.PartitionSettings; import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity; import ca.uhn.fhir.jpa.model.entity.ModelConfig; -import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.entity.ResourceTag; import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; @@ -66,12 +65,12 @@ import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper; import ca.uhn.fhir.jpa.util.BaseIterator; import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener; import ca.uhn.fhir.jpa.util.QueryChunker; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.util.SqlQueryList; import ca.uhn.fhir.model.api.IQueryParameterType; import ca.uhn.fhir.model.api.IResource; import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum; -import ca.uhn.fhir.model.primitive.InstantDt; import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; @@ -115,10 +114,6 @@ import javax.persistence.Query; import javax.persistence.Tuple; import javax.persistence.TypedQuery; import javax.persistence.criteria.CriteriaBuilder; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.From; -import javax.persistence.criteria.Predicate; -import javax.persistence.criteria.Root; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -160,29 +155,7 @@ public class SearchBuilder implements ISearchBuilder { public static boolean myUseMaxPageSize50ForTest = false; private final String myResourceName; private final Class myResourceType; - private final IDao myCallingDao; - @Autowired - protected IInterceptorBroadcaster myInterceptorBroadcaster; - @Autowired - protected IResourceTagDao myResourceTagDao; - @PersistenceContext(type = PersistenceContextType.TRANSACTION) - protected EntityManager myEntityManager; - @Autowired - private DaoConfig myDaoConfig; - @Autowired - private DaoRegistry myDaoRegistry; - @Autowired - private IResourceSearchViewDao myResourceSearchViewDao; - @Autowired - private FhirContext myContext; - @Autowired - private IIdHelperService myIdHelperService; - @Autowired(required = false) - private IFulltextSearchSvc myFulltextSearchSvc; - @Autowired(required = false) - private IElasticsearchSvc myIElasticsearchSvc; - @Autowired - private ISearchParamRegistry mySearchParamRegistry; + private List myAlsoIncludePids; private CriteriaBuilder myCriteriaBuilder; private SearchParameterMap myParams; @@ -192,25 +165,69 @@ public class SearchBuilder implements ISearchBuilder { private Set myPidSet; private boolean myHasNextIteratorQuery = false; private RequestPartitionId myRequestPartitionId; - @Autowired - private PartitionSettings myPartitionSettings; - @Autowired - private HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory; - @Autowired - private SqlObjectFactory mySqlBuilderFactory; - @Autowired - private HibernatePropertiesProvider myDialectProvider; - @Autowired - private ModelConfig myModelConfig; + @PersistenceContext(type = PersistenceContextType.TRANSACTION) + protected EntityManager myEntityManager; + @Autowired(required = false) + private IFulltextSearchSvc myFulltextSearchSvc; + @Autowired(required = false) + private IElasticsearchSvc myIElasticsearchSvc; + + private final HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory; + private final SqlObjectFactory mySqlBuilderFactory; + private final HibernatePropertiesProvider myDialectProvider; + private final ModelConfig myModelConfig; + private final ISearchParamRegistry mySearchParamRegistry; + private final PartitionSettings myPartitionSettings; + protected final IInterceptorBroadcaster myInterceptorBroadcaster; + protected final IResourceTagDao myResourceTagDao; + private final DaoRegistry myDaoRegistry; + private final IResourceSearchViewDao myResourceSearchViewDao; + private final FhirContext myContext; + private final IIdHelperService myIdHelperService; + + private final DaoConfig myDaoConfig; + + private final IDao myCallingDao; /** * Constructor */ - public SearchBuilder(IDao theDao, String theResourceName, Class theResourceType) { + public SearchBuilder( + IDao theDao, + String theResourceName, + DaoConfig theDaoConfig, + HapiFhirLocalContainerEntityManagerFactoryBean theEntityManagerFactory, + SqlObjectFactory theSqlBuilderFactory, + HibernatePropertiesProvider theDialectProvider, + ModelConfig theModelConfig, + ISearchParamRegistry theSearchParamRegistry, + PartitionSettings thePartitionSettings, + IInterceptorBroadcaster theInterceptorBroadcaster, + IResourceTagDao theResourceTagDao, + DaoRegistry theDaoRegistry, + IResourceSearchViewDao theResourceSearchViewDao, + FhirContext theContext, + IIdHelperService theIdHelperService, + Class theResourceType + ) { myCallingDao = theDao; myResourceName = theResourceName; myResourceType = theResourceType; + myDaoConfig = theDaoConfig; + + myEntityManagerFactory = theEntityManagerFactory; + mySqlBuilderFactory = theSqlBuilderFactory; + myDialectProvider = theDialectProvider; + myModelConfig = theModelConfig; + mySearchParamRegistry = theSearchParamRegistry; + myPartitionSettings = thePartitionSettings; + myInterceptorBroadcaster = theInterceptorBroadcaster; + myResourceTagDao = theResourceTagDao; + myDaoRegistry = theDaoRegistry; + myResourceSearchViewDao = theResourceSearchViewDao; + myContext = theContext; + myIdHelperService = theIdHelperService; } @Override @@ -1233,7 +1250,7 @@ public class SearchBuilder implements ISearchBuilder { if (theReverseMode) { if (theLastUpdated != null && (theLastUpdated.getLowerBoundAsInstant() != null || theLastUpdated.getUpperBoundAsInstant() != null)) { - pidsToInclude = new HashSet<>(filterResourceIdsByLastUpdated(theEntityManager, theLastUpdated, pidsToInclude)); + pidsToInclude = new HashSet<>(QueryParameterUtils.filterResourceIdsByLastUpdated(theEntityManager, theLastUpdated, pidsToInclude)); } } @@ -1452,11 +1469,6 @@ public class SearchBuilder implements ISearchBuilder { return myResourceName; } - @VisibleForTesting - public void setDaoConfigForUnitTest(DaoConfig theDaoConfig) { - myDaoConfig = theDaoConfig; - } - public class IncludesIterator extends BaseIterator implements Iterator { private final RequestDetails myRequest; @@ -1765,42 +1777,4 @@ public class SearchBuilder implements ISearchBuilder { myUseMaxPageSize50ForTest = theIsTest; } - private static List createLastUpdatedPredicates(final DateRangeParam theLastUpdated, CriteriaBuilder builder, From from) { - List lastUpdatedPredicates = new ArrayList<>(); - if (theLastUpdated != null) { - if (theLastUpdated.getLowerBoundAsInstant() != null) { - ourLog.debug("LastUpdated lower bound: {}", new InstantDt(theLastUpdated.getLowerBoundAsInstant())); - Predicate predicateLower = builder.greaterThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getLowerBoundAsInstant()); - lastUpdatedPredicates.add(predicateLower); - } - if (theLastUpdated.getUpperBoundAsInstant() != null) { - Predicate predicateUpper = builder.lessThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getUpperBoundAsInstant()); - lastUpdatedPredicates.add(predicateUpper); - } - } - return lastUpdatedPredicates; - } - - private static List filterResourceIdsByLastUpdated(EntityManager theEntityManager, final DateRangeParam theLastUpdated, Collection thePids) { - if (thePids.isEmpty()) { - return Collections.emptyList(); - } - CriteriaBuilder builder = theEntityManager.getCriteriaBuilder(); - CriteriaQuery cq = builder.createQuery(Long.class); - Root from = cq.from(ResourceTable.class); - cq.select(from.get("myId").as(Long.class)); - - List lastUpdatedPredicates = createLastUpdatedPredicates(theLastUpdated, builder, from); - lastUpdatedPredicates.add(from.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(thePids))); - - cq.where(SearchBuilder.toPredicateArray(lastUpdatedPredicates)); - TypedQuery query = theEntityManager.createQuery(cq); - - return ResourcePersistentId.fromLongList(query.getResultList()); - } - - public static Predicate[] toPredicateArray(List thePredicates) { - return thePredicates.toArray(new Predicate[0]); - } - } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/StorageInterceptorHooksFacade.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/StorageInterceptorHooksFacade.java new file mode 100644 index 00000000000..61111e4df30 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/StorageInterceptorHooksFacade.java @@ -0,0 +1,39 @@ +package ca.uhn.fhir.jpa.search.builder; + +import ca.uhn.fhir.interceptor.api.HookParams; +import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; +import ca.uhn.fhir.interceptor.api.Pointcut; +import ca.uhn.fhir.jpa.entity.Search; +import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; +import ca.uhn.fhir.rest.server.util.ICachedSearchDetails; + +/** + * facade over raw hook intererface + */ +public class StorageInterceptorHooksFacade { + private final IInterceptorBroadcaster myInterceptorBroadcaster; + public StorageInterceptorHooksFacade(IInterceptorBroadcaster theInterceptorBroadcaster) { + myInterceptorBroadcaster = theInterceptorBroadcaster; + } + + /** + * Interceptor call: STORAGE_PRESEARCH_REGISTERED + * + * @param theRequestDetails + * @param theParams + * @param search + */ + public void callStoragePresearchRegistered(RequestDetails theRequestDetails, SearchParameterMap theParams, Search search) { + HookParams params = new HookParams() + .add(ICachedSearchDetails.class, search) + .add(RequestDetails.class, theRequestDetails) + .addIfMatchesType(ServletRequestDetails.class, theRequestDetails) + .add(SearchParameterMap.class, theParams); + CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PRESEARCH_REGISTERED, params); + } + //private IInterceptorBroadcaster myInterceptorBroadcaster; +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/MissingParameterQueryParams.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/MissingParameterQueryParams.java new file mode 100644 index 00000000000..29d812e30ef --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/MissingParameterQueryParams.java @@ -0,0 +1,109 @@ +package ca.uhn.fhir.jpa.search.builder.models; + +import ca.uhn.fhir.i18n.Msg; +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; +import ca.uhn.fhir.model.api.IQueryParameterType; +import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; +import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn; + +import java.security.InvalidParameterException; +import java.util.List; + +public class MissingParameterQueryParams { + /** + * The sql builder + */ + private final SearchQueryBuilder mySqlBuilder; + + /** + * The parameter type + */ + private final RestSearchParameterTypeEnum myParamType; + + /** + * The list of query parameter types (only needed for validation) + */ + private final List myQueryParameterTypes; + + /** + * The missing boolean value from :missing=true/false + */ + private final boolean myIsMissing; + + /** + * The name of the parameter. + */ + private final String myParamName; + + /** + * The resource type + */ + private final String myResourceType; + + /** + * The column on which to join. + */ + private final DbColumn mySourceJoinColumn; + + /** + * The partition id + */ + private final RequestPartitionId myRequestPartitionId; + + public MissingParameterQueryParams( + SearchQueryBuilder theSqlBuilder, + RestSearchParameterTypeEnum theParamType, + List theList, + String theParamName, + String theResourceType, + DbColumn theSourceJoinColumn, + RequestPartitionId theRequestPartitionId + ) { + mySqlBuilder = theSqlBuilder; + myParamType = theParamType; + myQueryParameterTypes = theList; + if (theList.isEmpty()) { + // this will never happen + throw new InvalidParameterException(Msg.code(2140) + + " Invalid search parameter list. Cannot be empty!"); + } + myIsMissing = theList.get(0).getMissing(); + myParamName = theParamName; + myResourceType = theResourceType; + mySourceJoinColumn = theSourceJoinColumn; + myRequestPartitionId = theRequestPartitionId; + } + + public SearchQueryBuilder getSqlBuilder() { + return mySqlBuilder; + } + + public RestSearchParameterTypeEnum getParamType() { + return myParamType; + } + + public List getQueryParameterTypes() { + return myQueryParameterTypes; + } + + public boolean isMissing() { + return myIsMissing; + } + + public String getParamName() { + return myParamName; + } + + public String getResourceType() { + return myResourceType; + } + + public DbColumn getSourceJoinColumn() { + return mySourceJoinColumn; + } + + public ca.uhn.fhir.interceptor.model.RequestPartitionId getRequestPartitionId() { + return myRequestPartitionId; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/MissingQueryParameterPredicateParams.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/MissingQueryParameterPredicateParams.java new file mode 100644 index 00000000000..f409e077154 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/MissingQueryParameterPredicateParams.java @@ -0,0 +1,51 @@ +package ca.uhn.fhir.jpa.search.builder.models; + +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.search.builder.predicate.ResourceTablePredicateBuilder; + +public class MissingQueryParameterPredicateParams { + /** + * Base ResourceTable predicate builder + */ + private ResourceTablePredicateBuilder myResourceTablePredicateBuilder; + /** + * The missing boolean. + * True if looking for missing fields. + * False if looking for non-missing fields. + */ + private boolean myIsMissing; + /** + * The Search Parameter Name + */ + private String myParamName; + /** + * The partition id + */ + private RequestPartitionId myRequestPartitionId; + + public MissingQueryParameterPredicateParams(ca.uhn.fhir.jpa.search.builder.predicate.ResourceTablePredicateBuilder theResourceTablePredicateBuilder, + boolean theTheMissing, + String theParamName, + ca.uhn.fhir.interceptor.model.RequestPartitionId theRequestPartitionId) { + myResourceTablePredicateBuilder = theResourceTablePredicateBuilder; + myIsMissing = theTheMissing; + myParamName = theParamName; + myRequestPartitionId = theRequestPartitionId; + } + + public ca.uhn.fhir.jpa.search.builder.predicate.ResourceTablePredicateBuilder getResourceTablePredicateBuilder() { + return myResourceTablePredicateBuilder; + } + + public boolean isMissing() { + return myIsMissing; + } + + public String getParamName() { + return myParamName; + } + + public ca.uhn.fhir.interceptor.model.RequestPartitionId getRequestPartitionId() { + return myRequestPartitionId; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/PredicateBuilderCacheKey.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/PredicateBuilderCacheKey.java new file mode 100644 index 00000000000..b97fcb70a00 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/PredicateBuilderCacheKey.java @@ -0,0 +1,43 @@ +package ca.uhn.fhir.jpa.search.builder.models; + +import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +public class PredicateBuilderCacheKey { + private final DbColumn myDbColumn; + private final PredicateBuilderTypeEnum myType; + private final String myParamName; + private final int myHashCode; + + public PredicateBuilderCacheKey(DbColumn theDbColumn, PredicateBuilderTypeEnum theType, String theParamName) { + myDbColumn = theDbColumn; + myType = theType; + myParamName = theParamName; + myHashCode = new HashCodeBuilder().append(myDbColumn).append(myType).append(myParamName).toHashCode(); + } + + @Override + public boolean equals(Object theO) { + if (this == theO) { + return true; + } + + if (theO == null || getClass() != theO.getClass()) { + return false; + } + + PredicateBuilderCacheKey that = (PredicateBuilderCacheKey) theO; + + return new EqualsBuilder() + .append(myDbColumn, that.myDbColumn) + .append(myType, that.myType) + .append(myParamName, that.myParamName) + .isEquals(); + } + + @Override + public int hashCode() { + return myHashCode; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/PredicateBuilderCacheLookupResult.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/PredicateBuilderCacheLookupResult.java new file mode 100644 index 00000000000..6e84bcae3fb --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/PredicateBuilderCacheLookupResult.java @@ -0,0 +1,21 @@ +package ca.uhn.fhir.jpa.search.builder.models; + +import ca.uhn.fhir.jpa.search.builder.predicate.BaseJoiningPredicateBuilder; + +public class PredicateBuilderCacheLookupResult { + private final boolean myCacheHit; + private final T myResult; + + public PredicateBuilderCacheLookupResult(boolean theCacheHit, T theResult) { + myCacheHit = theCacheHit; + myResult = theResult; + } + + public boolean isCacheHit() { + return myCacheHit; + } + + public T getResult() { + return myResult; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/PredicateBuilderTypeEnum.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/PredicateBuilderTypeEnum.java new file mode 100644 index 00000000000..725d23700d2 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/PredicateBuilderTypeEnum.java @@ -0,0 +1,5 @@ +package ca.uhn.fhir.jpa.search.builder.models; + +public enum PredicateBuilderTypeEnum { + DATE, COORDS, NUMBER, QUANTITY, REFERENCE, SOURCE, STRING, TOKEN, TAG +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseJoiningPredicateBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseJoiningPredicateBuilder.java index 60136f7acd7..2980ab307fd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseJoiningPredicateBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseJoiningPredicateBuilder.java @@ -22,6 +22,7 @@ package ca.uhn.fhir.jpa.search.builder.predicate; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.model.config.PartitionSettings; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import com.healthmarketscience.sqlbuilder.Condition; import com.healthmarketscience.sqlbuilder.NotCondition; @@ -34,9 +35,9 @@ import javax.annotation.Nullable; import java.util.List; import java.util.stream.Collectors; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toAndPredicate; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toEqualToOrInPredicate; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toOrPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toAndPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toEqualToOrInPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toOrPredicate; public abstract class BaseJoiningPredicateBuilder extends BasePredicateBuilder { @@ -64,7 +65,7 @@ public abstract class BaseJoiningPredicateBuilder extends BasePredicateBuilder { if (partitionIdPredicate == null) { return theCondition; } - return toAndPredicate(partitionIdPredicate, theCondition); + return QueryParameterUtils.toAndPredicate(partitionIdPredicate, theCondition); } @@ -81,14 +82,14 @@ public abstract class BaseJoiningPredicateBuilder extends BasePredicateBuilder { } else if (theRequestPartitionId.hasDefaultPartitionId() && defaultPartitionIsNull) { List placeholders = generatePlaceholders(theRequestPartitionId.getPartitionIdsWithoutDefault()); UnaryCondition partitionNullPredicate = UnaryCondition.isNull(getPartitionIdColumn()); - Condition partitionIdsPredicate = toEqualToOrInPredicate(getPartitionIdColumn(), placeholders); - condition = toOrPredicate(partitionNullPredicate, partitionIdsPredicate); + Condition partitionIdsPredicate = QueryParameterUtils.toEqualToOrInPredicate(getPartitionIdColumn(), placeholders); + condition = QueryParameterUtils.toOrPredicate(partitionNullPredicate, partitionIdsPredicate); } else { List partitionIds = theRequestPartitionId.getPartitionIds(); partitionIds = replaceDefaultPartitionIdIfNonNull(getPartitionSettings(), partitionIds); List placeholders = generatePlaceholders(partitionIds); - condition = toEqualToOrInPredicate(getPartitionIdColumn(), placeholders); + condition = QueryParameterUtils.toEqualToOrInPredicate(getPartitionIdColumn(), placeholders); } return condition; } else { @@ -100,7 +101,7 @@ public abstract class BaseJoiningPredicateBuilder extends BasePredicateBuilder { Validate.notNull(theResourceIds, "theResourceIds must not be null"); // Handle the _id parameter by adding it to the tail - Condition inResourceIds = toEqualToOrInPredicate(getResourceIdColumn(), generatePlaceholders(theResourceIds)); + Condition inResourceIds = QueryParameterUtils.toEqualToOrInPredicate(getResourceIdColumn(), generatePlaceholders(theResourceIds)); if (theInverse) { inResourceIds = new NotCondition(inResourceIds); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseQuantityPredicateBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseQuantityPredicateBuilder.java index 5ae698f97c6..2e039b4a777 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseQuantityPredicateBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseQuantityPredicateBuilder.java @@ -25,7 +25,7 @@ import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser; import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam; import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParamQuantity; -import ca.uhn.fhir.jpa.search.builder.QueryStack; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.QuantityParam; @@ -80,7 +80,7 @@ public abstract class BaseQuantityPredicateBuilder extends BaseSearchParamPredic SearchFilterParser.CompareOperation operation = theOperation; if (operation == null && cmpValue != null) { - operation = QueryStack.toOperation(cmpValue); + operation = QueryParameterUtils.toOperation(cmpValue); } operation = defaultIfNull(operation, SearchFilterParser.CompareOperation.eq); Condition numericPredicate = NumberPredicateBuilder.createPredicateNumeric(this, operation, valueValue, myColumnValue, "invalidQuantityPrefix", myFhirContext, theParam); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseSearchParamPredicateBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseSearchParamPredicateBuilder.java index 496412a207a..2d830d5f88e 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseSearchParamPredicateBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/BaseSearchParamPredicateBuilder.java @@ -22,10 +22,15 @@ package ca.uhn.fhir.jpa.search.builder.predicate; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; +import ca.uhn.fhir.jpa.search.builder.models.MissingQueryParameterPredicateParams; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import com.healthmarketscience.sqlbuilder.BinaryCondition; import com.healthmarketscience.sqlbuilder.ComboCondition; import com.healthmarketscience.sqlbuilder.Condition; +import com.healthmarketscience.sqlbuilder.NotCondition; +import com.healthmarketscience.sqlbuilder.SelectQuery; +import com.healthmarketscience.sqlbuilder.UnaryCondition; import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn; import com.healthmarketscience.sqlbuilder.dbspec.basic.DbTable; @@ -33,9 +38,11 @@ import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.List; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toAndPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toAndPredicate; -public abstract class BaseSearchParamPredicateBuilder extends BaseJoiningPredicateBuilder { +public abstract class BaseSearchParamPredicateBuilder + extends BaseJoiningPredicateBuilder + implements ICanMakeMissingParamPredicate { private final DbColumn myColumnMissing; private final DbColumn myColumnResType; @@ -81,7 +88,7 @@ public abstract class BaseSearchParamPredicateBuilder extends BaseJoiningPredica andPredicates.add(hashIdentityPredicate); andPredicates.add(thePredicate); - return toAndPredicate(andPredicates); + return QueryParameterUtils.toAndPredicate(andPredicates); } @Nonnull @@ -98,6 +105,37 @@ public abstract class BaseSearchParamPredicateBuilder extends BaseJoiningPredica BinaryCondition.equalTo(getMissingColumn(), generatePlaceholder(theMissing)) ); return combineWithRequestPartitionIdPredicate(theRequestPartitionId, condition); + } + + @Override + public Condition createPredicateParamMissingValue(MissingQueryParameterPredicateParams theParams) { + SelectQuery subquery = new SelectQuery(); + subquery.addCustomColumns(1); + subquery.addFromTable(getTable()); + + long hashIdentity = BaseResourceIndexedSearchParam.calculateHashIdentity( + getPartitionSettings(), + theParams.getRequestPartitionId(), + theParams.getResourceTablePredicateBuilder().getResourceType(), + theParams.getParamName() + ); + + Condition subQueryCondition = ComboCondition.and( + BinaryCondition.equalTo(getResourceIdColumn(), + theParams.getResourceTablePredicateBuilder().getResourceIdColumn() + ), + BinaryCondition.equalTo(getColumnHashIdentity(), + generatePlaceholder(hashIdentity)) + ); + + subquery.addCondition(subQueryCondition); + + Condition unaryCondition = UnaryCondition.exists(subquery); + if (theParams.isMissing()) { + unaryCondition = new NotCondition(unaryCondition); + } + + return combineWithRequestPartitionIdPredicate(theParams.getRequestPartitionId(), unaryCondition); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ICanMakeMissingParamPredicate.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ICanMakeMissingParamPredicate.java new file mode 100644 index 00000000000..e0b773e6c33 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ICanMakeMissingParamPredicate.java @@ -0,0 +1,15 @@ +package ca.uhn.fhir.jpa.search.builder.predicate; + +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.search.builder.models.MissingQueryParameterPredicateParams; +import com.healthmarketscience.sqlbuilder.Condition; + +public interface ICanMakeMissingParamPredicate { + /** + * Creates the condition for searching for a missing field + * for a given SearchParameter type. + * + * Only use if {@link DaoConfig#getIndexMissingFields()} is disabled + */ + Condition createPredicateParamMissingValue(MissingQueryParameterPredicateParams theParams); +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceIdPredicateBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceIdPredicateBuilder.java index f9701cb3bd0..dd18d2752c2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceIdPredicateBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceIdPredicateBuilder.java @@ -23,7 +23,7 @@ package ca.uhn.fhir.jpa.search.builder.predicate; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.svc.IIdHelperService; import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser; -import ca.uhn.fhir.jpa.search.builder.QueryStack; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import ca.uhn.fhir.model.api.IQueryParameterType; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; @@ -131,7 +131,7 @@ public class ResourceIdPredicateBuilder extends BasePredicateBuilder { return queryRootTable.combineWithRequestPartitionIdPredicate(theRequestPartitionId, predicate); } } else { - return QueryStack.toEqualToOrInPredicate(theSourceJoinColumn, generatePlaceholders(resourceIds), operation == SearchFilterParser.CompareOperation.ne); + return QueryParameterUtils.toEqualToOrInPredicate(theSourceJoinColumn, generatePlaceholders(resourceIds), operation == SearchFilterParser.CompareOperation.ne); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceLinkPredicateBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceLinkPredicateBuilder.java index c48358c66b9..0a35138657a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceLinkPredicateBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceLinkPredicateBuilder.java @@ -40,7 +40,9 @@ import ca.uhn.fhir.jpa.dao.BaseStorageDao; import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser; import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.search.builder.QueryStack; +import ca.uhn.fhir.jpa.search.builder.models.MissingQueryParameterPredicateParams; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import ca.uhn.fhir.jpa.searchparam.MatchUrlService; import ca.uhn.fhir.jpa.searchparam.ResourceMetaParams; @@ -72,6 +74,9 @@ import com.google.common.collect.Lists; import com.healthmarketscience.sqlbuilder.BinaryCondition; import com.healthmarketscience.sqlbuilder.ComboCondition; import com.healthmarketscience.sqlbuilder.Condition; +import com.healthmarketscience.sqlbuilder.NotCondition; +import com.healthmarketscience.sqlbuilder.SelectQuery; +import com.healthmarketscience.sqlbuilder.UnaryCondition; import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; @@ -91,13 +96,15 @@ import java.util.ListIterator; import java.util.Set; import java.util.stream.Collectors; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toAndPredicate; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toEqualToOrInPredicate; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toOrPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toAndPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toEqualToOrInPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toOrPredicate; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.trim; -public class ResourceLinkPredicateBuilder extends BaseJoiningPredicateBuilder { +public class ResourceLinkPredicateBuilder + extends BaseJoiningPredicateBuilder + implements ICanMakeMissingParamPredicate { private static final Logger ourLog = LoggerFactory.getLogger(ResourceLinkPredicateBuilder.class); private final DbColumn myColumnSrcType; @@ -138,6 +145,14 @@ public class ResourceLinkPredicateBuilder extends BaseJoiningPredicateBuilder { myQueryStack = theQueryStack; } + private DbColumn getResourceTypeColumn() { + if (myReversed) { + return myColumnTargetResourceType; + } else { + return myColumnSrcType; + } + } + public DbColumn getColumnSourcePath() { return myColumnSrcPath; } @@ -241,13 +256,13 @@ public class ResourceLinkPredicateBuilder extends BaseJoiningPredicateBuilder { Condition targetPidCondition = null; if (!theTargetPidList.isEmpty()) { List placeholders = generatePlaceholders(theTargetPidList); - targetPidCondition = toEqualToOrInPredicate(myColumnTargetResourceId, placeholders, theInverse); + targetPidCondition = QueryParameterUtils.toEqualToOrInPredicate(myColumnTargetResourceId, placeholders, theInverse); } Condition targetUrlsCondition = null; if (!theTargetQualifiedUrls.isEmpty()) { List placeholders = generatePlaceholders(theTargetQualifiedUrls); - targetUrlsCondition = toEqualToOrInPredicate(myColumnTargetResourceUrl, placeholders, theInverse); + targetUrlsCondition = QueryParameterUtils.toEqualToOrInPredicate(myColumnTargetResourceUrl, placeholders, theInverse); } Condition joinedCondition; @@ -267,7 +282,7 @@ public class ResourceLinkPredicateBuilder extends BaseJoiningPredicateBuilder { @Nonnull public Condition createPredicateSourcePaths(List thePathsToMatch) { - return toEqualToOrInPredicate(myColumnSrcPath, generatePlaceholders(thePathsToMatch)); + return QueryParameterUtils.toEqualToOrInPredicate(myColumnSrcPath, generatePlaceholders(thePathsToMatch)); } public Condition createPredicateSourcePaths(String theResourceName, String theParamName, List theQualifiers) { @@ -338,7 +353,7 @@ public class ResourceLinkPredicateBuilder extends BaseJoiningPredicateBuilder { Condition condition = BinaryCondition.equalTo(myColumnTargetResourceType, generatePlaceholder(theReferenceParam.getValue())); - return toAndPredicate(typeCondition, condition); + return QueryParameterUtils.toAndPredicate(typeCondition, condition); } boolean foundChainMatch = false; @@ -415,7 +430,7 @@ public class ResourceLinkPredicateBuilder extends BaseJoiningPredicateBuilder { List> chainParamValues = Collections.singletonList(orValues); andPredicates.add(childQueryFactory.searchForIdsWithAndOr(myColumnTargetResourceId, subResourceName, chain, chainParamValues, theRequest, theRequestPartitionId, SearchContainedModeEnum.FALSE)); - orPredicates.add(toAndPredicate(andPredicates)); + orPredicates.add(QueryParameterUtils.toAndPredicate(andPredicates)); } if (candidateTargetTypes.isEmpty()) { @@ -429,14 +444,14 @@ public class ResourceLinkPredicateBuilder extends BaseJoiningPredicateBuilder { // If :not modifier for a token, switch OR with AND in the multi-type case Condition multiTypePredicate; if (paramInverted) { - multiTypePredicate = toAndPredicate(orPredicates); + multiTypePredicate = QueryParameterUtils.toAndPredicate(orPredicates); } else { - multiTypePredicate = toOrPredicate(orPredicates); + multiTypePredicate = QueryParameterUtils.toOrPredicate(orPredicates); } List pathsToMatch = createResourceLinkPaths(theResourceName, theParamName, theQualifiers); Condition pathPredicate = createPredicateSourcePaths(pathsToMatch); - return toAndPredicate(pathPredicate, multiTypePredicate); + return QueryParameterUtils.toAndPredicate(pathPredicate, multiTypePredicate); } @Nonnull @@ -695,7 +710,7 @@ public class ResourceLinkPredicateBuilder extends BaseJoiningPredicateBuilder { if (theTargetPids != null && theTargetPids.length >= 1) { // if resource ids are provided, we'll create the predicate // with ids in or equal to this value - condition = toEqualToOrInPredicate(myColumnTargetResourceId, generatePlaceholders(Arrays.asList(theTargetPids))); + condition = QueryParameterUtils.toEqualToOrInPredicate(myColumnTargetResourceId, generatePlaceholders(Arrays.asList(theTargetPids))); } else { // ... otherwise we look for resource types condition = BinaryCondition.equalTo(myColumnTargetResourceType, generatePlaceholder(theResourceName)); @@ -703,10 +718,35 @@ public class ResourceLinkPredicateBuilder extends BaseJoiningPredicateBuilder { if (!theSourceResourceNames.isEmpty()) { // if source resources are provided, add on predicate for _type operation - Condition typeCondition = toEqualToOrInPredicate(myColumnSrcType, generatePlaceholders(theSourceResourceNames)); - condition = toAndPredicate(List.of(condition, typeCondition)); + Condition typeCondition = QueryParameterUtils.toEqualToOrInPredicate(myColumnSrcType, generatePlaceholders(theSourceResourceNames)); + condition = QueryParameterUtils.toAndPredicate(List.of(condition, typeCondition)); } return condition; } + + + @Override + public Condition createPredicateParamMissingValue(MissingQueryParameterPredicateParams theParams) { + SelectQuery subquery = new SelectQuery(); + subquery.addCustomColumns(1); + subquery.addFromTable(getTable()); + + Condition subQueryCondition = ComboCondition.and( + BinaryCondition.equalTo(getResourceIdColumn(), + theParams.getResourceTablePredicateBuilder().getResourceIdColumn() + ), + BinaryCondition.equalTo(getResourceTypeColumn(), + generatePlaceholder(theParams.getResourceTablePredicateBuilder().getResourceType())) + ); + + subquery.addCondition(subQueryCondition); + + Condition unaryCondition = UnaryCondition.exists(subquery); + if (theParams.isMissing()) { + unaryCondition = new NotCondition(unaryCondition); + } + + return combineWithRequestPartitionIdPredicate(theParams.getRequestPartitionId(), unaryCondition); + } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceTablePredicateBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceTablePredicateBuilder.java index 6a3e3bb3fa1..ab5a2b87db9 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceTablePredicateBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/ResourceTablePredicateBuilder.java @@ -20,6 +20,7 @@ package ca.uhn.fhir.jpa.search.builder.predicate; * #L% */ +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import com.healthmarketscience.sqlbuilder.BinaryCondition; import com.healthmarketscience.sqlbuilder.Condition; @@ -29,8 +30,8 @@ import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn; import java.util.Set; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toAndPredicate; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toEqualToOrInPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toAndPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toEqualToOrInPredicate; public class ResourceTablePredicateBuilder extends BaseJoiningPredicateBuilder { private final DbColumn myColumnResId; @@ -62,10 +63,10 @@ public class ResourceTablePredicateBuilder extends BaseJoiningPredicateBuilder { if (getResourceType() != null) { typePredicate = BinaryCondition.equalTo(myColumnResType, generatePlaceholder(getResourceType())); } - return toAndPredicate( - typePredicate, - UnaryCondition.isNull(myColumnResDeletedAt) - ); + return QueryParameterUtils.toAndPredicate( + typePredicate, + UnaryCondition.isNull(myColumnResDeletedAt) + ); } public DbColumn getLastUpdatedColumn() { @@ -73,7 +74,7 @@ public class ResourceTablePredicateBuilder extends BaseJoiningPredicateBuilder { } public Condition createLanguagePredicate(Set theValues, boolean theNegated) { - Condition condition = toEqualToOrInPredicate(myColumnLanguage, generatePlaceholders(theValues)); + Condition condition = QueryParameterUtils.toEqualToOrInPredicate(myColumnLanguage, generatePlaceholders(theValues)); if (theNegated) { condition = new NotCondition(condition); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/StringPredicateBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/StringPredicateBuilder.java index 52861d4c6d5..2719d761d2f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/StringPredicateBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/StringPredicateBuilder.java @@ -25,8 +25,8 @@ import ca.uhn.fhir.context.RuntimeSearchParam; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser; import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; -import ca.uhn.fhir.jpa.search.builder.QueryStack; import ca.uhn.fhir.model.api.IPrimitiveDatatype; import ca.uhn.fhir.model.api.IQueryParameterType; import ca.uhn.fhir.rest.param.StringParam; @@ -82,7 +82,7 @@ public class StringPredicateBuilder extends BaseSearchParamPredicateBuilder { StringPredicateBuilder theFrom, SearchFilterParser.CompareOperation operation) { String rawSearchTerm; - String paramName = QueryStack.getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); + String paramName = QueryParameterUtils.getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); if (theParameter instanceof TokenParam) { TokenParam id = (TokenParam) theParameter; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/TokenPredicateBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/TokenPredicateBuilder.java index 7d7befa696a..0b824a2e2af 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/TokenPredicateBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/TokenPredicateBuilder.java @@ -37,7 +37,7 @@ import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser; import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam; import ca.uhn.fhir.jpa.model.entity.ModelConfig; import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken; -import ca.uhn.fhir.jpa.search.builder.QueryStack; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import ca.uhn.fhir.jpa.term.api.ITermReadSvc; import ca.uhn.fhir.model.api.IQueryParameterType; @@ -67,9 +67,9 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toAndPredicate; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toEqualToOrInPredicate; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toOrPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toAndPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toEqualToOrInPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toOrPredicate; import static org.apache.commons.lang3.StringUtils.defaultIfBlank; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -136,7 +136,7 @@ public class TokenPredicateBuilder extends BaseSearchParamPredicateBuilder { final List codes = new ArrayList<>(); - String paramName = QueryStack.getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); + String paramName = QueryParameterUtils.getParamNameWithPrefix(theSpnamePrefix, theSearchParam.getName()); SearchFilterParser.CompareOperation operation = theOperation; @@ -248,7 +248,7 @@ public class TokenPredicateBuilder extends BaseSearchParamPredicateBuilder { Condition hashIdentityPredicate = BinaryCondition.equalTo(getColumnHashIdentity(), generatePlaceholder(hashIdentity)); Condition hashValuePredicate = createPredicateOrList(theResourceName, paramName, sortedCodesList, false); - predicate = toAndPredicate(hashIdentityPredicate, hashValuePredicate); + predicate = QueryParameterUtils.toAndPredicate(hashIdentityPredicate, hashValuePredicate); } else { @@ -383,7 +383,7 @@ public class TokenPredicateBuilder extends BaseSearchParamPredicateBuilder { if (!haveMultipleColumns && conditions.length > 1) { List values = Arrays.asList(hashes); - return toEqualToOrInPredicate(columns[0], generatePlaceholders(values), !theWantEquals); + return QueryParameterUtils.toEqualToOrInPredicate(columns[0], generatePlaceholders(values), !theWantEquals); } for (int i = 0; i < conditions.length; i++) { @@ -396,9 +396,9 @@ public class TokenPredicateBuilder extends BaseSearchParamPredicateBuilder { } if (conditions.length > 1) { if (theWantEquals) { - return toOrPredicate(conditions); + return QueryParameterUtils.toOrPredicate(conditions); } else { - return toAndPredicate(conditions); + return QueryParameterUtils.toAndPredicate(conditions); } } else { return conditions[0]; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/UriPredicateBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/UriPredicateBuilder.java index 0b984a6d639..8a1f5a845da 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/UriPredicateBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/UriPredicateBuilder.java @@ -28,6 +28,7 @@ import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao; import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser; import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri; import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; import ca.uhn.fhir.model.api.IQueryParameterType; @@ -47,7 +48,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import static ca.uhn.fhir.jpa.search.builder.QueryStack.toEqualToOrInPredicate; +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.toEqualToOrInPredicate; import static ca.uhn.fhir.jpa.search.builder.predicate.StringPredicateBuilder.createLeftAndRightMatchLikeExpression; import static ca.uhn.fhir.jpa.search.builder.predicate.StringPredicateBuilder.createLeftMatchLikeExpression; import static ca.uhn.fhir.jpa.search.builder.predicate.StringPredicateBuilder.createRightMatchLikeExpression; @@ -127,7 +128,7 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder { continue; } - Condition uriPredicate = toEqualToOrInPredicate(myColumnUri, generatePlaceholders(toFind)); + Condition uriPredicate = QueryParameterUtils.toEqualToOrInPredicate(myColumnUri, generatePlaceholders(toFind)); Condition hashAndUriPredicate = combineWithHashIdentityPredicate(getResourceType(), theParamName, uriPredicate); codePredicates.add(hashAndUriPredicate); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/PredicateBuilderFactory.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/PredicateBuilderFactory.java new file mode 100644 index 00000000000..f45dfb1376d --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/PredicateBuilderFactory.java @@ -0,0 +1,92 @@ +package ca.uhn.fhir.jpa.search.builder.sql; + +import ca.uhn.fhir.jpa.search.builder.QueryStack; +import ca.uhn.fhir.jpa.search.builder.predicate.CoordsPredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.predicate.DatePredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.predicate.ICanMakeMissingParamPredicate; +import ca.uhn.fhir.jpa.search.builder.predicate.NumberPredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.predicate.QuantityPredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.predicate.ResourceLinkPredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.predicate.StringPredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.predicate.TokenPredicateBuilder; +import ca.uhn.fhir.jpa.search.builder.predicate.UriPredicateBuilder; +import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PredicateBuilderFactory { + + private static final Logger ourLog = LoggerFactory.getLogger(PredicateBuilderFactory.class); + + public static ICanMakeMissingParamPredicate createPredicateBuilderForParamType( + RestSearchParameterTypeEnum theParamType, + SearchQueryBuilder theBuilder, + QueryStack theQueryStack + ) { + switch (theParamType) { + case NUMBER: + return createNumberPredicateBuilder(theBuilder); + case DATE: + return createDatePredicateBuilder(theBuilder); + case STRING: + return createStringPredicateBuilder(theBuilder); + case TOKEN: + return createTokenPredicateBuilder(theBuilder); + case QUANTITY: + return createQuantityPredicateBuilder(theBuilder); + case URI: + return createUriPredicateBuilder(theBuilder); + case REFERENCE: + return createReferencePredicateBuilder(theQueryStack, theBuilder); + case HAS: + case SPECIAL: + return createCoordsPredicateBuilder(theBuilder); + case COMPOSITE: + default: + // we don't expect to see this + ourLog.error("Invalid param type " + theParamType.name()); + return null; + } + } + + private static StringPredicateBuilder createStringPredicateBuilder(SearchQueryBuilder theBuilder) { + StringPredicateBuilder sp = theBuilder.getSqlBuilderFactory().stringIndexTable(theBuilder); + return sp; + } + + private static NumberPredicateBuilder createNumberPredicateBuilder(SearchQueryBuilder theBuilder) { + NumberPredicateBuilder np = theBuilder.getSqlBuilderFactory().numberIndexTable(theBuilder); + return np; + } + + private static QuantityPredicateBuilder createQuantityPredicateBuilder(SearchQueryBuilder theBuilder) { + QuantityPredicateBuilder qp = theBuilder.getSqlBuilderFactory().quantityIndexTable(theBuilder); + return qp; + } + + private static CoordsPredicateBuilder createCoordsPredicateBuilder(SearchQueryBuilder theBuilder) { + CoordsPredicateBuilder cp = theBuilder.getSqlBuilderFactory().coordsPredicateBuilder(theBuilder); + return cp; + } + + private static TokenPredicateBuilder createTokenPredicateBuilder(SearchQueryBuilder theBuilder) { + TokenPredicateBuilder tp = theBuilder.getSqlBuilderFactory().tokenIndexTable(theBuilder); + return tp; + } + + private static DatePredicateBuilder createDatePredicateBuilder(SearchQueryBuilder theBuilder) { + DatePredicateBuilder dp = theBuilder.getSqlBuilderFactory().dateIndexTable(theBuilder); + return dp; + } + + private static UriPredicateBuilder createUriPredicateBuilder(SearchQueryBuilder theBuilder) { + UriPredicateBuilder up = theBuilder.getSqlBuilderFactory().uriIndexTable(theBuilder); + return up; + } + + private static ResourceLinkPredicateBuilder createReferencePredicateBuilder(QueryStack theQueryStack, SearchQueryBuilder theBuilder) { + ResourceLinkPredicateBuilder retVal = theBuilder.getSqlBuilderFactory().referenceIndexTable(theQueryStack, theBuilder, false); + return retVal; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java index 01c44545b74..d35d885b198 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java @@ -217,21 +217,21 @@ public class SearchQueryBuilder { * Add and return a predicate builder (or a root query if no root query exists yet) for selecting on a QUANTITY search parameter */ public QuantityPredicateBuilder addQuantityPredicateBuilder(@Nullable DbColumn theSourceJoinColumn) { - + QuantityPredicateBuilder retVal = mySqlBuilderFactory.quantityIndexTable(this); addTable(retVal, theSourceJoinColumn); - + return retVal; } public QuantityNormalizedPredicateBuilder addQuantityNormalizedPredicateBuilder(@Nullable DbColumn theSourceJoinColumn) { - + QuantityNormalizedPredicateBuilder retVal = mySqlBuilderFactory.quantityNormalizedIndexTable(this); addTable(retVal, theSourceJoinColumn); - + return retVal; } - + /** * Add and return a predicate builder (or a root query if no root query exists yet) for selecting on a _source search parameter */ @@ -305,6 +305,9 @@ public class SearchQueryBuilder { return retVal; } + public SqlObjectFactory getSqlBuilderFactory() { + return mySqlBuilderFactory; + } public ResourceIdPredicateBuilder newResourceIdBuilder() { return mySqlBuilderFactory.resourceId(this); @@ -395,7 +398,7 @@ public class SearchQueryBuilder { if (maxResultsToFetch != null || offset != null) { maxResultsToFetch = defaultIfNull(maxResultsToFetch, 10000); - + AbstractLimitHandler limitHandler = (AbstractLimitHandler) myDialect.getLimitHandler(); RowSelection selection = new RowSelection(); selection.setFirstRow(offset); @@ -598,15 +601,15 @@ public class SearchQueryBuilder { } public void excludeResourceIdsPredicate(Set theExsitinghPidSetToExclude) { - + // Do nothing if it's empty if (theExsitinghPidSetToExclude == null || theExsitinghPidSetToExclude.isEmpty()) return; - + List excludePids = ResourcePersistentId.toLongList(theExsitinghPidSetToExclude); - + ourLog.trace("excludePids = " + excludePids); - + DbColumn resourceIdColumn = getOrCreateFirstPredicateBuilder().getResourceIdColumn(); InCondition predicate = new InCondition(resourceIdColumn, generatePlaceholders(excludePids)); predicate.setNegate(true); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchContinuationTask.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchContinuationTask.java new file mode 100644 index 00000000000..f5ba198437f --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchContinuationTask.java @@ -0,0 +1,85 @@ +package ca.uhn.fhir.jpa.search.builder.tasks; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; +import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.search.ExceptionService; +import ca.uhn.fhir.jpa.search.SearchStrategyFactory; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; +import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import ca.uhn.fhir.rest.server.IPagingProvider; +import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.util.List; + +public class SearchContinuationTask extends SearchTask { + + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchContinuationTask.class); + + private final ExceptionService myExceptionSvc; + + public SearchContinuationTask( + SearchTaskParameters theCreationParams, + PlatformTransactionManager theManagedTxManager, + FhirContext theContext, + SearchStrategyFactory theSearchStrategyFactory, + IInterceptorBroadcaster theInterceptorBroadcaster, + SearchBuilderFactory theSearchBuilderFactory, + ISearchResultCacheSvc theSearchResultCacheSvc, + DaoConfig theDaoConfig, + ISearchCacheSvc theSearchCacheSvc, + IPagingProvider thePagingProvider, + ExceptionService theExceptionSvc + ) { + super( + theCreationParams, + theManagedTxManager, + theContext, + theSearchStrategyFactory, + theInterceptorBroadcaster, + theSearchBuilderFactory, + theSearchResultCacheSvc, + theDaoConfig, + theSearchCacheSvc, + thePagingProvider + ); + + myExceptionSvc = theExceptionSvc; + } + + @Override + public Void call() { + try { + TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); + txTemplate.afterPropertiesSet(); + txTemplate.execute(t -> { + List previouslyAddedResourcePids = mySearchResultCacheSvc.fetchAllResultPids(getSearch()); + if (previouslyAddedResourcePids == null) { + throw myExceptionSvc.newUnknownSearchException(getSearch().getUuid()); + } + + ourLog.trace("Have {} previously added IDs in search: {}", previouslyAddedResourcePids.size(), getSearch().getUuid()); + setPreviouslyAddedResourcePids(previouslyAddedResourcePids); + return null; + }); + } catch (Throwable e) { + ourLog.error("Failure processing search", e); + getSearch().setFailureMessage(e.getMessage()); + getSearch().setStatus(SearchStatusEnum.FAILED); + if (e instanceof BaseServerResponseException) { + getSearch().setFailureCode(((BaseServerResponseException) e).getStatusCode()); + } + + saveSearch(); + return null; + } + + return super.call(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTask.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTask.java new file mode 100644 index 00000000000..14de79b4813 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTask.java @@ -0,0 +1,666 @@ +package ca.uhn.fhir.jpa.search.builder.tasks; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.i18n.Msg; +import ca.uhn.fhir.interceptor.api.HookParams; +import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; +import ca.uhn.fhir.interceptor.api.Pointcut; +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.api.dao.IDao; +import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; +import ca.uhn.fhir.jpa.dao.IResultIterator; +import ca.uhn.fhir.jpa.dao.ISearchBuilder; +import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; +import ca.uhn.fhir.jpa.entity.Search; +import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails; +import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; +import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.search.ExceptionService; +import ca.uhn.fhir.jpa.search.SearchStrategyFactory; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; +import ca.uhn.fhir.jpa.util.SearchParameterMapCalculator; +import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import ca.uhn.fhir.rest.server.IPagingProvider; +import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; +import ca.uhn.fhir.util.AsyncUtil; +import ca.uhn.fhir.util.StopWatch; +import co.elastic.apm.api.ElasticApm; +import co.elastic.apm.api.Span; +import co.elastic.apm.api.Transaction; +import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.springframework.orm.jpa.JpaDialect; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.vendor.HibernateJpaDialect; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallbackWithoutResult; +import org.springframework.transaction.support.TransactionTemplate; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static ca.uhn.fhir.jpa.util.QueryParameterUtils.UNIT_TEST_CAPTURE_STACK; +import static ca.uhn.fhir.jpa.util.SearchParameterMapCalculator.isWantCount; +import static ca.uhn.fhir.jpa.util.SearchParameterMapCalculator.isWantOnlyCount; +import static java.util.Objects.nonNull; +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; + +/** + * A search task is a Callable task that runs in + * a thread pool to handle an individual search. One instance + * is created for any requested search and runs from the + * beginning to the end of the search. + *

+ * Understand: + * This class executes in its own thread separate from the + * web server client thread that made the request. We do that + * so that we can return to the client as soon as possible, + * but keep the search going in the background (and have + * the next page of results ready to go when the client asks). + */ +public class SearchTask implements Callable { + + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchTask.class); + + private final SearchParameterMap myParams; + private final IDao myCallingDao; + private final String myResourceType; + private final ArrayList mySyncedPids = new ArrayList<>(); + private final CountDownLatch myInitialCollectionLatch = new CountDownLatch(1); + private final CountDownLatch myCompletionLatch; + private final ArrayList myUnsyncedPids = new ArrayList<>(); + private final RequestDetails myRequest; + private final RequestPartitionId myRequestPartitionId; + private final SearchRuntimeDetails mySearchRuntimeDetails; + private final Transaction myParentTransaction; + private Search mySearch; + private boolean myAbortRequested; + private int myCountSavedTotal = 0; + private int myCountSavedThisPass = 0; + private int myCountBlockedThisPass = 0; + private boolean myAdditionalPrefetchThresholdsRemaining; + private List myPreviouslyAddedResourcePids; + private Integer myMaxResultsToFetch; + + private final Consumer myOnRemove; + + private final int mySyncSize; + private final Integer myLoadingThrottleForUnitTests; + + private boolean myCustomIsolationSupported; + + // injected beans + protected final PlatformTransactionManager myManagedTxManager; + protected final FhirContext myContext; + private final IInterceptorBroadcaster myInterceptorBroadcaster; + private final SearchBuilderFactory mySearchBuilderFactory; + protected final ISearchResultCacheSvc mySearchResultCacheSvc; + private final DaoConfig myDaoConfig; + private final ISearchCacheSvc mySearchCacheSvc; + private final IPagingProvider myPagingProvider; + + /** + * Constructor + */ + public SearchTask( + SearchTaskParameters theCreationParams, + PlatformTransactionManager theManagedTxManager, + FhirContext theContext, + SearchStrategyFactory theSearchStrategyFactory, + IInterceptorBroadcaster theInterceptorBroadcaster, + SearchBuilderFactory theSearchBuilderFactory, + ISearchResultCacheSvc theSearchResultCacheSvc, + DaoConfig theDaoConfig, + ISearchCacheSvc theSearchCacheSvc, + IPagingProvider thePagingProvider + ) { + // beans + myManagedTxManager = theManagedTxManager; + myContext = theContext; + myInterceptorBroadcaster = theInterceptorBroadcaster; + mySearchBuilderFactory = theSearchBuilderFactory; + mySearchResultCacheSvc = theSearchResultCacheSvc; + myDaoConfig = theDaoConfig; + mySearchCacheSvc = theSearchCacheSvc; + myPagingProvider = thePagingProvider; + + // values + myOnRemove = theCreationParams.OnRemove; + mySearch = theCreationParams.Search; + myCallingDao = theCreationParams.CallingDao; + myParams = theCreationParams.Params; + myResourceType = theCreationParams.ResourceType; + myRequest = theCreationParams.Request; + myCompletionLatch = new CountDownLatch(1); + mySyncSize = theCreationParams.SyncSize; + myLoadingThrottleForUnitTests = theCreationParams.getLoadingThrottleForUnitTests(); + + mySearchRuntimeDetails = new SearchRuntimeDetails(myRequest, mySearch.getUuid()); + mySearchRuntimeDetails.setQueryString(myParams.toNormalizedQueryString(myCallingDao.getContext())); + myRequestPartitionId = theCreationParams.RequestPartitionId; + myParentTransaction = ElasticApm.currentTransaction(); + + if (myManagedTxManager instanceof JpaTransactionManager) { + JpaDialect jpaDialect = ((JpaTransactionManager) myManagedTxManager).getJpaDialect(); + if (jpaDialect instanceof HibernateJpaDialect) { + myCustomIsolationSupported = true; + } + } + + if (!myCustomIsolationSupported) { + ourLog.warn("JPA dialect does not support transaction isolation! This can have an impact on search performance."); + } + } + + /** + * This method is called by the server HTTP thread, and + * will block until at least one page of results have been + * fetched from the DB, and will never block after that. + */ + public Integer awaitInitialSync() { + ourLog.trace("Awaiting initial sync"); + do { + ourLog.trace("Search {} aborted: {}", getSearch().getUuid(), !isNotAborted()); + if (AsyncUtil.awaitLatchAndThrowInternalErrorExceptionOnInterrupt(getInitialCollectionLatch(), 250L, TimeUnit.MILLISECONDS)) { + break; + } + } while (getSearch().getStatus() == SearchStatusEnum.LOADING); + ourLog.trace("Initial sync completed"); + + return getSearch().getTotalCount(); + } + + public Search getSearch() { + return mySearch; + } + + public CountDownLatch getInitialCollectionLatch() { + return myInitialCollectionLatch; + } + + public void setPreviouslyAddedResourcePids(List thePreviouslyAddedResourcePids) { + myPreviouslyAddedResourcePids = thePreviouslyAddedResourcePids; + myCountSavedTotal = myPreviouslyAddedResourcePids.size(); + } + + private ISearchBuilder newSearchBuilder() { + Class resourceTypeClass = myContext.getResourceDefinition(myResourceType).getImplementingClass(); + return mySearchBuilderFactory.newSearchBuilder(myCallingDao, myResourceType, resourceTypeClass); + } + + @Nonnull + public List getResourcePids(int theFromIndex, int theToIndex) { + ourLog.debug("Requesting search PIDs from {}-{}", theFromIndex, theToIndex); + + boolean keepWaiting; + do { + synchronized (mySyncedPids) { + ourLog.trace("Search status is {}", mySearch.getStatus()); + boolean haveEnoughResults = mySyncedPids.size() >= theToIndex; + if (!haveEnoughResults) { + switch (mySearch.getStatus()) { + case LOADING: + keepWaiting = true; + break; + case PASSCMPLET: + /* + * If we get here, it means that the user requested resources that crossed the + * current pre-fetch boundary. For example, if the prefetch threshold is 50 and the + * user has requested resources 0-60, then they would get 0-50 back but the search + * coordinator would then stop searching.SearchCoordinatorSvcImplTest + */ + keepWaiting = false; + break; + case FAILED: + case FINISHED: + case GONE: + default: + keepWaiting = false; + break; + } + } else { + keepWaiting = false; + } + } + + if (keepWaiting) { + ourLog.info("Waiting as we only have {} results - Search status: {}", mySyncedPids.size(), mySearch.getStatus()); + AsyncUtil.sleep(500L); + } + } while (keepWaiting); + + ourLog.debug("Proceeding, as we have {} results", mySyncedPids.size()); + + ArrayList retVal = new ArrayList<>(); + synchronized (mySyncedPids) { + QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(mySearch); + + int toIndex = theToIndex; + if (mySyncedPids.size() < toIndex) { + toIndex = mySyncedPids.size(); + } + for (int i = theFromIndex; i < toIndex; i++) { + retVal.add(mySyncedPids.get(i)); + } + } + + ourLog.trace("Done syncing results - Wanted {}-{} and returning {} of {}", theFromIndex, theToIndex, retVal.size(), mySyncedPids.size()); + + return retVal; + } + + public void saveSearch() { + TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + txTemplate.execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theArg0) { + doSaveSearch(); + } + + }); + } + + private void saveUnsynced(final IResultIterator theResultIter) { + TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); + txTemplate.execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theArg0) { + if (mySearch.getId() == null) { + doSaveSearch(); + } + + ArrayList unsyncedPids = myUnsyncedPids; + int countBlocked = 0; + + // Interceptor call: STORAGE_PREACCESS_RESOURCES + // This can be used to remove results from the search result details before + // the user has a chance to know that they were in the results + if (mySearchRuntimeDetails.getRequestDetails() != null && unsyncedPids.isEmpty() == false) { + JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(unsyncedPids, () -> newSearchBuilder()); + HookParams params = new HookParams() + .add(IPreResourceAccessDetails.class, accessDetails) + .add(RequestDetails.class, mySearchRuntimeDetails.getRequestDetails()) + .addIfMatchesType(ServletRequestDetails.class, mySearchRuntimeDetails.getRequestDetails()); + CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params); + + for (int i = unsyncedPids.size() - 1; i >= 0; i--) { + if (accessDetails.isDontReturnResourceAtIndex(i)) { + unsyncedPids.remove(i); + myCountBlockedThisPass++; + myCountSavedTotal++; + countBlocked++; + } + } + } + + // Actually store the results in the query cache storage + myCountSavedTotal += unsyncedPids.size(); + myCountSavedThisPass += unsyncedPids.size(); + mySearchResultCacheSvc.storeResults(mySearch, mySyncedPids, unsyncedPids); + + synchronized (mySyncedPids) { + int numSyncedThisPass = unsyncedPids.size(); + ourLog.trace("Syncing {} search results - Have more: {}", numSyncedThisPass, theResultIter.hasNext()); + mySyncedPids.addAll(unsyncedPids); + unsyncedPids.clear(); + + if (theResultIter.hasNext() == false) { + int skippedCount = theResultIter.getSkippedCount(); + int nonSkippedCount = theResultIter.getNonSkippedCount(); + int totalFetched = skippedCount + myCountSavedThisPass + myCountBlockedThisPass; + ourLog.trace("MaxToFetch[{}] SkippedCount[{}] CountSavedThisPass[{}] CountSavedThisTotal[{}] AdditionalPrefetchRemaining[{}]", myMaxResultsToFetch, skippedCount, myCountSavedThisPass, myCountSavedTotal, myAdditionalPrefetchThresholdsRemaining); + + if (nonSkippedCount == 0 || (myMaxResultsToFetch != null && totalFetched < myMaxResultsToFetch)) { + ourLog.trace("Setting search status to FINISHED"); + mySearch.setStatus(SearchStatusEnum.FINISHED); + mySearch.setTotalCount(myCountSavedTotal - countBlocked); + } else if (myAdditionalPrefetchThresholdsRemaining) { + ourLog.trace("Setting search status to PASSCMPLET"); + mySearch.setStatus(SearchStatusEnum.PASSCMPLET); + mySearch.setSearchParameterMap(myParams); + } else { + ourLog.trace("Setting search status to FINISHED"); + mySearch.setStatus(SearchStatusEnum.FINISHED); + mySearch.setTotalCount(myCountSavedTotal - countBlocked); + } + } + } + + mySearch.setNumFound(myCountSavedTotal); + mySearch.setNumBlocked(mySearch.getNumBlocked() + countBlocked); + + int numSynced; + synchronized (mySyncedPids) { + numSynced = mySyncedPids.size(); + } + + if (myDaoConfig.getCountSearchResultsUpTo() == null || + myDaoConfig.getCountSearchResultsUpTo() <= 0 || + myDaoConfig.getCountSearchResultsUpTo() <= numSynced) { + myInitialCollectionLatch.countDown(); + } + + doSaveSearch(); + + ourLog.trace("saveUnsynced() - pre-commit"); + } + }); + ourLog.trace("saveUnsynced() - post-commit"); + + } + + public boolean isNotAborted() { + return myAbortRequested == false; + } + + public void markComplete() { + myCompletionLatch.countDown(); + } + + public CountDownLatch getCompletionLatch() { + return myCompletionLatch; + } + + /** + * Request that the task abort as soon as possible + */ + public void requestImmediateAbort() { + myAbortRequested = true; + } + + /** + * This is the method which actually performs the search. + * It is called automatically by the thread pool. + */ + @Override + public Void call() { + StopWatch sw = new StopWatch(); + Span span = myParentTransaction.startSpan("db", "query", "search"); + span.setName("FHIR Database Search"); + try { + // Create an initial search in the DB and give it an ID + saveSearch(); + + TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); + + if (myCustomIsolationSupported) { + txTemplate.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED); + } + + txTemplate.execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theStatus) { + doSearch(); + } + }); + + mySearchRuntimeDetails.setSearchStatus(mySearch.getStatus()); + if (mySearch.getStatus() == SearchStatusEnum.FINISHED) { + HookParams params = new HookParams() + .add(RequestDetails.class, myRequest) + .addIfMatchesType(ServletRequestDetails.class, myRequest) + .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); + CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_COMPLETE, params); + } else { + HookParams params = new HookParams() + .add(RequestDetails.class, myRequest) + .addIfMatchesType(ServletRequestDetails.class, myRequest) + .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); + CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_PASS_COMPLETE, params); + } + + ourLog.trace("Have completed search for [{}{}] and found {} resources in {}ms - Status is {}", mySearch.getResourceType(), mySearch.getSearchQueryString(), mySyncedPids.size(), sw.getMillis(), mySearch.getStatus()); + + } catch (Throwable t) { + + /* + * Don't print a stack trace for client errors (i.e. requests that + * aren't valid because the client screwed up).. that's just noise + * in the logs and who needs that. + */ + boolean logged = false; + if (t instanceof BaseServerResponseException) { + BaseServerResponseException exception = (BaseServerResponseException) t; + if (exception.getStatusCode() >= 400 && exception.getStatusCode() < 500) { + logged = true; + ourLog.warn("Failed during search due to invalid request: {}", t.toString()); + } + } + + if (!logged) { + ourLog.error("Failed during search loading after {}ms", sw.getMillis(), t); + } + myUnsyncedPids.clear(); + Throwable rootCause = ExceptionUtils.getRootCause(t); + rootCause = defaultIfNull(rootCause, t); + + String failureMessage = rootCause.getMessage(); + + int failureCode = InternalErrorException.STATUS_CODE; + if (t instanceof BaseServerResponseException) { + failureCode = ((BaseServerResponseException) t).getStatusCode(); + } + + if (System.getProperty(UNIT_TEST_CAPTURE_STACK) != null) { + failureMessage += "\nStack\n" + ExceptionUtils.getStackTrace(rootCause); + } + + mySearch.setFailureMessage(failureMessage); + mySearch.setFailureCode(failureCode); + mySearch.setStatus(SearchStatusEnum.FAILED); + + mySearchRuntimeDetails.setSearchStatus(mySearch.getStatus()); + HookParams params = new HookParams() + .add(RequestDetails.class, myRequest) + .addIfMatchesType(ServletRequestDetails.class, myRequest) + .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); + CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FAILED, params); + + saveSearch(); + span.captureException(t); + } finally { + myOnRemove.accept(mySearch.getUuid()); + + myInitialCollectionLatch.countDown(); + markComplete(); + span.end(); + + } + return null; + } + + private void doSaveSearch() { + Search newSearch = mySearchCacheSvc.save(mySearch); + + // mySearchDao.save is not supposed to return null, but in unit tests + // it can if the mock search dao isn't set up to handle that + if (newSearch != null) { + mySearch = newSearch; + } + } + + /** + * This method actually creates the database query to perform the + * search, and starts it. + */ + private void doSearch() { + /* + * If the user has explicitly requested a _count, perform a + * + * SELECT COUNT(*) .... + * + * before doing anything else. + */ + boolean myParamWantOnlyCount = isWantOnlyCount(myParams); + boolean myParamOrDefaultWantCount = nonNull(myParams.getSearchTotalMode()) ? isWantCount(myParams) : SearchParameterMapCalculator.isWantCount(myDaoConfig.getDefaultTotalMode()); + + if (myParamWantOnlyCount || myParamOrDefaultWantCount) { + ourLog.trace("Performing count"); + ISearchBuilder sb = newSearchBuilder(); + + /* + * createCountQuery + * NB: (see createQuery below) + * Because FulltextSearchSvcImpl will (internally) + * mutate the myParams (searchmap), + * (specifically removing the _content and _text filters) + * we will have to clone those parameters here so that + * the "correct" params are used in createQuery below + */ + Long count = sb.createCountQuery(myParams.clone(), mySearch.getUuid(), myRequest, myRequestPartitionId); + + ourLog.trace("Got count {}", count); + + TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); + txTemplate.execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theArg0) { + mySearch.setTotalCount(count.intValue()); + if (myParamWantOnlyCount) { + mySearch.setStatus(SearchStatusEnum.FINISHED); + } + doSaveSearch(); + } + }); + if (myParamWantOnlyCount) { + return; + } + } + + ourLog.trace("Done count"); + ISearchBuilder sb = newSearchBuilder(); + + /* + * Figure out how many results we're actually going to fetch from the + * database in this pass. This calculation takes into consideration the + * "pre-fetch thresholds" specified in DaoConfig#getSearchPreFetchThresholds() + * as well as the value of the _count parameter. + */ + int currentlyLoaded = defaultIfNull(mySearch.getNumFound(), 0); + int minWanted = 0; + if (myParams.getCount() != null) { + minWanted = myParams.getCount(); + minWanted = Math.min(minWanted, myPagingProvider.getMaximumPageSize()); + minWanted += currentlyLoaded; + } + + for (Iterator iter = myDaoConfig.getSearchPreFetchThresholds().iterator(); iter.hasNext(); ) { + int next = iter.next(); + if (next != -1 && next <= currentlyLoaded) { + continue; + } + + if (next == -1) { + sb.setMaxResultsToFetch(null); + } else { + myMaxResultsToFetch = Math.max(next, minWanted); + sb.setMaxResultsToFetch(myMaxResultsToFetch); + } + + if (iter.hasNext()) { + myAdditionalPrefetchThresholdsRemaining = true; + } + + // If we get here's we've found an appropriate threshold + break; + } + + /* + * Provide any PID we loaded in previous search passes to the + * SearchBuilder so that we don't get duplicates coming from running + * the same query again. + * + * We could possibly accomplish this in a different way by using sorted + * results in our SQL query and specifying an offset. I don't actually + * know if that would be faster or not. At some point should test this + * idea. + */ + if (myPreviouslyAddedResourcePids != null) { + sb.setPreviouslyAddedResourcePids(myPreviouslyAddedResourcePids); + mySyncedPids.addAll(myPreviouslyAddedResourcePids); + } + + /* + * createQuery + * Construct the SQL query we'll be sending to the database + * + * NB: (See createCountQuery above) + * We will pass the original myParams here (not a copy) + * because we actually _want_ the mutation of the myParams to happen. + * Specifically because SearchBuilder itself will _expect_ + * not to have these parameters when dumping back + * to our DB. + * + * This is an odd implementation behaviour, but the change + * for this will require a lot more handling at higher levels + */ + try (IResultIterator resultIterator = sb.createQuery(myParams, mySearchRuntimeDetails, myRequest, myRequestPartitionId)) { + assert (resultIterator != null); + + /* + * The following loop actually loads the PIDs of the resources + * matching the search off of the disk and into memory. After + * every X results, we commit to the HFJ_SEARCH table. + */ + int syncSize = mySyncSize; + while (resultIterator.hasNext()) { + myUnsyncedPids.add(resultIterator.next()); + + boolean shouldSync = myUnsyncedPids.size() >= syncSize; + + if (myDaoConfig.getCountSearchResultsUpTo() != null && + myDaoConfig.getCountSearchResultsUpTo() > 0 && + myDaoConfig.getCountSearchResultsUpTo() < myUnsyncedPids.size()) { + shouldSync = false; + } + + if (myUnsyncedPids.size() > 50000) { + shouldSync = true; + } + + // If no abort was requested, bail out + Validate.isTrue(isNotAborted(), "Abort has been requested"); + + if (shouldSync) { + saveUnsynced(resultIterator); + } + + if (myLoadingThrottleForUnitTests != null) { + AsyncUtil.sleep(myLoadingThrottleForUnitTests); + } + + } + + // If no abort was requested, bail out + Validate.isTrue(isNotAborted(), "Abort has been requested"); + + saveUnsynced(resultIterator); + + } catch (IOException e) { + ourLog.error("IO failure during database access", e); + throw new InternalErrorException(Msg.code(1166) + e); + } + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTaskParameters.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTaskParameters.java new file mode 100644 index 00000000000..84079141de3 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTaskParameters.java @@ -0,0 +1,48 @@ +package ca.uhn.fhir.jpa.search.builder.tasks; + +import ca.uhn.fhir.jpa.api.dao.IDao; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.api.server.RequestDetails; + +import java.util.function.Consumer; + +public class SearchTaskParameters { + // parameters + public ca.uhn.fhir.jpa.entity.Search Search; + public IDao CallingDao; + public SearchParameterMap Params; + public String ResourceType; + public RequestDetails Request; + public ca.uhn.fhir.interceptor.model.RequestPartitionId RequestPartitionId; + public Consumer OnRemove; + public int SyncSize; + + private Integer myLoadingThrottleForUnitTests; + + public SearchTaskParameters(ca.uhn.fhir.jpa.entity.Search theSearch, + IDao theCallingDao, + SearchParameterMap theParams, + String theResourceType, + RequestDetails theRequest, + ca.uhn.fhir.interceptor.model.RequestPartitionId theRequestPartitionId, + Consumer theOnRemove, + int theSyncSize + ) { + Search = theSearch; + CallingDao = theCallingDao; + Params = theParams; + ResourceType = theResourceType; + Request = theRequest; + RequestPartitionId = theRequestPartitionId; + OnRemove = theOnRemove; + SyncSize = theSyncSize; + } + + public Integer getLoadingThrottleForUnitTests() { + return myLoadingThrottleForUnitTests; + } + + public void setLoadingThrottleForUnitTests(Integer theLoadingThrottleForUnitTests) { + myLoadingThrottleForUnitTests = theLoadingThrottleForUnitTests; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/QueryParameterUtils.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/QueryParameterUtils.java new file mode 100644 index 00000000000..1d1457389a9 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/QueryParameterUtils.java @@ -0,0 +1,229 @@ +package ca.uhn.fhir.jpa.util; + +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser; +import ca.uhn.fhir.jpa.entity.Search; +import ca.uhn.fhir.jpa.entity.SearchInclude; +import ca.uhn.fhir.jpa.entity.SearchTypeEnum; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.api.Include; +import ca.uhn.fhir.model.primitive.InstantDt; +import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import ca.uhn.fhir.rest.param.DateRangeParam; +import ca.uhn.fhir.rest.param.ParamPrefixEnum; +import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException; +import com.healthmarketscience.sqlbuilder.BinaryCondition; +import com.healthmarketscience.sqlbuilder.ComboCondition; +import com.healthmarketscience.sqlbuilder.Condition; +import com.healthmarketscience.sqlbuilder.InCondition; +import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn; +import org.apache.commons.collections4.BidiMap; +import org.apache.commons.collections4.bidimap.DualHashBidiMap; +import org.apache.commons.collections4.bidimap.UnmodifiableBidiMap; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.persistence.EntityManager; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.From; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; + +public class QueryParameterUtils { + private static final Logger ourLog = LoggerFactory.getLogger(QueryParameterUtils.class); + public static final int DEFAULT_SYNC_SIZE = 250; + public static final String UNIT_TEST_CAPTURE_STACK = "unit_test_capture_stack"; + + private static final BidiMap ourCompareOperationToParamPrefix; + + static { + DualHashBidiMap compareOperationToParamPrefix = new DualHashBidiMap<>(); + compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.ap, ParamPrefixEnum.APPROXIMATE); + compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.eq, ParamPrefixEnum.EQUAL); + compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.gt, ParamPrefixEnum.GREATERTHAN); + compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.ge, ParamPrefixEnum.GREATERTHAN_OR_EQUALS); + compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.lt, ParamPrefixEnum.LESSTHAN); + compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.le, ParamPrefixEnum.LESSTHAN_OR_EQUALS); + compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.ne, ParamPrefixEnum.NOT_EQUAL); + compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.eb, ParamPrefixEnum.ENDS_BEFORE); + compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.sa, ParamPrefixEnum.STARTS_AFTER); + ourCompareOperationToParamPrefix = UnmodifiableBidiMap.unmodifiableBidiMap(compareOperationToParamPrefix); + } + + + @Nullable + public static Condition toAndPredicate(List theAndPredicates) { + List andPredicates = theAndPredicates.stream().filter(t -> t != null).collect(Collectors.toList()); + if (andPredicates.size() == 0) { + return null; + } else if (andPredicates.size() == 1) { + return andPredicates.get(0); + } else { + return ComboCondition.and(andPredicates.toArray(new Condition[0])); + } + } + + @Nullable + public static Condition toOrPredicate(List theOrPredicates) { + List orPredicates = theOrPredicates.stream().filter(t -> t != null).collect(Collectors.toList()); + if (orPredicates.size() == 0) { + return null; + } else if (orPredicates.size() == 1) { + return orPredicates.get(0); + } else { + return ComboCondition.or(orPredicates.toArray(new Condition[0])); + } + } + + @Nullable + public static Condition toOrPredicate(Condition... theOrPredicates) { + return toOrPredicate(Arrays.asList(theOrPredicates)); + } + + @Nullable + public static Condition toAndPredicate(Condition... theAndPredicates) { + return toAndPredicate(Arrays.asList(theAndPredicates)); + } + + @Nonnull + public static Condition toEqualToOrInPredicate(DbColumn theColumn, List theValuePlaceholders, boolean theInverse) { + if (theInverse) { + return toNotEqualToOrNotInPredicate(theColumn, theValuePlaceholders); + } else { + return toEqualToOrInPredicate(theColumn, theValuePlaceholders); + } + } + + @Nonnull + public static Condition toEqualToOrInPredicate(DbColumn theColumn, List theValuePlaceholders) { + if (theValuePlaceholders.size() == 1) { + return BinaryCondition.equalTo(theColumn, theValuePlaceholders.get(0)); + } + return new InCondition(theColumn, theValuePlaceholders); + } + + @Nonnull + public static Condition toNotEqualToOrNotInPredicate(DbColumn theColumn, List theValuePlaceholders) { + if (theValuePlaceholders.size() == 1) { + return BinaryCondition.notEqualTo(theColumn, theValuePlaceholders.get(0)); + } + return new InCondition(theColumn, theValuePlaceholders).setNegate(true); + } + + public static SearchFilterParser.CompareOperation toOperation(ParamPrefixEnum thePrefix) { + SearchFilterParser.CompareOperation retVal = null; + if (thePrefix != null && ourCompareOperationToParamPrefix.containsValue(thePrefix)) { + retVal = ourCompareOperationToParamPrefix.getKey(thePrefix); + } + return ObjectUtils.defaultIfNull(retVal, SearchFilterParser.CompareOperation.eq); + } + + public static ParamPrefixEnum fromOperation(SearchFilterParser.CompareOperation thePrefix) { + ParamPrefixEnum retVal = null; + if (thePrefix != null && ourCompareOperationToParamPrefix.containsKey(thePrefix)) { + retVal = ourCompareOperationToParamPrefix.get(thePrefix); + } + return ObjectUtils.defaultIfNull(retVal, ParamPrefixEnum.EQUAL); + } + + public static String getChainedPart(String parameter) { + return parameter.substring(parameter.indexOf(".") + 1); + } + + public static String getParamNameWithPrefix(String theSpnamePrefix, String theParamName) { + + if (StringUtils.isBlank(theSpnamePrefix)) + return theParamName; + + return theSpnamePrefix + "." + theParamName; + } + + public static Predicate[] toPredicateArray(List thePredicates) { + return thePredicates.toArray(new Predicate[0]); + } + + private static List createLastUpdatedPredicates(final DateRangeParam theLastUpdated, CriteriaBuilder builder, From from) { + List lastUpdatedPredicates = new ArrayList<>(); + if (theLastUpdated != null) { + if (theLastUpdated.getLowerBoundAsInstant() != null) { + ourLog.debug("LastUpdated lower bound: {}", new InstantDt(theLastUpdated.getLowerBoundAsInstant())); + Predicate predicateLower = builder.greaterThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getLowerBoundAsInstant()); + lastUpdatedPredicates.add(predicateLower); + } + if (theLastUpdated.getUpperBoundAsInstant() != null) { + Predicate predicateUpper = builder.lessThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getUpperBoundAsInstant()); + lastUpdatedPredicates.add(predicateUpper); + } + } + return lastUpdatedPredicates; + } + + public static List filterResourceIdsByLastUpdated(EntityManager theEntityManager, final DateRangeParam theLastUpdated, Collection thePids) { + if (thePids.isEmpty()) { + return Collections.emptyList(); + } + CriteriaBuilder builder = theEntityManager.getCriteriaBuilder(); + CriteriaQuery cq = builder.createQuery(Long.class); + Root from = cq.from(ResourceTable.class); + cq.select(from.get("myId").as(Long.class)); + + List lastUpdatedPredicates = createLastUpdatedPredicates(theLastUpdated, builder, from); + lastUpdatedPredicates.add(from.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(thePids))); + + cq.where(toPredicateArray(lastUpdatedPredicates)); + TypedQuery query = theEntityManager.createQuery(cq); + + return ResourcePersistentId.fromLongList(query.getResultList()); + } + + public static void verifySearchHasntFailedOrThrowInternalErrorException(Search theSearch) { + if (theSearch.getStatus() == SearchStatusEnum.FAILED) { + Integer status = theSearch.getFailureCode(); + status = defaultIfNull(status, 500); + + String message = theSearch.getFailureMessage(); + throw BaseServerResponseException.newInstance(status, message); + } + } + + public static void populateSearchEntity(SearchParameterMap theParams, String theResourceType, String theSearchUuid, String theQueryString, Search theSearch, RequestPartitionId theRequestPartitionId) { + theSearch.setDeleted(false); + theSearch.setUuid(theSearchUuid); + theSearch.setCreated(new Date()); + theSearch.setTotalCount(null); + theSearch.setNumFound(0); + theSearch.setPreferredPageSize(theParams.getCount()); + theSearch.setSearchType(theParams.getEverythingMode() != null ? SearchTypeEnum.EVERYTHING : SearchTypeEnum.SEARCH); + theSearch.setLastUpdated(theParams.getLastUpdated()); + theSearch.setResourceType(theResourceType); + theSearch.setStatus(SearchStatusEnum.LOADING); + theSearch.setSearchQueryString(theQueryString, theRequestPartitionId); + + if (theParams.hasIncludes()) { + for (Include next : theParams.getIncludes()) { + theSearch.addInclude(new SearchInclude(theSearch, next.getValue(), false, next.isRecurse())); + } + } + + for (Include next : theParams.getRevIncludes()) { + theSearch.addInclude(new SearchInclude(theSearch, next.getValue(), true, next.isRecurse())); + } + } +} diff --git a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/provider/ResourceProviderDstu2Test.java b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/provider/ResourceProviderDstu2Test.java index 89b67bab674..ed8b176808e 100644 --- a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/provider/ResourceProviderDstu2Test.java +++ b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/provider/ResourceProviderDstu2Test.java @@ -4,6 +4,7 @@ import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.model.api.ExtensionDt; import ca.uhn.fhir.model.api.IResource; import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum; @@ -144,7 +145,7 @@ public class ResourceProviderDstu2Test extends BaseResourceProviderDstu2Test { myDaoConfig.setReuseCachedSearchResultsForMillis(new DaoConfig().getReuseCachedSearchResultsForMillis()); mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null); - mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(SearchCoordinatorSvcImpl.DEFAULT_SYNC_SIZE); + mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(false); } diff --git a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/BaseSearchSvc.java b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/BaseSearchSvc.java index 5248e2ae675..57dab9223f7 100644 --- a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/BaseSearchSvc.java +++ b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/BaseSearchSvc.java @@ -12,7 +12,9 @@ import ca.uhn.fhir.model.dstu2.resource.Patient; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import org.hl7.fhir.instance.model.api.IBaseResource; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.BeanFactory; import org.springframework.transaction.PlatformTransactionManager; import java.util.ArrayList; @@ -41,7 +43,10 @@ public class BaseSearchSvc { protected DaoRegistry myDaoRegistry; @Mock - protected DaoConfig myDaoConfig; + protected BeanFactory myBeanFactory; + + @Spy + protected DaoConfig myDaoConfig = new DaoConfig(); protected static final FhirContext ourCtx = FhirContext.forDstu3Cached(); diff --git a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java index 6dacadc4e5f..2d103fc3f8f 100644 --- a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java +++ b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java @@ -1,8 +1,10 @@ package ca.uhn.fhir.jpa.search; +import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.config.SearchConfig; import ca.uhn.fhir.jpa.dao.IResultIterator; import ca.uhn.fhir.jpa.dao.ISearchBuilder; import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; @@ -10,15 +12,20 @@ import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchContinuationTask; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchTask; +import ca.uhn.fhir.jpa.search.builder.tasks.SearchTaskParameters; import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.BaseIterator; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.rest.param.StringParam; +import ca.uhn.fhir.rest.server.IPagingProvider; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; import org.hamcrest.Matchers; @@ -28,8 +35,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; -import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,6 +73,7 @@ import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -75,9 +83,6 @@ import static org.mockito.Mockito.when; public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ private static final Logger ourLog = LoggerFactory.getLogger(SearchCoordinatorSvcImplTest.class); - @InjectMocks - private SearchCoordinatorSvcImpl mySvc; - @Mock private SearchStrategyFactory mySearchStrategyFactory; @Mock @@ -95,34 +100,46 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ private IRequestPartitionHelperSvc myPartitionHelperSvc; @Mock private ISynchronousSearchSvc mySynchronousSearchSvc; + @Spy + protected FhirContext myContext = FhirContext.forR4(); + @Spy + private ExceptionService myExceptionSvc = new ExceptionService(myContext); + private SearchCoordinatorSvcImpl mySvc; @AfterEach public void after() { - System.clearProperty(SearchCoordinatorSvcImpl.UNIT_TEST_CAPTURE_STACK); + System.clearProperty(QueryParameterUtils.UNIT_TEST_CAPTURE_STACK); super.after(); } @BeforeEach public void before() { - System.setProperty(SearchCoordinatorSvcImpl.UNIT_TEST_CAPTURE_STACK, "true"); + System.setProperty(QueryParameterUtils.UNIT_TEST_CAPTURE_STACK, "true"); myCurrentSearch = null; - mySvc.setTransactionManagerForUnitTest(myTxManager); - mySvc.setContextForUnitTest(ourCtx); - mySvc.setSearchCacheServicesForUnitTest(mySearchCacheSvc, mySearchResultCacheSvc); - mySvc.setDaoRegistryForUnitTest(myDaoRegistry); - mySvc.setInterceptorBroadcasterForUnitTest(myInterceptorBroadcaster); - mySvc.setSearchBuilderFactoryForUnitTest(mySearchBuilderFactory); - mySvc.setPersistedJpaBundleProviderFactoryForUnitTest(myPersistedJpaBundleProviderFactory); - mySvc.setRequestPartitionHelperService(myPartitionHelperSvc); - mySvc.setSynchronousSearchSvc(mySynchronousSearchSvc); - - DaoConfig daoConfig = new DaoConfig(); - mySvc.setDaoConfigForUnitTest(daoConfig); - + // Mockito has problems wiring up all + // the dependencies; particularly those in extended + // classes. This forces them in + mySvc = new SearchCoordinatorSvcImpl( + myContext, + myDaoConfig, + myInterceptorBroadcaster, + myTxManager, + mySearchCacheSvc, + mySearchResultCacheSvc, + myDaoRegistry, + mySearchBuilderFactory, + mySynchronousSearchSvc, + myPersistedJpaBundleProviderFactory, + myPartitionHelperSvc, + null, // search param registry + mySearchStrategyFactory, + myExceptionSvc, + myBeanFactory + ); } @Test @@ -136,6 +153,7 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ List pids = createPidSequence(800); IResultIterator iter = new FailAfterNIterator(new SlowIterator(pids.iterator(), 2), 300); when(mySearchBuilder.createQuery(same(params), any(), any(), nullable(RequestPartitionId.class))).thenReturn(iter); + mockSearchTask(); try { mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null, RequestPartitionId.allPartitions()); @@ -161,7 +179,6 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ return null; }).when(mySearchResultCacheSvc).storeResults(any(), anyList(), anyList()); - SearchParameterMap params = new SearchParameterMap(); params.add("name", new StringParam("ANAME")); @@ -176,6 +193,8 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ return search; }); + mockSearchTask(); + // Do all the stubbing before starting any work, since we want to avoid threading issues IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null, RequestPartitionId.allPartitions()); @@ -253,6 +272,7 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ List pids = createPidSequence(800); SlowIterator iter = new SlowIterator(pids.iterator(), 2); when(mySearchBuilder.createQuery(same(params), any(), any(), nullable(RequestPartitionId.class))).thenReturn(iter); + mockSearchTask(); doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); @@ -276,10 +296,10 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ } private void initAsyncSearches() { - when(myPersistedJpaBundleProviderFactory.newInstanceFirstPage(nullable(RequestDetails.class), nullable(Search.class), nullable(SearchCoordinatorSvcImpl.SearchTask.class), nullable(ISearchBuilder.class))).thenAnswer(t->{ + when(myPersistedJpaBundleProviderFactory.newInstanceFirstPage(nullable(RequestDetails.class), nullable(Search.class), nullable(SearchTask.class), nullable(ISearchBuilder.class))).thenAnswer(t->{ RequestDetails requestDetails = t.getArgument(0, RequestDetails.class); Search search = t.getArgument(1, Search.class); - SearchCoordinatorSvcImpl.SearchTask searchTask = t.getArgument(2, SearchCoordinatorSvcImpl.SearchTask.class); + SearchTask searchTask = t.getArgument(2, SearchTask.class); ISearchBuilder searchBuilder = t.getArgument(3, ISearchBuilder.class); PersistedJpaSearchFirstPageBundleProvider retVal = new PersistedJpaSearchFirstPageBundleProvider(search, searchTask, searchBuilder, requestDetails); retVal.setDaoConfigForUnitTest(new DaoConfig()); @@ -299,6 +319,9 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ List pids = createPidSequence(800); SlowIterator iter = new SlowIterator(pids.iterator(), 500); when(mySearchBuilder.createQuery(same(params), any(), any(), nullable(RequestPartitionId.class))).thenReturn(iter); + mockSearchTask(); + when(myInterceptorBroadcaster.callHooks(any(), any())) + .thenReturn(true); ourLog.info("Registering the first search"); new Thread(() -> mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null, RequestPartitionId.allPartitions())).start(); @@ -355,6 +378,8 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ }); doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); + mockSearchTask(); + IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null, RequestPartitionId.allPartitions()); assertNotNull(result.getUuid()); assertEquals(790, result.size()); @@ -388,6 +413,7 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ List pids = createPidSequence(100); SlowIterator iter = new SlowIterator(pids.iterator(), 2); when(mySearchBuilder.createQuery(same(params), any(), any(), nullable(RequestPartitionId.class))).thenReturn(iter); + mockSearchTask(); doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); @@ -570,6 +596,7 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ search.setStatus(SearchStatusEnum.LOADING); return Optional.of(search); }); + mockSearchTask(); when(mySearchResultCacheSvc.fetchAllResultPids(any())).thenReturn(null); @@ -711,4 +738,46 @@ public class SearchCoordinatorSvcImplTest extends BaseSearchSvc{ } } + + private void mockSearchTask() { + IPagingProvider pagingProvider = mock(IPagingProvider.class); + lenient().when(pagingProvider.getMaximumPageSize()) + .thenReturn(500); + when(myBeanFactory.getBean(anyString(), any(SearchTaskParameters.class))) + .thenAnswer(invocation -> { + String type = invocation.getArgument(0); + switch (type) { + case SearchConfig.SEARCH_TASK: + return new SearchTask( + invocation.getArgument(1), + myTxManager, + ourCtx, + mySearchStrategyFactory, + myInterceptorBroadcaster, + mySearchBuilderFactory, + mySearchResultCacheSvc, + myDaoConfig, + mySearchCacheSvc, + pagingProvider + ); + case SearchConfig.CONTINUE_TASK: + return new SearchContinuationTask( + invocation.getArgument(1), + myTxManager, + ourCtx, + mySearchStrategyFactory, + myInterceptorBroadcaster, + mySearchBuilderFactory, + mySearchResultCacheSvc, + myDaoConfig, + mySearchCacheSvc, + pagingProvider, + myExceptionSvc + ); + default: + fail("Invalid bean type: " + type); + return null; + } + }); + } } diff --git a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java index 9719a80d81e..09dd39edb07 100644 --- a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java @@ -5,6 +5,7 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import ca.uhn.fhir.model.primitive.InstantDt; import ca.uhn.fhir.model.primitive.UriDt; @@ -192,7 +193,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { myDaoConfig.setReuseCachedSearchResultsForMillis(new DaoConfig().getReuseCachedSearchResultsForMillis()); mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null); - mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(SearchCoordinatorSvcImpl.DEFAULT_SYNC_SIZE); + mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(false); } diff --git a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/search/PagingMultinodeProviderDstu3Test.java b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/search/PagingMultinodeProviderDstu3Test.java index 2dd971b8a5f..fb7818d4c49 100644 --- a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/search/PagingMultinodeProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/search/PagingMultinodeProviderDstu3Test.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.search; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.provider.dstu3.BaseResourceProviderDstu3Test; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.parser.StrictErrorHandler; import org.hl7.fhir.dstu3.model.Bundle; import org.hl7.fhir.dstu3.model.Patient; @@ -28,7 +29,7 @@ public class PagingMultinodeProviderDstu3Test extends BaseResourceProviderDstu3T myDaoConfig.setAllowExternalReferences(new DaoConfig().isAllowExternalReferences()); mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null); - mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(SearchCoordinatorSvcImpl.DEFAULT_SYNC_SIZE); + mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(false); } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java index 634375496a3..3518d6488a3 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java @@ -13,6 +13,7 @@ import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.searchparam.MatchUrlService; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.rest.api.SearchTotalModeEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.SummaryEnum; @@ -92,7 +93,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { public void before() { mySearchCoordinatorSvcImpl = (SearchCoordinatorSvcImpl) AopProxyUtils.getSingletonTarget(mySearchCoordinatorSvc); mySearchCoordinatorSvcImpl.setLoadingThrottleForUnitTests(null); - mySearchCoordinatorSvcImpl.setSyncSizeForUnitTests(SearchCoordinatorSvcImpl.DEFAULT_SYNC_SIZE); + mySearchCoordinatorSvcImpl.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); myCaptureQueriesListener.setCaptureQueryStackTrace(true); myDaoConfig.setAdvancedHSearchIndexing(false); } @@ -100,7 +101,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { @AfterEach public final void after() { mySearchCoordinatorSvcImpl.setLoadingThrottleForUnitTests(null); - mySearchCoordinatorSvcImpl.setSyncSizeForUnitTests(SearchCoordinatorSvcImpl.DEFAULT_SYNC_SIZE); + mySearchCoordinatorSvcImpl.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); myDaoConfig.setSearchPreFetchThresholds(new DaoConfig().getSearchPreFetchThresholds()); myCaptureQueriesListener.setCaptureQueryStackTrace(false); myDaoConfig.setIndexMissingFields(new DaoConfig().getIndexMissingFields()); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java index 0b5212dc534..fa2079f2474 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java @@ -17,10 +17,10 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.entity.TagTypeEnum; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.model.util.UcumServiceUtil; -import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.searchparam.SearchParamConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.util.TestUtil; import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum; @@ -4179,7 +4179,7 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test { runInTransaction(() -> { ca.uhn.fhir.jpa.entity.Search search = new ca.uhn.fhir.jpa.entity.Search(); - SearchCoordinatorSvcImpl.populateSearchEntity(map, "Encounter", uuid, normalized, search, RequestPartitionId.allPartitions()); + QueryParameterUtils.populateSearchEntity(map, "Encounter", uuid, normalized, search, RequestPartitionId.allPartitions()); search.setStatus(SearchStatusEnum.FAILED); search.setFailureCode(500); search.setFailureMessage("FOO"); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java index de76a45fae0..f0a88880517 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java @@ -11,6 +11,7 @@ import ca.uhn.fhir.jpa.model.util.UcumServiceUtil; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.term.ZipCollectionBuilder; import ca.uhn.fhir.jpa.test.config.TestR4Config; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import ca.uhn.fhir.model.primitive.IdDt; import ca.uhn.fhir.model.primitive.InstantDt; @@ -28,6 +29,8 @@ import ca.uhn.fhir.rest.client.api.IGenericClient; import ca.uhn.fhir.rest.client.api.IHttpRequest; import ca.uhn.fhir.rest.client.api.IHttpResponse; import ca.uhn.fhir.rest.client.interceptor.CapturingInterceptor; +import ca.uhn.fhir.rest.gclient.ICriterion; +import ca.uhn.fhir.rest.gclient.NumberClientParam; import ca.uhn.fhir.rest.gclient.StringClientParam; import ca.uhn.fhir.rest.param.DateRangeParam; import ca.uhn.fhir.rest.param.NumberParam; @@ -71,6 +74,7 @@ import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Attachment; import org.hl7.fhir.r4.model.AuditEvent; +import org.hl7.fhir.r4.model.BaseResource; import org.hl7.fhir.r4.model.Basic; import org.hl7.fhir.r4.model.Binary; import org.hl7.fhir.r4.model.Bundle; @@ -128,6 +132,7 @@ import org.hl7.fhir.r4.model.Questionnaire; import org.hl7.fhir.r4.model.Questionnaire.QuestionnaireItemType; import org.hl7.fhir.r4.model.QuestionnaireResponse; import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.Resource; import org.hl7.fhir.r4.model.SearchParameter; import org.hl7.fhir.r4.model.ServiceRequest; import org.hl7.fhir.r4.model.StringType; @@ -142,8 +147,11 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.AopTestUtils; @@ -162,6 +170,7 @@ import java.net.SocketTimeoutException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Collections; import java.util.Date; import java.util.HashSet; @@ -169,6 +178,7 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; +import java.util.stream.Stream; import static ca.uhn.fhir.jpa.config.r4.FhirContextR4Config.DEFAULT_PRESERVE_VERSION_REFS; import static ca.uhn.fhir.jpa.util.TestUtil.sleepOneClick; @@ -227,7 +237,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { myDaoConfig.setIndexMissingFields(new DaoConfig().getIndexMissingFields()); mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null); - mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(SearchCoordinatorSvcImpl.DEFAULT_SYNC_SIZE); + mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(false); mySearchCoordinatorSvcRaw.cancelAllActiveSearches(); myDaoConfig.getModelConfig().setNormalizedQuantitySearchLevel(NormalizedQuantitySearchLevel.NORMALIZED_QUANTITY_SEARCH_NOT_SUPPORTED); @@ -7144,4 +7154,315 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { return oId; } + @Nested + public class MissingSearchParameterTests { + + private interface XtoY { + Y doTask(X theInput); + } + + private static class MissingSearchTestParameters { + /** + * The setting for IndexMissingFields + */ + public final DaoConfig.IndexEnabledEnum myEnableMissingFieldsValue; + + /** + * Whether to use :missing=true/false + */ + public final boolean myIsMissing; + + /** + * Whether or not the field is populated or not. + * True -> populate field. + * False -> not populated + */ + public final boolean myIsValuePresentOnResource; + + public MissingSearchTestParameters( + DaoConfig.IndexEnabledEnum theEnableMissingFields, + boolean theIsMissing, + boolean theHasField + ) { + myEnableMissingFieldsValue = theEnableMissingFields; + myIsMissing = theIsMissing; + myIsValuePresentOnResource = theHasField; + } + } + + private IParser myParser; + + @BeforeEach + public void init() { + myParser = myFhirContext.newJsonParser(); + myParser.setPrettyPrint(true); + + myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.DISABLED); + } + + /** + * Verifies that the returned Bundle contains the resource + * with the id provided. + * @param theBundle - returned bundle + * @param theType - provided resource id + */ + private void verifyFoundBundle(Bundle theBundle, IIdType theType) { + ourLog.info(myParser.encodeResourceToString(theBundle)); + + assertEquals(1, theBundle.getTotal()); + List list = toUnqualifiedVersionlessIds(theBundle); + ourLog.info(list.size() + " resources found"); + IIdType type = list.get(0); + assertEquals(theType.toString(), type.toString()); + } + + private void verifyBundleIsEmpty(Bundle theBundle) { + ourLog.info(myParser.encodeResourceToString(theBundle)); + assertEquals(0, theBundle.getTotal()); + } + + private IIdType createResource(Resource theResource) { + IIdType id = myClient.create() + .resource(theResource) + .prettyPrint() + .encodedXml() + .execute() + .getId() + .toUnqualifiedVersionless(); + theResource.setId(id); + ourLog.info("Created:\n{}", + myParser.encodeResourceToString(theResource)); + return id; + } + + /** + * Runs the search on the given resource type with the given (missing) criteria + * @param theResourceClass - the resource type class + * @param theCriteria - the missing critia to use + * @return - the found bundle + */ + private Bundle doSearch(Class theResourceClass, ICriterion theCriteria) { + //@formatter:off + return myClient + .search() + .forResource(theResourceClass) + .where(theCriteria) + .count(100) + .prettyPrint() + .returnBundle(Bundle.class) + .execute(); + //@formatter:on + } + + /** + * The method that generates parameters for tests + */ + private static Stream provideParameters() { + return Stream.of( + // 1 + Arguments.of(new MissingSearchTestParameters(DaoConfig.IndexEnabledEnum.ENABLED, true, true)), + // 2 + Arguments.of(new MissingSearchTestParameters(DaoConfig.IndexEnabledEnum.ENABLED, false, false)), + // 3 + Arguments.of(new MissingSearchTestParameters(DaoConfig.IndexEnabledEnum.ENABLED, false, true)), + // 4 + Arguments.of(new MissingSearchTestParameters(DaoConfig.IndexEnabledEnum.ENABLED, true, false)), + // 5 + Arguments.of(new MissingSearchTestParameters(DaoConfig.IndexEnabledEnum.DISABLED, true, true)), + // 6 + Arguments.of(new MissingSearchTestParameters(DaoConfig.IndexEnabledEnum.DISABLED, false, true)), + // 7 + Arguments.of(new MissingSearchTestParameters(DaoConfig.IndexEnabledEnum.DISABLED, true, false)), + // 8 + Arguments.of(new MissingSearchTestParameters(DaoConfig.IndexEnabledEnum.DISABLED, false, false)) + ); + } + + /** + * Runs the actual test for whichever search parameter and given inputs we want. + */ + private void runTest( + MissingSearchTestParameters theParams, + XtoY theResourceProvider, + XtoY theRunner + ) { + String testMethod = new Exception().getStackTrace()[1].getMethodName(); + ourLog.info( + "\nStarting {}.\nMissing fields indexed: {},\nHas Field Present: {},\nReturn resources with Missing Field: {}.\nWe expect {} returned result(s).", + testMethod, + theParams.myEnableMissingFieldsValue.name(), + theParams.myIsValuePresentOnResource, + theParams.myIsMissing, + theParams.myIsValuePresentOnResource == theParams.myIsMissing ? "0" : "1" + ); + + // setup + myDaoConfig.setIndexMissingFields(theParams.myEnableMissingFieldsValue); + + // create our resource + Resource resource = theResourceProvider.doTask(theParams.myIsValuePresentOnResource); + + // save the resource + IIdType resourceId = createResource(resource); + + // run test + Bundle found = theRunner.doTask(theParams.myIsMissing); + + if ((theParams.myIsMissing && !theParams.myIsValuePresentOnResource) + || (theParams.myIsValuePresentOnResource && !theParams.myIsMissing)) { + verifyFoundBundle(found, resourceId); + } else { + verifyBundleIsEmpty(found); + } + } + + @ParameterizedTest + @MethodSource("provideParameters") + public void testMissingStringParameter(MissingSearchTestParameters theParams) { + runTest( + theParams, (hasField) -> { + Organization org = new Organization(); + if (hasField) { + org.setName("anything"); + } + return org; + }, (isMissing) -> { + return doSearch(Organization.class, Organization.NAME.isMissing(isMissing)); + } + ); + } + + @ParameterizedTest + @MethodSource("provideParameters") + public void testMissingDateParameter(MissingSearchTestParameters theParams) { + runTest(theParams, + (hasField) -> { + Patient patient = new Patient(); + if (hasField) { + patient.setBirthDate(new Date(2000, Calendar.DECEMBER, 25)); + } + return patient; + }, (isMissing) -> { + return doSearch(Patient.class, Patient.BIRTHDATE.isMissing(isMissing)); + }); + } + + @ParameterizedTest + @MethodSource("provideParameters") + public void testMissingTokenClientParameter(MissingSearchTestParameters theParams) { + runTest(theParams, + hasField -> { + Patient patient = new Patient(); + if (hasField) { + patient.setGender(AdministrativeGender.FEMALE); + } + return patient; + }, isMissing -> { + return doSearch(Patient.class, Patient.GENDER.isMissing(isMissing)); + }); + } + + @ParameterizedTest + @MethodSource("provideParameters") + public void testMissingReferenceClientParameter(MissingSearchTestParameters theParams) { + runTest(theParams, + hasField -> { + Patient patient = new Patient(); + if (hasField) { + Practitioner practitioner = new Practitioner(); + IIdType practitionerId = createResource(practitioner); + + patient.setGeneralPractitioner(Collections.singletonList(new Reference(practitionerId))); + } + return patient; + }, isMissing -> { + return doSearch(Patient.class, Patient.GENERAL_PRACTITIONER.isMissing(isMissing)); + }); + } + + @ParameterizedTest + @MethodSource("provideParameters") + public void testMissingReferenceClientParameterOnIndexedContainedResources(MissingSearchTestParameters theParams) { + myModelConfig.setIndexOnContainedResources(true); + runTest(theParams, + hasField -> { + Observation obs = new Observation(); + if (hasField) { + Encounter enc = new Encounter(); + IIdType id = createResource(enc); + + obs.setEncounter(new Reference(id)); + } + return obs; + }, isMissing -> { + ICriterion criterion = Observation.ENCOUNTER.isMissing(isMissing); + return doSearch(Observation.class, criterion); + }); + + myModelConfig.setIndexOnContainedResources(false); + } + + @ParameterizedTest + @MethodSource("provideParameters") + public void testMissingURLParameter(MissingSearchTestParameters theParams) { + runTest(theParams, + hasField -> { + String methodName = new Exception().getStackTrace()[0].getMethodName(); + + SearchParameter sp = new SearchParameter(); + sp.addBase("MolecularSequence"); + sp.setCode(methodName); + sp.setType(Enumerations.SearchParamType.NUMBER); + sp.setExpression("MolecularSequence.variant-end"); + sp.setXpathUsage(SearchParameter.XPathUsageType.NORMAL); + sp.setStatus(Enumerations.PublicationStatus.ACTIVE); + if (hasField) { + sp.setUrl("http://example.com"); + } + return sp; + }, isMissing -> { + return doSearch(SearchParameter.class, SearchParameter.URL.isMissing(isMissing)); + }); + } + + @ParameterizedTest + @MethodSource("provideParameters") + public void testMissingQuantityClientParameter(MissingSearchTestParameters theParams) { + runTest(theParams, + hasField -> { + Observation obs = new Observation(); + if (hasField) { + obs.setValue(new Quantity(3)); + } + return obs; + }, isMissing -> { + return doSearch(Observation.class, Observation.VALUE_QUANTITY.isMissing(isMissing)); + }); + } + + @ParameterizedTest + @MethodSource("provideParameters") + public void testMissingNumberClientParameter(MissingSearchTestParameters theParams) { + runTest(theParams, + hasField -> { + String methodName = new Exception().getStackTrace()[0].getMethodName(); + + MolecularSequence molecularSequence = new MolecularSequence(); + if (hasField) { + molecularSequence.setVariant(Collections.singletonList( + new MolecularSequence.MolecularSequenceVariantComponent().setEnd(1) + )); + } + + return molecularSequence; + }, isMissing -> { + NumberClientParam numberClientParam = new NumberClientParam("variant-end"); + return doSearch( + MolecularSequence.class, + numberClientParam.isMissing(isMissing) + ); + }); + } + } + } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderSummaryModeR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderSummaryModeR4Test.java index 219ab097ce3..c22c3690856 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderSummaryModeR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderSummaryModeR4Test.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.provider.r4; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.rest.api.SearchTotalModeEnum; import ca.uhn.fhir.rest.api.SummaryEnum; import ca.uhn.fhir.rest.gclient.StringClientParam; @@ -28,7 +29,7 @@ public class ResourceProviderSummaryModeR4Test extends BaseResourceProviderR4Tes super.after(); myDaoConfig.setCountSearchResultsUpTo(null); mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null); - mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(SearchCoordinatorSvcImpl.DEFAULT_SYNC_SIZE); + mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); myDaoConfig.setSearchPreFetchThresholds(new DaoConfig().getSearchPreFetchThresholds()); myDaoConfig.setDefaultTotalMode(null); } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/search/r4/PagingMultinodeProviderR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/search/r4/PagingMultinodeProviderR4Test.java index 92a1f405e96..448df50f7e3 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/search/r4/PagingMultinodeProviderR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/search/r4/PagingMultinodeProviderR4Test.java @@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.search.r4; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.provider.r4.BaseResourceProviderR4Test; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.parser.StrictErrorHandler; import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.Patient; @@ -28,7 +29,7 @@ public class PagingMultinodeProviderR4Test extends BaseResourceProviderR4Test { myDaoConfig.setAllowExternalReferences(new DaoConfig().isAllowExternalReferences()); mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null); - mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(SearchCoordinatorSvcImpl.DEFAULT_SYNC_SIZE); + mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(false); }