From eb1d1c2b27c31edd0a10d83d857af63e01db6506 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Thu, 14 May 2020 12:08:55 -0400 Subject: [PATCH] Changes to better support chunking of queries and paging of results for lastn operation and to monitor performance. --- .../ca/uhn/fhir/interceptor/api/Pointcut.java | 40 +- .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 86 +-- .../PerformanceTracingLoggingInterceptor.java | 5 + .../search/lastn/ElasticsearchSvcImpl.java | 35 +- .../jpa/search/lastn/IElasticsearchSvc.java | 3 +- .../ca/uhn/fhir/jpa/config/TestR4Config.java | 5 + .../config/TestR4ConfigWithElasticSearch.java | 3 + .../TestR4ConfigWithElasticsearchClient.java | 1 + .../fhir/jpa/dao/r4/BaseR4SearchLastN.java | 543 ++++++++++++++++ .../FhirResourceDaoR4SearchLastNAsyncIT.java | 133 ++++ .../r4/FhirResourceDaoR4SearchLastNIT.java | 578 +----------------- ...bservationIndexedSearchParamLastNR4IT.java | 20 +- ...lasticsearchSvcMultipleObservationsIT.java | 22 +- ...tNElasticsearchSvcSingleObservationIT.java | 4 +- .../model/search/SearchRuntimeDetails.java | 9 + 15 files changed, 867 insertions(+), 620 deletions(-) create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java index 672fa91f0d7..3ac4cffd0e1 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java @@ -1719,7 +1719,7 @@ public enum Pointcut { /** * Performance Tracing Hook: - * This hook is invoked when a search has failed for any reason. When this pointcut + * This hook is invoked when a search has completed. When this pointcut * is invoked, a pass in the Search Coordinator has completed successfully, but * not all possible resources have been loaded yet so a future paging request * may trigger a new task that will load further resources. @@ -1757,6 +1757,44 @@ public enum Pointcut { "ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails" ), + /** + * Performance Tracing Hook: + * This hook is invoked when a query involving an external index (e.g. Elasticsearch) has completed. When this pointcut + * is invoked, an initial list of resource IDs has been generated which will be used as part of a subsequent database query. + *

+ * Note that this is a performance tracing hook. Use with caution in production + * systems, since calling it may (or may not) carry a cost. + *

+ * Hooks may accept the following parameters: + * + *

+ * Hooks should return void. + *

+ */ + JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE(void.class, + "ca.uhn.fhir.rest.api.server.RequestDetails", + "ca.uhn.fhir.rest.server.servlet.ServletRequestDetails", + "ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails" + ), + /** * Performance Tracing Hook: * Invoked when the storage engine is about to reuse the results of diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index b03011ce626..3027019dc1f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -131,7 +131,7 @@ public class SearchBuilder implements ISearchBuilder { */ // NB: keep public public static final int MAXIMUM_PAGE_SIZE = 800; - public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 4; + public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50; public static boolean myIsTest = false; private static final List EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<>()); @@ -249,7 +249,7 @@ public class SearchBuilder implements ISearchBuilder { init(theParams, theSearchUuid, theRequestPartitionId); - List> queries = createQuery(null, null, true, theRequest); + List> queries = createQuery(null, null, true, theRequest, null); return new CountQueryIterator(queries.get(0)); } @@ -283,7 +283,8 @@ public class SearchBuilder implements ISearchBuilder { } - private List> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) { + private List> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, + SearchRuntimeDetails theSearchRuntimeDetails) { List pids = new ArrayList<>(); /* @@ -310,17 +311,26 @@ public class SearchBuilder implements ISearchBuilder { throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); } } - Integer myMaxObservationsPerCode = null; + Integer myMaxObservationsPerCode; if(myParams.getLastNMax() != null) { myMaxObservationsPerCode = myParams.getLastNMax(); } else { throw new InvalidRequestException("Max parameter is required for $lastn operation"); } - List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); + List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode, theMaximumResults); for (String lastnResourceId : lastnResourceIds) { pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); } } + if (theSearchRuntimeDetails != null) { + theSearchRuntimeDetails.setFoundIndexMatchesCount(pids.size()); + HookParams params = new HookParams() + .add(RequestDetails.class, theRequest) + .addIfMatchesType(ServletRequestDetails.class, theRequest) + .add(SearchRuntimeDetails.class, theSearchRuntimeDetails); + JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params); + } + if (pids.isEmpty()) { // Will never match pids = Collections.singletonList(new ResourcePersistentId(-1L)); @@ -331,24 +341,27 @@ public class SearchBuilder implements ISearchBuilder { ArrayList> myQueries = new ArrayList<>(); if (!pids.isEmpty()) { + if (theMaximumResults != null && pids.size() > theMaximumResults) { + pids.subList(0,theMaximumResults-1); + } new QueryChunker().chunk(ResourcePersistentId.toLongList(pids), t->{ - doCreateChunkedQueries(t, sort, theMaximumResults, theCount, theRequest, myQueries); + doCreateChunkedQueries(t, sort, theCount, theRequest, myQueries); }); } else { - myQueries.add(createQuery(sort,theMaximumResults, theCount, theRequest, null)); + myQueries.add(createChunkedQuery(sort,theMaximumResults, theCount, theRequest, null)); } return myQueries; } - private void doCreateChunkedQueries(List thePids, SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, ArrayList> theQueries) { - if(thePids.size() < MAXIMUM_PAGE_SIZE) { - thePids = normalizeIdListForLastNInClause(thePids); + private void doCreateChunkedQueries(List thePids, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList> theQueries) { + if(thePids.size() < getMaximumPageSize()) { + normalizeIdListForLastNInClause(thePids); } - theQueries.add(createQuery(sort, theMaximumResults, theCount, theRequest, thePids)); + theQueries.add(createChunkedQuery(sort, thePids.size(), theCount, theRequest, thePids)); } - private TypedQuery createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List thePidList) { + private TypedQuery createChunkedQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List thePidList) { CriteriaQuery outerQuery; /* * Sort @@ -501,11 +514,6 @@ public class SearchBuilder implements ISearchBuilder { } private List normalizeIdListForLastNInClause(List lastnResourceIds) { - List retVal = new ArrayList<>(); - for (Long lastnResourceId : lastnResourceIds) { - retVal.add(lastnResourceId); - } - /* The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info: @@ -514,23 +522,23 @@ public class SearchBuilder implements ISearchBuilder { Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of arguments never exceeds the maximum specified below. */ - int listSize = retVal.size(); + int listSize = lastnResourceIds.size(); if(listSize > 1 && listSize < 10) { - padIdListWithPlaceholders(retVal, 10); + padIdListWithPlaceholders(lastnResourceIds, 10); } else if (listSize > 10 && listSize < 50) { - padIdListWithPlaceholders(retVal, 50); + padIdListWithPlaceholders(lastnResourceIds, 50); } else if (listSize > 50 && listSize < 100) { - padIdListWithPlaceholders(retVal, 100); + padIdListWithPlaceholders(lastnResourceIds, 100); } else if (listSize > 100 && listSize < 200) { - padIdListWithPlaceholders(retVal, 200); + padIdListWithPlaceholders(lastnResourceIds, 200); } else if (listSize > 200 && listSize < 500) { - padIdListWithPlaceholders(retVal, 500); + padIdListWithPlaceholders(lastnResourceIds, 500); } else if (listSize > 500 && listSize < 800) { - padIdListWithPlaceholders(retVal, 800); + padIdListWithPlaceholders(lastnResourceIds, 800); } - return retVal; + return lastnResourceIds; } private void padIdListWithPlaceholders(List theIdList, int preferredListSize) { @@ -664,10 +672,17 @@ public class SearchBuilder implements ISearchBuilder { private void doLoadPids(Collection thePids, Collection theIncludedPids, List theResourceListToPopulate, boolean theForHistoryOperation, - Map thePosition, RequestDetails theRequest) { + Map thePosition) { + + List myLongPersistentIds; + if(thePids.size() < getMaximumPageSize()) { + myLongPersistentIds = normalizeIdListForLastNInClause(ResourcePersistentId.toLongList(thePids)); + } else { + myLongPersistentIds = ResourcePersistentId.toLongList(thePids); + } // -- get the resource from the searchView - Collection resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(ResourcePersistentId.toLongList(thePids)); + Collection resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(myLongPersistentIds); //-- preload all tags with tag definition if any Map> tagMap = getResourceTagMap(resourceSearchViewList); @@ -768,7 +783,7 @@ public class SearchBuilder implements ISearchBuilder { List pids = new ArrayList<>(thePids); new QueryChunker().chunk(pids, t->{ - doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position, theDetails); + doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position); }); } @@ -1096,9 +1111,8 @@ public class SearchBuilder implements ISearchBuilder { private final RequestDetails myRequest; private Iterator myCurrentIterator; - private Set myCurrentPids; + private final Set myCurrentPids; private ResourcePersistentId myNext; - private int myPageSize = myDaoConfig.getEverythingIncludesFetchPageSize(); IncludesIterator(Set thePidSet, RequestDetails theRequest) { myCurrentPids = new HashSet<>(thePidSet); @@ -1151,12 +1165,12 @@ public class SearchBuilder implements ISearchBuilder { private ResourcePersistentId myNext; private Iterator myPreResultsIterator; private ScrollableResultsIterator myResultsIterator; - private SortSpec mySort; + private final SortSpec mySort; private boolean myStillNeedToFetchIncludes; private int mySkipCount = 0; private int myNonSkipCount = 0; - private List> myQueryList; + private List> myQueryList = new ArrayList<>(); private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) { mySearchRuntimeDetails = theSearchRuntimeDetails; @@ -1210,7 +1224,7 @@ public class SearchBuilder implements ISearchBuilder { if (myNext == null) { while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) { // Update iterator with next chunk if necessary. - if (!myResultsIterator.hasNext() && !myQueryList.isEmpty()) { + if (!myResultsIterator.hasNext()) { retrieveNextIteratorQuery(); } @@ -1312,8 +1326,10 @@ public class SearchBuilder implements ISearchBuilder { } private void initializeIteratorQuery(Integer theMaxResultsToFetch) { - if (myQueryList == null || myQueryList.isEmpty()) { - myQueryList = createQuery(mySort, theMaxResultsToFetch, false, myRequest); + if (myQueryList.isEmpty()) { + // Capture times for Lucene/Elasticsearch queries as well + mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); + myQueryList = createQuery(mySort, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails); } mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/PerformanceTracingLoggingInterceptor.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/PerformanceTracingLoggingInterceptor.java index a01b6f08203..6cca9668499 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/PerformanceTracingLoggingInterceptor.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/PerformanceTracingLoggingInterceptor.java @@ -79,6 +79,11 @@ public class PerformanceTracingLoggingInterceptor { log("SqlQuery {} failed in {} - Found {} matches", theOutcome.getSearchUuid(), theOutcome.getQueryStopwatch(), theOutcome.getFoundMatchesCount()); } + @Hook(value = Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE) + public void indexSearchQueryComplete(SearchRuntimeDetails theOutcome) { + log("Index query for {} completed in {} - Found {} matches", theOutcome.getSearchUuid(), theOutcome.getQueryStopwatch(), theOutcome.getFoundIndexMatchesCount()); + } + @Hook(value = Pointcut.JPA_PERFTRACE_INFO) public void info(StorageProcessingMessage theMessage) { log("[INFO] " + theMessage); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 47c42cd6b6e..49af7d3510d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -2,7 +2,6 @@ package ca.uhn.fhir.jpa.search.lastn; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.IQueryParameterType; -import ca.uhn.fhir.model.dstu2.resource.Observation; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; @@ -49,9 +48,9 @@ import static org.apache.commons.lang3.StringUtils.isBlank; public class ElasticsearchSvcImpl implements IElasticsearchSvc { - RestHighLevelClient myRestHighLevelClient; + private final RestHighLevelClient myRestHighLevelClient; - ObjectMapper objectMapper = new ObjectMapper(); + private final ObjectMapper objectMapper = new ObjectMapper(); private final String GROUP_BY_SUBJECT = "group_by_subject"; private final String GROUP_BY_CODE = "group_by_code"; @@ -201,14 +200,15 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { @Override // TODO: Should eliminate dependency on SearchParameterMap in API. - public List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { + public List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch) { String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; try { List responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, topHitsInclude); List observationIds = new ArrayList<>(); for (SearchResponse response : responses) { // observationIds.addAll(buildObservationIdList(response)); - observationIds.addAll(buildObservationList(response, t -> t.getIdentifier(), theSearchParameterMap)); + Integer maxResultsToAdd = theMaxResultsToFetch - observationIds.size(); + observationIds.addAll(buildObservationList(response, t -> t.getIdentifier(), theSearchParameterMap, maxResultsToAdd)); } return observationIds; } catch (IOException theE) { @@ -216,7 +216,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } } - private List buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, String[] topHitsInclude) { + private List buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, + String[] topHitsInclude) { List responses = new ArrayList<>(); if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { ArrayList subjectReferenceCriteria = new ArrayList<>(); @@ -254,12 +255,12 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { @VisibleForTesting // TODO: Should eliminate dependency on SearchParameterMap in API. - List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { + List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch) { try { List responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, null); List observationDocuments = new ArrayList<>(); for (SearchResponse response : responses) { - observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap)); + observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap, theMaxResultsToFetch)); } return observationDocuments; } catch (IOException theE) { @@ -337,12 +338,22 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return theObservationList; } - private List buildObservationList(SearchResponse theSearchResponse, Function setValue, SearchParameterMap theSearchParameterMap) throws IOException { + private List buildObservationList(SearchResponse theSearchResponse, Function setValue, + SearchParameterMap theSearchParameterMap, Integer theMaxResultsToFetch) throws IOException { List theObservationList = new ArrayList<>(); if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } String indexedObservation = lastNMatch.getSourceAsString(); ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); theObservationList.add(setValue.apply(observationJson)); @@ -351,7 +362,13 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } } else { for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(theSearchResponse)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } String indexedObservation = lastNMatch.getSourceAsString(); ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); theObservationList.add(setValue.apply(observationJson)); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java index 091d85b742f..592b6939d6d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -5,6 +5,5 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import java.util.List; public interface IElasticsearchSvc { - - List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode); + List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java index 3955361d5fd..dfa4c95bd21 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java @@ -102,10 +102,14 @@ public class TestR4Config extends BaseJavaConfigR4 { }; retVal.setDriver(new org.h2.Driver()); +// retVal.setDriver(new org.postgresql.Driver()); retVal.setUrl("jdbc:h2:mem:testdb_r4"); +// retVal.setUrl("jdbc:postgresql://localhost:5432/hapi"); retVal.setMaxWaitMillis(10000); retVal.setUsername(""); +// retVal.setUsername("hapi"); retVal.setPassword(""); +// retVal.setPassword("HapiFHIR"); retVal.setMaxTotal(ourMaxThreads); SLF4JLogLevel level = SLF4JLogLevel.INFO; @@ -145,6 +149,7 @@ public class TestR4Config extends BaseJavaConfigR4 { extraProperties.put("hibernate.show_sql", "false"); extraProperties.put("hibernate.hbm2ddl.auto", "update"); extraProperties.put("hibernate.dialect", H2Dialect.class.getName()); +// extraProperties.put("hibernate.dialect", org.hibernate.dialect.PostgreSQL95Dialect.class.getName()); extraProperties.put("hibernate.search.model_mapping", ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory.class.getName()); extraProperties.put("hibernate.search.default.directory_provider", "local-heap"); extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java index ebf3fc9e094..1b90253383d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java @@ -24,7 +24,9 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config { private static final String ELASTIC_VERSION = "6.5.4"; protected final String elasticsearchHost = "localhost"; protected final String elasticsearchUserId = ""; +// protected final String elasticsearchUserId = "elastic"; protected final String elasticsearchPassword = ""; +// protected final String elasticsearchPassword = "changeme"; @Override @@ -34,6 +36,7 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config { // Force elasticsearch to start first int httpPort = embeddedElasticSearch().getHttpPort(); +// int httpPort = 9301; ourLog.info("ElasticSearch started on port: {}", httpPort); new ElasticsearchHibernatePropertiesBuilder() diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java index 9ff6bc4c491..607457c8edd 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java @@ -10,6 +10,7 @@ public class TestR4ConfigWithElasticsearchClient extends TestR4ConfigWithElastic @Bean() public ElasticsearchSvcImpl myElasticsearchSvc() { int elasticsearchPort = embeddedElasticSearch().getHttpPort(); +// int elasticsearchPort = 9301; return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java new file mode 100644 index 00000000000..bcfde6bac4f --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java @@ -0,0 +1,543 @@ +package ca.uhn.fhir.jpa.dao.r4; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoPatient; +import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; +import ca.uhn.fhir.jpa.dao.BaseJpaTest; +import ca.uhn.fhir.jpa.dao.SearchBuilder; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceAndListParam; +import ca.uhn.fhir.rest.param.ReferenceOrListParam; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenAndListParam; +import ca.uhn.fhir.rest.param.TokenOrListParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.server.RestfulServer; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.util.TestUtil; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.StringType; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.transaction.PlatformTransactionManager; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.matchesPattern; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class }) +public class BaseR4SearchLastN extends BaseJpaTest { + + @Autowired + @Qualifier("myPatientDaoR4") + protected IFhirResourceDaoPatient myPatientDao; + + @Autowired + @Qualifier("myObservationDaoR4") + protected IFhirResourceDaoObservation myObservationDao; + + @Autowired + protected DaoConfig myDaoConfig; + + @Autowired + protected FhirContext myFhirCtx; + + @Autowired + protected PlatformTransactionManager myPlatformTransactionManager; + + @Override + protected FhirContext getContext() { + return myFhirCtx; + } + + @Override + protected PlatformTransactionManager getTxManager() { + return myPlatformTransactionManager; + } + + protected final String observationCd0 = "code0"; + protected final String observationCd1 = "code1"; + protected final String observationCd2 = "code2"; + private final String observationCd3 = "code3"; + + protected final String categoryCd0 = "category0"; + private final String categoryCd1 = "category1"; + private final String categoryCd2 = "category2"; + private final String categoryCd3 = "category3"; + + protected final String codeSystem = "http://mycode.com"; + private final String categorySystem = "http://mycategory.com"; + + // Using static variables including the flag below so that we can initalize the database and indexes once + // (all of the tests only read from the DB and indexes and so no need to re-initialze them for each test). + private static boolean dataLoaded = false; + + protected static IIdType patient0Id = null; + protected static IIdType patient1Id = null; + protected static IIdType patient2Id = null; + + private static final Map observationPatientMap = new HashMap<>(); + private static final Map observationCategoryMap = new HashMap<>(); + private static final Map observationCodeMap = new HashMap<>(); + private static final Map observationEffectiveMap = new HashMap<>(); + + @Before + public void beforeCreateTestPatientsAndObservations() { + // Using a static flag to ensure that test data and elasticsearch index is only created once. + // Creating this data and the index is time consuming and as such want to avoid having to repeat for each test. + // Normally would use a static @BeforeClass method for this purpose, but Autowired objects cannot be accessed in static methods. + if(!dataLoaded) { + Patient pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Arthur"); + patient0Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + createObservationsForPatient(patient0Id); + pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Johnathan"); + patient1Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + createObservationsForPatient(patient1Id); + pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Michael"); + patient2Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + createObservationsForPatient(patient2Id); + dataLoaded = true; + + } + + } + + private void createObservationsForPatient(IIdType thePatientId) { + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd2, 5); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd0, 10); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd1, 5); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd2, categoryCd2, 5); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd3, categoryCd3, 5); + } + + private void createFiveObservationsForPatientCodeCategory(IIdType thePatientId, String theObservationCode, String theCategoryCode, + Integer theTimeOffset) { + Calendar observationDate = new GregorianCalendar(); + + for (int idx=0; idx<5; idx++ ) { + Observation obs = new Observation(); + obs.getSubject().setReferenceElement(thePatientId); + obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem); + obs.setValue(new StringType(theObservationCode + "_0")); + observationDate.add(Calendar.HOUR, -theTimeOffset+idx); + Date effectiveDtm = observationDate.getTime(); + obs.setEffective(new DateTimeType(effectiveDtm)); + obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem); + String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue(); + observationPatientMap.put(observationId, thePatientId.getValue()); + observationCategoryMap.put(observationId, theCategoryCode); + observationCodeMap.put(observationId, theObservationCode); + observationEffectiveMap.put(observationId, effectiveDtm); + } + } + + protected ServletRequestDetails mockSrd() { + return mySrd; + } + + @Test + public void testLastNAllPatients() { + + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + sortedObservationCodes.add(observationCd3); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null,105); + } + + @Test + public void testLastNNoPatients() { + + SearchParameterMap params = new SearchParameterMap(); + params.setLastNMax(1); + + params.setLastN(true); + Map requestParameters = new HashMap<>(); + when(mySrd.getParameters()).thenReturn(requestParameters); + + List actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + + assertEquals(4, actual.size()); + } + + private void executeTestCase(SearchParameterMap params, List sortedPatients, List sortedObservationCodes, List theCategories, int expectedObservationCount) { + List actual; + params.setLastN(true); + + Map requestParameters = new HashMap<>(); + params.setLastNMax(100); + + when(mySrd.getParameters()).thenReturn(requestParameters); + + actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + + assertEquals(expectedObservationCount, actual.size()); + + validateSorting(actual, sortedPatients, sortedObservationCodes, theCategories); + } + + private void validateSorting(List theObservationIds, List thePatientIds, List theCodes, List theCategores) { + int theNextObservationIdx = 0; + // Validate patient grouping + for (String patientId : thePatientIds) { + assertEquals(patientId, observationPatientMap.get(theObservationIds.get(theNextObservationIdx))); + theNextObservationIdx = validateSortingWithinPatient(theObservationIds,theNextObservationIdx,theCodes, theCategores, patientId); + } + assertEquals(theObservationIds.size(), theNextObservationIdx); + } + + private int validateSortingWithinPatient(List theObservationIds, int theFirstObservationIdxForPatient, List theCodes, + List theCategories, String thePatientId) { + int theNextObservationIdx = theFirstObservationIdxForPatient; + for (String codeValue : theCodes) { + assertEquals(codeValue, observationCodeMap.get(theObservationIds.get(theNextObservationIdx))); + // Validate sorting within code group + theNextObservationIdx = validateSortingWithinCode(theObservationIds,theNextObservationIdx, + observationCodeMap.get(theObservationIds.get(theNextObservationIdx)), theCategories, thePatientId); + } + return theNextObservationIdx; + } + + private int validateSortingWithinCode(List theObservationIds, int theFirstObservationIdxForPatientAndCode, String theObservationCode, + List theCategories, String thePatientId) { + int theNextObservationIdx = theFirstObservationIdxForPatientAndCode; + Date lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); + theNextObservationIdx++; + while(theObservationCode.equals(observationCodeMap.get(theObservationIds.get(theNextObservationIdx))) + && thePatientId.equals(observationPatientMap.get(theObservationIds.get(theNextObservationIdx)))) { + // Check that effective date is before that of the previous observation. + assertTrue(lastEffectiveDt.compareTo(observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx))) > 0); + lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); + + // Check that observation is in one of the specified categories (if applicable) + if (theCategories != null && !theCategories.isEmpty()) { + assertTrue(theCategories.contains(observationCategoryMap.get(theObservationIds.get(theNextObservationIdx)))); + } + theNextObservationIdx++; + if (theNextObservationIdx >= theObservationIds.size()) { + // Have reached the end of the Observation list. + break; + } + } + return theNextObservationIdx; + } + + @Test + public void testLastNSinglePatient() { + + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + sortedObservationCodes.add(observationCd3); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,35); + + params = new SearchParameterMap(); + ReferenceParam patientParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); + + sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + sortedObservationCodes.add(observationCd3); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,35); + } + + protected ReferenceAndListParam buildReferenceAndListParam(ReferenceParam... theReference) { + ReferenceOrListParam myReferenceOrListParam = new ReferenceOrListParam(); + for (ReferenceParam referenceParam : theReference) { + myReferenceOrListParam.addOr(referenceParam); + } + return new ReferenceAndListParam().addAnd(myReferenceOrListParam); + } + + @Test + public void testLastNMultiplePatients() { + + // Two Subject parameters. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + sortedObservationCodes.add(observationCd3); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null,70); + + // Two Patient parameters + params = new SearchParameterMap(); + ReferenceParam patientParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam patientParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(patientParam1, patientParam3)); + + sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + executeTestCase(params,sortedPatients, sortedObservationCodes, null,70); + + } + + @Test + public void testLastNSingleCategory() { + + // One category parameter. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd0); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); + + // Another category parameter. + params = new SearchParameterMap(); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + categoryParam = new TokenParam(categorySystem, categoryCd2); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + myCategories = new ArrayList<>(); + myCategories.add(categoryCd2); + + sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); + + } + + @Test + public void testLastNMultipleCategories() { + + // Two category parameters. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd0); + TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd1); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd0); + myCategories.add(categoryCd1); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 60); + } + + @Test + public void testLastNSingleCode() { + + // One code parameter. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + TokenParam code = new TokenParam(codeSystem, observationCd0); + params.add(Observation.SP_CODE, buildTokenAndListParam(code)); + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null, 45); + + // Another code parameter. + params = new SearchParameterMap(); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + code = new TokenParam(codeSystem, observationCd2); + params.add(Observation.SP_CODE, buildTokenAndListParam(code)); + sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null, 15); + + } + + @Test + public void testLastNMultipleCodes() { + + // Two code parameters. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); + TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1); + params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null, 75); + + } + + @Test + public void testLastNSinglePatientCategoryCode() { + + // One patient, category and code. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + TokenParam code = new TokenParam(codeSystem, observationCd0); + params.add(Observation.SP_CODE, buildTokenAndListParam(code)); + TokenParam category = new TokenParam(categorySystem, categoryCd2); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(category)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 5); + + } + + @Test + public void testLastNMultiplePatientsCategoriesCodes() { + + // Two patients, categories and codes. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + + TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); + TokenParam codeParam2 = new TokenParam(codeSystem, observationCd2); + params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd2); + + TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd1); + TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd2); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd1); + myCategories.add(categoryCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); + + } + + protected TokenAndListParam buildTokenAndListParam(TokenParam... theToken) { + TokenOrListParam myTokenOrListParam = new TokenOrListParam(); + for (TokenParam tokenParam : theToken) { + myTokenOrListParam.addOr(tokenParam); + } + return new TokenAndListParam().addAnd(myTokenOrListParam); + } + + @AfterClass + public static void afterClassClearContext() { + TestUtil.clearAllStaticFieldsForUnitTest(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java new file mode 100644 index 00000000000..7978c956e15 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java @@ -0,0 +1,133 @@ +package ca.uhn.fhir.jpa.dao.r4; + +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.dao.SearchBuilder; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.server.RestfulServer; +import ca.uhn.fhir.util.TestUtil; +import org.hl7.fhir.r4.model.Observation; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.matchesPattern; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +public class FhirResourceDaoR4SearchLastNAsyncIT extends BaseR4SearchLastN { + + @Autowired + protected DaoConfig myDaoConfig; + + private List originalPreFetchThresholds; + + @Before + public void before() { + + RestfulServer myServer = new RestfulServer(myFhirCtx); + myServer.setPagingProvider(myDatabaseBackedPagingProvider); + + when(mySrd.getServer()).thenReturn(myServer); + + // Set pre-fetch sizes small so that most tests are forced to do multiple fetches. + // This will allow testing a common use case where result set is larger than first fetch size but smaller than the normal query chunk size. + originalPreFetchThresholds = myDaoConfig.getSearchPreFetchThresholds(); + List mySmallerPreFetchThresholds = new ArrayList<>(); + mySmallerPreFetchThresholds.add(20); + mySmallerPreFetchThresholds.add(400); + mySmallerPreFetchThresholds.add(-1); + myDaoConfig.setSearchPreFetchThresholds(mySmallerPreFetchThresholds); + + SearchBuilder.setIsTest(true); + + } + + @After + public void after() { + myDaoConfig.setSearchPreFetchThresholds(originalPreFetchThresholds); + SearchBuilder.setIsTest(false); + } + + @Test + public void testLastNChunking() { + + // Set up search parameters that will return 75 Observations. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); + TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1); + params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + + params.setLastN(true); + params.setLastNMax(100); + + Map requestParameters = new HashMap<>(); + when(mySrd.getParameters()).thenReturn(requestParameters); + + // Set chunk size to 50 + SearchBuilder.setIsTest(true); + + // Expand default fetch sizes to ensure all observations are returned in first page: + List myBiggerPreFetchThresholds = new ArrayList<>(); + myBiggerPreFetchThresholds.add(100); + myBiggerPreFetchThresholds.add(1000); + myBiggerPreFetchThresholds.add(-1); + myDaoConfig.setSearchPreFetchThresholds(myBiggerPreFetchThresholds); + + myCaptureQueriesListener.clear(); + List results = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + assertEquals(75, results.size()); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + List queries = myCaptureQueriesListener + .getSelectQueriesForCurrentThread() + .stream() + .map(t -> t.getSql(true, false)) + .collect(Collectors.toList()); + + // 1 query to lookup up Search from cache, and 2 chunked queries to retrieve resources by PID. + assertEquals(3, queries.size()); + + // The first chunked query should have a full complement of PIDs + StringBuilder firstQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'"); + for (int pidIndex = 1; pidIndex<50; pidIndex++) { + firstQueryPattern.append(" , '[0-9]+'"); + } + firstQueryPattern.append("\\).*"); + assertThat(queries.get(1), matchesPattern(firstQueryPattern.toString())); + + // the second chunked query should be padded with "-1". + StringBuilder secondQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'"); + for (int pidIndex = 1; pidIndex<25; pidIndex++) { + secondQueryPattern.append(" , '[0-9]+'"); + } + for (int pidIndex = 0; pidIndex<25; pidIndex++) { + secondQueryPattern.append(" , '-1'"); + } + secondQueryPattern.append("\\).*"); + assertThat(queries.get(2), matchesPattern(secondQueryPattern.toString())); + + } + + @AfterClass + public static void afterClassClearContext() { + TestUtil.clearAllStaticFieldsForUnitTest(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java index 20ab38370c9..8614a9e7c7e 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java @@ -1,29 +1,15 @@ package ca.uhn.fhir.jpa.dao.r4; -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.jpa.api.dao.*; -import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; -import ca.uhn.fhir.jpa.dao.BaseJpaTest; import ca.uhn.fhir.jpa.dao.SearchBuilder; -import ca.uhn.fhir.jpa.rp.r4.ObservationResourceProvider; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener; import ca.uhn.fhir.rest.param.*; -import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.util.TestUtil; -import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.*; import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.PlatformTransactionManager; import java.util.*; import java.util.stream.Collectors; @@ -33,553 +19,38 @@ import static org.junit.Assert.*; import static org.mockito.Mockito.when; @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class }) -public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { - - @Autowired - @Qualifier("myPatientDaoR4") - protected IFhirResourceDaoPatient myPatientDao; - - @Autowired - @Qualifier("myObservationDaoR4") - protected IFhirResourceDaoObservation myObservationDao; - - @Autowired - protected DaoConfig myDaoConfig; - - @Autowired - protected FhirContext myFhirCtx; - - @Autowired - protected PlatformTransactionManager myPlatformTransactionManager; - - @Override - protected FhirContext getContext() { - return myFhirCtx; - } - - @Override - protected PlatformTransactionManager getTxManager() { - return myPlatformTransactionManager; - } - - @Autowired - protected CircularQueueCaptureQueriesListener myCaptureQueriesListener; - - ObservationResourceProvider observationRp = new ObservationResourceProvider(); - - private final String observationCd0 = "code0"; - private final String observationCd1 = "code1"; - private final String observationCd2 = "code2"; - - private final String categoryCd0 = "category0"; - private final String categoryCd1 = "category1"; - private final String categoryCd2 = "category2"; - - private final String codeSystem = "http://mycode.com"; - private final String categorySystem = "http://mycategory.com"; - - // Using static variables including the flag below so that we can initalize the database and indexes once - // (all of the tests only read from the DB and indexes and so no need to re-initialze them for each test). - private static boolean dataLoaded = false; - - private static IIdType patient0Id = null; - private static IIdType patient1Id = null; - private static IIdType patient2Id = null; - - private static final Map observationPatientMap = new HashMap<>(); - private static final Map observationCategoryMap = new HashMap<>(); - private static final Map observationCodeMap = new HashMap<>(); - private static final Map observationEffectiveMap = new HashMap<>(); - - @Before - public void beforeCreateTestPatientsAndObservations() { - // Using a static flag here to ensure that load is only done once. Reason for this is that we cannot - // access Autowired objects in @BeforeClass method. - if(!dataLoaded) { - Patient pt = new Patient(); - pt.addName().setFamily("Lastn").addGiven("Arthur"); - patient0Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); - createObservationsForPatient(patient0Id); - pt = new Patient(); - pt.addName().setFamily("Lastn").addGiven("Johnathan"); - patient1Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); - createObservationsForPatient(patient1Id); - pt = new Patient(); - pt.addName().setFamily("Lastn").addGiven("Michael"); - patient2Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); - createObservationsForPatient(patient2Id); - dataLoaded = true; - } - - observationRp.setDao(myObservationDao); - - } +public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN { @After public void resetMaximumPageSize() { SearchBuilder.setIsTest(false); } - private void createObservationsForPatient(IIdType thePatientId) { - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd2, 5); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd0, 10); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd1, 5); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd2, categoryCd2, 5); - } - - private void createFiveObservationsForPatientCodeCategory(IIdType thePatientId, String theObservationCode, String theCategoryCode, - Integer theTimeOffset) { - Calendar observationDate = new GregorianCalendar(); - - for (int idx=0; idx<5; idx++ ) { - Observation obs = new Observation(); - obs.getSubject().setReferenceElement(thePatientId); - obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem); - obs.setValue(new StringType(theObservationCode + "_0")); - observationDate.add(Calendar.HOUR, -theTimeOffset+idx); - Date effectiveDtm = observationDate.getTime(); - obs.setEffective(new DateTimeType(effectiveDtm)); - obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem); - String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue(); - observationPatientMap.put(observationId, thePatientId.getValue()); - observationCategoryMap.put(observationId, theCategoryCode); - observationCodeMap.put(observationId, theObservationCode); - observationEffectiveMap.put(observationId, effectiveDtm); - } - } - - private ServletRequestDetails mockSrd() { - return mySrd; - } - @Test - public void testLastNAllPatients() { + public void testLastNChunking() { + // Set up search parameters that will return 75 Observations. SearchParameterMap params = new SearchParameterMap(); ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null,90); - } - - @Test - public void testLastNNoPatients() { - - SearchParameterMap params = new SearchParameterMap(); - params.setLastNMax(1); - - List sortedPatients = new ArrayList<>(); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - -// executeTestCase(params, sortedPatients, sortedObservationCodes, null,3); - params.setLastN(true); - Map requestParameters = new HashMap<>(); - when(mySrd.getParameters()).thenReturn(requestParameters); - - List actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); - - assertEquals(3, actual.size()); - } - - private void executeTestCase(SearchParameterMap params, List sortedPatients, List sortedObservationCodes, List theCategories, int expectedObservationCount) { - List actual; - params.setLastN(true); - - Map requestParameters = new HashMap<>(); - params.setLastNMax(100); - - when(mySrd.getParameters()).thenReturn(requestParameters); - - actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); - - assertEquals(expectedObservationCount, actual.size()); - - validateSorting(actual, sortedPatients, sortedObservationCodes, theCategories); - } - - private void validateSorting(List theObservationIds, List thePatientIds, List theCodes, List theCategores) { - int theNextObservationIdx = 0; - // Validate patient grouping - for (String patientId : thePatientIds) { - assertEquals(patientId, observationPatientMap.get(theObservationIds.get(theNextObservationIdx))); - theNextObservationIdx = validateSortingWithinPatient(theObservationIds,theNextObservationIdx,theCodes, theCategores, patientId); - } - assertEquals(theObservationIds.size(), theNextObservationIdx); - } - - private int validateSortingWithinPatient(List theObservationIds, int theFirstObservationIdxForPatient, List theCodes, - List theCategories, String thePatientId) { - int theNextObservationIdx = theFirstObservationIdxForPatient; - for (String codeValue : theCodes) { - assertEquals(codeValue, observationCodeMap.get(theObservationIds.get(theNextObservationIdx))); - // Validate sorting within code group - theNextObservationIdx = validateSortingWithinCode(theObservationIds,theNextObservationIdx, - observationCodeMap.get(theObservationIds.get(theNextObservationIdx)), theCategories, thePatientId); - } - return theNextObservationIdx; - } - - private int validateSortingWithinCode(List theObservationIds, int theFirstObservationIdxForPatientAndCode, String theObservationCode, - List theCategories, String thePatientId) { - int theNextObservationIdx = theFirstObservationIdxForPatientAndCode; - Date lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); - theNextObservationIdx++; - while(theObservationCode.equals(observationCodeMap.get(theObservationIds.get(theNextObservationIdx))) - && thePatientId.equals(observationPatientMap.get(theObservationIds.get(theNextObservationIdx)))) { - // Check that effective date is before that of the previous observation. - assertTrue(lastEffectiveDt.compareTo(observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx))) > 0); - lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); - - // Check that observation is in one of the specified categories (if applicable) - if (theCategories != null && !theCategories.isEmpty()) { - assertTrue(theCategories.contains(observationCategoryMap.get(theObservationIds.get(theNextObservationIdx)))); - } - theNextObservationIdx++; - if (theNextObservationIdx >= theObservationIds.size()) { - // Have reached the end of the Observation list. - break; - } - } - return theNextObservationIdx; - } - - @Test - public void testLastNSinglePatient() { - - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients,sortedObservationCodes, null,30); - - params = new SearchParameterMap(); - ReferenceParam patientParam = new ReferenceParam("Patient", "", patient0Id.getValue()); - params.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); - - sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - - sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients,sortedObservationCodes, null,30); - } - - private ReferenceAndListParam buildReferenceAndListParam(ReferenceParam... theReference) { - ReferenceOrListParam myReferenceOrListParam = new ReferenceOrListParam(); - for (ReferenceParam referenceParam : theReference) { - myReferenceOrListParam.addOr(referenceParam); - } - return new ReferenceAndListParam().addAnd(myReferenceOrListParam); - } - - @Test - public void testLastNMultiplePatients() { - - // Two Subject parameters. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null,60); - - // Two Patient parameters - params = new SearchParameterMap(); - ReferenceParam patientParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam patientParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(patientParam1, patientParam3)); - - sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - executeTestCase(params,sortedPatients, sortedObservationCodes, null,60); - - } - - @Test - public void testLastNSingleCategory() { - - // One category parameter. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - - TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - List myCategories = new ArrayList<>(); - myCategories.add(categoryCd0); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); - - // Another category parameter. - params = new SearchParameterMap(); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - categoryParam = new TokenParam(categorySystem, categoryCd2); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - myCategories = new ArrayList<>(); - myCategories.add(categoryCd2); - - sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); - - } - - @Test - public void testLastNMultipleCategories() { - - // Two category parameters. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - - TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd0); - TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd1); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); - List myCategories = new ArrayList<>(); - myCategories.add(categoryCd0); - myCategories.add(categoryCd1); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 60); - } - - @Test - public void testLastNSingleCode() { - - // One code parameter. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - - TokenParam code = new TokenParam(codeSystem, observationCd0); - params.add(Observation.SP_CODE, buildTokenAndListParam(code)); - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null, 45); - - // Another code parameter. - params = new SearchParameterMap(); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - code = new TokenParam(codeSystem, observationCd2); - params.add(Observation.SP_CODE, buildTokenAndListParam(code)); - sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null, 15); - - } - - @Test - public void testLastNMultipleCodes() { - - // Two code parameters. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1); params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null, 75); - - } - - @Test - public void testLastNSinglePatientCategoryCode() { - - // One patient, category and code. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - TokenParam code = new TokenParam(codeSystem, observationCd0); - params.add(Observation.SP_CODE, buildTokenAndListParam(code)); - TokenParam category = new TokenParam(categorySystem, categoryCd2); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(category)); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - - List myCategories = new ArrayList<>(); - myCategories.add(categoryCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 5); - - } - - @Test - public void testLastNMultiplePatientsCategoriesCodes() { - - // Two patients, categories and codes. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - - TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); - TokenParam codeParam2 = new TokenParam(codeSystem, observationCd2); - params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd2); - - TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd1); - TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd2); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); - List myCategories = new ArrayList<>(); - myCategories.add(categoryCd1); - myCategories.add(categoryCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); - - } - - private TokenAndListParam buildTokenAndListParam(TokenParam... theToken) { - TokenOrListParam myTokenOrListParam = new TokenOrListParam(); - for (TokenParam tokenParam : theToken) { - myTokenOrListParam.addOr(tokenParam); - } - return new TokenAndListParam().addAnd(myTokenOrListParam); - } - - @Test - public void testLastNWithChunkedQuery() { - SearchBuilder.setIsTest(true); - Integer numberOfObservations = SearchBuilder.getMaximumPageSize()+1; - Calendar observationDate = new GregorianCalendar(); - - List myObservationIds = new ArrayList<>(); - List myPatientIds = new ArrayList<>(); - List myPatientReferences = new ArrayList<>(); - for (int idx=0; idx actual; params.setLastN(true); + params.setLastNMax(100); Map requestParameters = new HashMap<>(); - params.setLastNMax(1); - - params.setCount(numberOfObservations); - when(mySrd.getParameters()).thenReturn(requestParameters); - myCaptureQueriesListener.clear(); - actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + // Set chunk size to 50 + SearchBuilder.setIsTest(true); + myCaptureQueriesListener.clear(); + List results = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + assertEquals(75, results.size()); myCaptureQueriesListener.logSelectQueriesForCurrentThread(); List queries = myCaptureQueriesListener .getSelectQueriesForCurrentThread() @@ -587,22 +58,29 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { .map(t -> t.getSql(true, false)) .collect(Collectors.toList()); - // First chunked query - String resultingQueryNotFormatted = queries.get(0); - assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '[0-9]+' , '[0-9]+' , '[0-9]+'\\).*")); + // Two chunked queries executed by the QueryIterator (in current thread) and two chunked queries to retrieve resources by PID. + assertEquals(4, queries.size()); - // Second chunked query chunk - resultingQueryNotFormatted = queries.get(1); - assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '-1' , '-1' , '-1'\\).*")); - - assertEquals(numberOfObservations, (Integer)actual.size()); - for(IIdType observationId : myObservationIds) { - myObservationDao.delete(observationId); + // The first and third chunked queries should have a full complement of PIDs + StringBuilder firstQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'"); + for (int pidIndex = 1; pidIndex<50; pidIndex++) { + firstQueryPattern.append(" , '[0-9]+'"); } + firstQueryPattern.append("\\).*"); + assertThat(queries.get(0), matchesPattern(firstQueryPattern.toString())); + assertThat(queries.get(2), matchesPattern(firstQueryPattern.toString())); - for (IIdType patientId : myPatientIds) { - myPatientDao.delete(patientId); + // the second and fourth chunked queries should be padded with "-1". + StringBuilder secondQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'"); + for (int pidIndex = 1; pidIndex<25; pidIndex++) { + secondQueryPattern.append(" , '[0-9]+'"); } + for (int pidIndex = 0; pidIndex<25; pidIndex++) { + secondQueryPattern.append(" , '-1'"); + } + secondQueryPattern.append("\\).*"); + assertThat(queries.get(1), matchesPattern(secondQueryPattern.toString())); + assertThat(queries.get(3), matchesPattern(secondQueryPattern.toString())); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java index 944dae05e7d..fd1942840da 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java @@ -107,7 +107,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 100); assertEquals(1, observationIdsOnly.size()); assertEquals(SINGLE_OBSERVATION_PID, observationIdsOnly.get(0)); @@ -181,14 +181,14 @@ public class PersistObservationIndexedSearchParamLastNR4IT { SearchParameterMap searchParameterMap = new SearchParameterMap(); searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams); //searchParameterMap. - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(100, observationIdsOnly.size()); // Filter the results by category code. TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 100); assertEquals(50, observationIdsOnly.size()); @@ -277,7 +277,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { SearchParameterMap searchParameterMap = new SearchParameterMap(); searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams); - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(100, observationIdsOnly.size()); assertTrue(observationIdsOnly.contains("55")); @@ -295,7 +295,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { observation = myResourceIndexedObservationLastNDao.findForIdentifier("55"); assertNull(observation); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(99, observationIdsOnly.size()); assertTrue(!observationIdsOnly.contains("55")); @@ -316,7 +316,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(1, observationIdsOnly.size()); assertTrue(observationIdsOnly.contains(SINGLE_OBSERVATION_PID)); @@ -339,7 +339,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { assertEquals(newEffectiveDtm.getValue(), updatedObservationEntity.getEffectiveDtm()); // Repeat earlier Elasticsearch query. This time, should return no matches. - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(0, observationIdsOnly.size()); // Try again with the new patient ID. @@ -348,7 +348,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam))); searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); // Should see the observation returned now. assertEquals(1, observationIdsOnly.size()); @@ -398,11 +398,11 @@ public class PersistObservationIndexedSearchParamLastNR4IT { SearchParameterMap searchParameterMap = new SearchParameterMap(); // execute Observation ID search - Composite Aggregation - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1, 200); assertEquals(20, observationIdsOnly.size()); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 200); assertEquals(38, observationIdsOnly.size()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index 21a7e08b8dc..25215936831 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -81,7 +81,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { subjectParam = new ReferenceParam("Patient", "", "9"); searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3, 100); assertEquals(60, observations.size()); @@ -153,7 +153,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { TokenParam codeParam2 = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-2"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(20, observations.size()); @@ -165,7 +165,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(20, observations.size()); @@ -198,7 +198,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { TokenParam codeParam = new TokenParam("test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(5, observations.size()); @@ -215,7 +215,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(10, observations.size()); } @@ -232,7 +232,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { codeParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(5, observations.size()); @@ -248,7 +248,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(0, observations.size()); // Invalid subject @@ -259,7 +259,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(0, observations.size()); // Invalid observation code @@ -270,7 +270,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-999"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(0, observations.size()); // Invalid category code @@ -281,7 +281,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(0, observations.size()); } @@ -393,7 +393,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { @Test public void testLastNNoParamsQuery() { SearchParameterMap searchParameterMap = new SearchParameterMap(); - List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 1); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 1, 100); assertEquals(2, observations.size()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java index b78e14821b3..416c949f6ce 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java @@ -103,13 +103,13 @@ public class LastNElasticsearchSvcSingleObservationIT { searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); // execute Observation ID search - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 100); assertEquals(1, observationIdsOnly.size()); assertEquals(RESOURCEPID, observationIdsOnly.get(0)); // execute Observation search for all search fields - List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3, 100); validateFullObservationSearch(observations); } diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java index 858dcc0c7ec..3acb7743771 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java @@ -37,6 +37,7 @@ public class SearchRuntimeDetails { private boolean myLoadSynchronous; private String myQueryString; private SearchStatusEnum mySearchStatus; + private int myFoundIndexMatchesCount; public SearchRuntimeDetails(RequestDetails theRequestDetails, String theSearchUuid) { myRequestDetails = theRequestDetails; mySearchUuid = theSearchUuid; @@ -67,6 +68,14 @@ public class SearchRuntimeDetails { myFoundMatchesCount = theFoundMatchesCount; } + public int getFoundIndexMatchesCount() { + return myFoundIndexMatchesCount; + } + + public void setFoundIndexMatchesCount(int theFoundIndexMatchesCount) { + myFoundIndexMatchesCount = theFoundIndexMatchesCount; + } + public boolean getLoadSynchronous() { return myLoadSynchronous; }