honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changes to better support chunking of queries and paging of results for lastn operation and to monitor performance.

This commit is contained in:
ianmarshall 2020-05-14 12:08:55 -04:00
parent 8fcdc78077
commit eb1d1c2b27
15 changed files with 867 additions and 620 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java
View File

@ -1719,7 +1719,7 @@ public enum Pointcut {
/**
* <b>Performance Tracing Hook:</b>
* This hook is invoked when a search has failed for any reason. When this pointcut
* This hook is invoked when a search has completed. When this pointcut
* is invoked, a pass in the Search Coordinator has completed successfully, but
* not all possible resources have been loaded yet so a future paging request
* may trigger a new task that will load further resources.
@ -1757,6 +1757,44 @@ public enum Pointcut {
"ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails"
),
/**
* <b>Performance Tracing Hook:</b>
* This hook is invoked when a query involving an external index (e.g. Elasticsearch) has completed. When this pointcut
* is invoked, an initial list of resource IDs has been generated which will be used as part of a subsequent database query.
* <p>
* Note that this is a performance tracing hook. Use with caution in production
* systems, since calling it may (or may not) carry a cost.
* </p>
* Hooks may accept the following parameters:
* <ul>
* <li>
* ca.uhn.fhir.rest.api.server.RequestDetails - A bean containing details about the request that is about to be processed, including details such as the
* resource type and logical ID (if any) and other FHIR-specific aspects of the request which have been
* pulled out of the servlet request. Note that the bean
* properties are not all guaranteed to be populated, depending on how early during processing the
* exception occurred.
* </li>
* <li>
* ca.uhn.fhir.rest.server.servlet.ServletRequestDetails - A bean containing details about the request that is about to be processed, including details such as the
* resource type and logical ID (if any) and other FHIR-specific aspects of the request which have been
* pulled out of the servlet request. This parameter is identical to the RequestDetails parameter above but will
* only be populated when operating in a RestfulServer implementation. It is provided as a convenience.
* </li>
* <li>
* ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails - Contains details about the search being
* performed. Hooks should not modify this object.
* </li>
* </ul>
* <p>
* Hooks should return <code>void</code>.
* </p>
*/
JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE(void.class,
"ca.uhn.fhir.rest.api.server.RequestDetails",
"ca.uhn.fhir.rest.server.servlet.ServletRequestDetails",
"ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails"
),
/**
* <b>Performance Tracing Hook:</b>
* Invoked when the storage engine is about to reuse the results of

86
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java
View File

@ -131,7 +131,7 @@ public class SearchBuilder implements ISearchBuilder {
*/
// NB: keep public
public static final int MAXIMUM_PAGE_SIZE = 800;
public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 4;
public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50;
public static boolean myIsTest = false;
private static final List<ResourcePersistentId> EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<>());
@ -249,7 +249,7 @@ public class SearchBuilder implements ISearchBuilder {
init(theParams, theSearchUuid, theRequestPartitionId);
List<TypedQuery<Long>> queries = createQuery(null, null, true, theRequest);
List<TypedQuery<Long>> queries = createQuery(null, null, true, theRequest, null);
return new CountQueryIterator(queries.get(0));
}
@ -283,7 +283,8 @@ public class SearchBuilder implements ISearchBuilder {
}
private List<TypedQuery<Long>> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) {
private List<TypedQuery<Long>> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest,
SearchRuntimeDetails theSearchRuntimeDetails) {
List<ResourcePersistentId> pids = new ArrayList<>();
/*
@ -310,17 +311,26 @@ public class SearchBuilder implements ISearchBuilder {
throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request");
}
}
Integer myMaxObservationsPerCode = null;
Integer myMaxObservationsPerCode;
if(myParams.getLastNMax() != null) {
myMaxObservationsPerCode = myParams.getLastNMax();
} else {
throw new InvalidRequestException("Max parameter is required for $lastn operation");
}
List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode);
List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode, theMaximumResults);
for (String lastnResourceId : lastnResourceIds) {
pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId));
}
}
if (theSearchRuntimeDetails != null) {
theSearchRuntimeDetails.setFoundIndexMatchesCount(pids.size());
HookParams params = new HookParams()
.add(RequestDetails.class, theRequest)
.addIfMatchesType(ServletRequestDetails.class, theRequest)
.add(SearchRuntimeDetails.class, theSearchRuntimeDetails);
JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params);
}
if (pids.isEmpty()) {
// Will never match
pids = Collections.singletonList(new ResourcePersistentId(-1L));
@ -331,24 +341,27 @@ public class SearchBuilder implements ISearchBuilder {
ArrayList<TypedQuery<Long>> myQueries = new ArrayList<>();
if (!pids.isEmpty()) {
if (theMaximumResults != null && pids.size() > theMaximumResults) {
pids.subList(0,theMaximumResults-1);
}
new QueryChunker<Long>().chunk(ResourcePersistentId.toLongList(pids), t->{
doCreateChunkedQueries(t, sort, theMaximumResults, theCount, theRequest, myQueries);
doCreateChunkedQueries(t, sort, theCount, theRequest, myQueries);
});
} else {
myQueries.add(createQuery(sort,theMaximumResults, theCount, theRequest, null));
myQueries.add(createChunkedQuery(sort,theMaximumResults, theCount, theRequest, null));
}
return myQueries;
}
private void doCreateChunkedQueries(List<Long> thePids, SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, ArrayList<TypedQuery<Long>> theQueries) {
if(thePids.size() < MAXIMUM_PAGE_SIZE) {
thePids = normalizeIdListForLastNInClause(thePids);
private void doCreateChunkedQueries(List<Long> thePids, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<TypedQuery<Long>> theQueries) {
if(thePids.size() < getMaximumPageSize()) {
normalizeIdListForLastNInClause(thePids);
}
theQueries.add(createQuery(sort, theMaximumResults, theCount, theRequest, thePids));
theQueries.add(createChunkedQuery(sort, thePids.size(), theCount, theRequest, thePids));
}
private TypedQuery<Long> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List<Long> thePidList) {
private TypedQuery<Long> createChunkedQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List<Long> thePidList) {
CriteriaQuery<Long> outerQuery;
/*
* Sort
@ -501,11 +514,6 @@ public class SearchBuilder implements ISearchBuilder {
}
private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) {
List<Long> retVal = new ArrayList<>();
for (Long lastnResourceId : lastnResourceIds) {
retVal.add(lastnResourceId);
}
/*
The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying
numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info:
@ -514,23 +522,23 @@ public class SearchBuilder implements ISearchBuilder {
Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of
arguments never exceeds the maximum specified below.
*/
int listSize = retVal.size();
int listSize = lastnResourceIds.size();
if(listSize > 1 && listSize < 10) {
padIdListWithPlaceholders(retVal, 10);
padIdListWithPlaceholders(lastnResourceIds, 10);
} else if (listSize > 10 && listSize < 50) {
padIdListWithPlaceholders(retVal, 50);
padIdListWithPlaceholders(lastnResourceIds, 50);
} else if (listSize > 50 && listSize < 100) {
padIdListWithPlaceholders(retVal, 100);
padIdListWithPlaceholders(lastnResourceIds, 100);
} else if (listSize > 100 && listSize < 200) {
padIdListWithPlaceholders(retVal, 200);
padIdListWithPlaceholders(lastnResourceIds, 200);
} else if (listSize > 200 && listSize < 500) {
padIdListWithPlaceholders(retVal, 500);
padIdListWithPlaceholders(lastnResourceIds, 500);
} else if (listSize > 500 && listSize < 800) {
padIdListWithPlaceholders(retVal, 800);
padIdListWithPlaceholders(lastnResourceIds, 800);
}
return retVal;
return lastnResourceIds;
}
private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
@ -664,10 +672,17 @@ public class SearchBuilder implements ISearchBuilder {
private void doLoadPids(Collection<ResourcePersistentId> thePids, Collection<ResourcePersistentId> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation,
Map<ResourcePersistentId, Integer> thePosition, RequestDetails theRequest) {
Map<ResourcePersistentId, Integer> thePosition) {
List<Long> myLongPersistentIds;
if(thePids.size() < getMaximumPageSize()) {
myLongPersistentIds = normalizeIdListForLastNInClause(ResourcePersistentId.toLongList(thePids));
} else {
myLongPersistentIds = ResourcePersistentId.toLongList(thePids);
}
// -- get the resource from the searchView
Collection<ResourceSearchView> resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(ResourcePersistentId.toLongList(thePids));
Collection<ResourceSearchView> resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(myLongPersistentIds);
//-- preload all tags with tag definition if any
Map<ResourcePersistentId, Collection<ResourceTag>> tagMap = getResourceTagMap(resourceSearchViewList);
@ -768,7 +783,7 @@ public class SearchBuilder implements ISearchBuilder {
List<ResourcePersistentId> pids = new ArrayList<>(thePids);
new QueryChunker<ResourcePersistentId>().chunk(pids, t->{
doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position, theDetails);
doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position);
});
}
@ -1096,9 +1111,8 @@ public class SearchBuilder implements ISearchBuilder {
private final RequestDetails myRequest;
private Iterator<ResourcePersistentId> myCurrentIterator;
private Set<ResourcePersistentId> myCurrentPids;
private final Set<ResourcePersistentId> myCurrentPids;
private ResourcePersistentId myNext;
private int myPageSize = myDaoConfig.getEverythingIncludesFetchPageSize();
IncludesIterator(Set<ResourcePersistentId> thePidSet, RequestDetails theRequest) {
myCurrentPids = new HashSet<>(thePidSet);
@ -1151,12 +1165,12 @@ public class SearchBuilder implements ISearchBuilder {
private ResourcePersistentId myNext;
private Iterator<ResourcePersistentId> myPreResultsIterator;
private ScrollableResultsIterator<Long> myResultsIterator;
private SortSpec mySort;
private final SortSpec mySort;
private boolean myStillNeedToFetchIncludes;
private int mySkipCount = 0;
private int myNonSkipCount = 0;
private List<TypedQuery<Long>> myQueryList;
private List<TypedQuery<Long>> myQueryList = new ArrayList<>();
private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
mySearchRuntimeDetails = theSearchRuntimeDetails;
@ -1210,7 +1224,7 @@ public class SearchBuilder implements ISearchBuilder {
if (myNext == null) {
while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) {
// Update iterator with next chunk if necessary.
if (!myResultsIterator.hasNext() && !myQueryList.isEmpty()) {
if (!myResultsIterator.hasNext()) {
retrieveNextIteratorQuery();
}
@ -1312,8 +1326,10 @@ public class SearchBuilder implements ISearchBuilder {
}
private void initializeIteratorQuery(Integer theMaxResultsToFetch) {
if (myQueryList == null || myQueryList.isEmpty()) {
myQueryList = createQuery(mySort, theMaxResultsToFetch, false, myRequest);
if (myQueryList.isEmpty()) {
// Capture times for Lucene/Elasticsearch queries as well
mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
myQueryList = createQuery(mySort, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails);
}
mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());

5
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/PerformanceTracingLoggingInterceptor.java
View File

@ -79,6 +79,11 @@ public class PerformanceTracingLoggingInterceptor {
log("SqlQuery {} failed in {} - Found {} matches", theOutcome.getSearchUuid(), theOutcome.getQueryStopwatch(), theOutcome.getFoundMatchesCount());
}
@Hook(value = Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE)
public void indexSearchQueryComplete(SearchRuntimeDetails theOutcome) {
log("Index query for {} completed in {} - Found {} matches", theOutcome.getSearchUuid(), theOutcome.getQueryStopwatch(), theOutcome.getFoundIndexMatchesCount());
}
@Hook(value = Pointcut.JPA_PERFTRACE_INFO)
public void info(StorageProcessingMessage theMessage) {
log("[INFO] " + theMessage);

35
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java
View File

@ -2,7 +2,6 @@ package ca.uhn.fhir.jpa.search.lastn;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.dstu2.resource.Observation;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.jpa.search.lastn.json.CodeJson;
@ -49,9 +48,9 @@ import static org.apache.commons.lang3.StringUtils.isBlank;
public class ElasticsearchSvcImpl implements IElasticsearchSvc {
RestHighLevelClient myRestHighLevelClient;
private final RestHighLevelClient myRestHighLevelClient;
ObjectMapper objectMapper = new ObjectMapper();
private final ObjectMapper objectMapper = new ObjectMapper();
private final String GROUP_BY_SUBJECT = "group_by_subject";
private final String GROUP_BY_CODE = "group_by_code";
@ -201,14 +200,15 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
@Override
// TODO: Should eliminate dependency on SearchParameterMap in API.
public List<String> executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) {
public List<String> executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch) {
String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME};
try {
List<SearchResponse> responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, topHitsInclude);
List<String> observationIds = new ArrayList<>();
for (SearchResponse response : responses) {
// observationIds.addAll(buildObservationIdList(response));
observationIds.addAll(buildObservationList(response, t -> t.getIdentifier(), theSearchParameterMap));
Integer maxResultsToAdd = theMaxResultsToFetch - observationIds.size();
observationIds.addAll(buildObservationList(response, t -> t.getIdentifier(), theSearchParameterMap, maxResultsToAdd));
}
return observationIds;
} catch (IOException theE) {
@ -216,7 +216,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
}
}
private List<SearchResponse> buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, String[] topHitsInclude) {
private List<SearchResponse> buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode,
String[] topHitsInclude) {
List<SearchResponse> responses = new ArrayList<>();
if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) {
ArrayList<String> subjectReferenceCriteria = new ArrayList<>();
@ -254,12 +255,12 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
@VisibleForTesting
// TODO: Should eliminate dependency on SearchParameterMap in API.
List<ObservationJson> executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) {
List<ObservationJson> executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch) {
try {
List<SearchResponse> responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, null);
List<ObservationJson> observationDocuments = new ArrayList<>();
for (SearchResponse response : responses) {
observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap));
observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap, theMaxResultsToFetch));
}
return observationDocuments;
} catch (IOException theE) {
@ -337,12 +338,22 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
return theObservationList;
}
private <T> List<T> buildObservationList(SearchResponse theSearchResponse, Function<ObservationJson,T> setValue, SearchParameterMap theSearchParameterMap) throws IOException {
private <T> List<T> buildObservationList(SearchResponse theSearchResponse, Function<ObservationJson,T> setValue,
SearchParameterMap theSearchParameterMap, Integer theMaxResultsToFetch) throws IOException {
List<T> theObservationList = new ArrayList<>();
if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) {
for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) {
if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
break;
}
for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) {
if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
break;
}
for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) {
if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
break;
}
String indexedObservation = lastNMatch.getSourceAsString();
ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class);
theObservationList.add(setValue.apply(observationJson));
@ -351,7 +362,13 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
}
} else {
for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(theSearchResponse)) {
if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
break;
}
for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) {
if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
break;
}
String indexedObservation = lastNMatch.getSourceAsString();
ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class);
theObservationList.add(setValue.apply(observationJson));

3
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java
View File

@ -5,6 +5,5 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import java.util.List;
public interface IElasticsearchSvc {
List<String> executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode);
List<String> executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch);
}

5
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java
View File

@ -102,10 +102,14 @@ public class TestR4Config extends BaseJavaConfigR4 {
};
retVal.setDriver(new org.h2.Driver());
// retVal.setDriver(new org.postgresql.Driver());
retVal.setUrl("jdbc:h2:mem:testdb_r4");
// retVal.setUrl("jdbc:postgresql://localhost:5432/hapi");
retVal.setMaxWaitMillis(10000);
retVal.setUsername("");
// retVal.setUsername("hapi");
retVal.setPassword("");
// retVal.setPassword("HapiFHIR");
retVal.setMaxTotal(ourMaxThreads);
SLF4JLogLevel level = SLF4JLogLevel.INFO;
@ -145,6 +149,7 @@ public class TestR4Config extends BaseJavaConfigR4 {
extraProperties.put("hibernate.show_sql", "false");
extraProperties.put("hibernate.hbm2ddl.auto", "update");
extraProperties.put("hibernate.dialect", H2Dialect.class.getName());
// extraProperties.put("hibernate.dialect", org.hibernate.dialect.PostgreSQL95Dialect.class.getName());
extraProperties.put("hibernate.search.model_mapping", ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory.class.getName());
extraProperties.put("hibernate.search.default.directory_provider", "local-heap");
extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT");

3
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java
View File

@ -24,7 +24,9 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config {
private static final String ELASTIC_VERSION = "6.5.4";
protected final String elasticsearchHost = "localhost";
protected final String elasticsearchUserId = "";
// protected final String elasticsearchUserId = "elastic";
protected final String elasticsearchPassword = "";
// protected final String elasticsearchPassword = "changeme";
@Override
@ -34,6 +36,7 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config {
// Force elasticsearch to start first
int httpPort = embeddedElasticSearch().getHttpPort();
// int httpPort = 9301;
ourLog.info("ElasticSearch started on port: {}", httpPort);
new ElasticsearchHibernatePropertiesBuilder()

1
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java
View File

@ -10,6 +10,7 @@ public class TestR4ConfigWithElasticsearchClient extends TestR4ConfigWithElastic
@Bean()
public ElasticsearchSvcImpl myElasticsearchSvc() {
int elasticsearchPort = embeddedElasticSearch().getHttpPort();
// int elasticsearchPort = 9301;
return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword);
}

543
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java Normal file
View File

@ -0,0 +1,543 @@
package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoPatient;
import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient;
import ca.uhn.fhir.jpa.dao.BaseJpaTest;
import ca.uhn.fhir.jpa.dao.SearchBuilder;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.rest.param.ReferenceAndListParam;
import ca.uhn.fhir.rest.param.ReferenceOrListParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.TokenAndListParam;
import ca.uhn.fhir.rest.param.TokenOrListParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.server.RestfulServer;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.util.TestUtil;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.DateTimeType;
import org.hl7.fhir.r4.model.Observation;
import org.hl7.fhir.r4.model.Patient;
import org.hl7.fhir.r4.model.StringType;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.transaction.PlatformTransactionManager;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.matchesPattern;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class })
public class BaseR4SearchLastN extends BaseJpaTest {
@Autowired
@Qualifier("myPatientDaoR4")
protected IFhirResourceDaoPatient<Patient> myPatientDao;
@Autowired
@Qualifier("myObservationDaoR4")
protected IFhirResourceDaoObservation<Observation> myObservationDao;
@Autowired
protected DaoConfig myDaoConfig;
@Autowired
protected FhirContext myFhirCtx;
@Autowired
protected PlatformTransactionManager myPlatformTransactionManager;
@Override
protected FhirContext getContext() {
return myFhirCtx;
}
@Override
protected PlatformTransactionManager getTxManager() {
return myPlatformTransactionManager;
}
protected final String observationCd0 = "code0";
protected final String observationCd1 = "code1";
protected final String observationCd2 = "code2";
private final String observationCd3 = "code3";
protected final String categoryCd0 = "category0";
private final String categoryCd1 = "category1";
private final String categoryCd2 = "category2";
private final String categoryCd3 = "category3";
protected final String codeSystem = "http://mycode.com";
private final String categorySystem = "http://mycategory.com";
// Using static variables including the flag below so that we can initalize the database and indexes once
// (all of the tests only read from the DB and indexes and so no need to re-initialze them for each test).
private static boolean dataLoaded = false;
protected static IIdType patient0Id = null;
protected static IIdType patient1Id = null;
protected static IIdType patient2Id = null;
private static final Map<String, String> observationPatientMap = new HashMap<>();
private static final Map<String, String> observationCategoryMap = new HashMap<>();
private static final Map<String, String> observationCodeMap = new HashMap<>();
private static final Map<String, Date> observationEffectiveMap = new HashMap<>();
@Before
public void beforeCreateTestPatientsAndObservations() {
// Using a static flag to ensure that test data and elasticsearch index is only created once.
// Creating this data and the index is time consuming and as such want to avoid having to repeat for each test.
// Normally would use a static @BeforeClass method for this purpose, but Autowired objects cannot be accessed in static methods.
if(!dataLoaded) {
Patient pt = new Patient();
pt.addName().setFamily("Lastn").addGiven("Arthur");
patient0Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
createObservationsForPatient(patient0Id);
pt = new Patient();
pt.addName().setFamily("Lastn").addGiven("Johnathan");
patient1Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
createObservationsForPatient(patient1Id);
pt = new Patient();
pt.addName().setFamily("Lastn").addGiven("Michael");
patient2Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
createObservationsForPatient(patient2Id);
dataLoaded = true;
}
}
private void createObservationsForPatient(IIdType thePatientId) {
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd2, 5);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd0, 10);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd1, 5);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd2, categoryCd2, 5);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd3, categoryCd3, 5);
}
private void createFiveObservationsForPatientCodeCategory(IIdType thePatientId, String theObservationCode, String theCategoryCode,
Integer theTimeOffset) {
Calendar observationDate = new GregorianCalendar();
for (int idx=0; idx<5; idx++ ) {
Observation obs = new Observation();
obs.getSubject().setReferenceElement(thePatientId);
obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem);
obs.setValue(new StringType(theObservationCode + "_0"));
observationDate.add(Calendar.HOUR, -theTimeOffset+idx);
Date effectiveDtm = observationDate.getTime();
obs.setEffective(new DateTimeType(effectiveDtm));
obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem);
String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue();
observationPatientMap.put(observationId, thePatientId.getValue());
observationCategoryMap.put(observationId, theCategoryCode);
observationCodeMap.put(observationId, theObservationCode);
observationEffectiveMap.put(observationId, effectiveDtm);
}
}
protected ServletRequestDetails mockSrd() {
return mySrd;
}
@Test
public void testLastNAllPatients() {
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
sortedObservationCodes.add(observationCd2);
sortedObservationCodes.add(observationCd3);
executeTestCase(params, sortedPatients, sortedObservationCodes, null,105);
}
@Test
public void testLastNNoPatients() {
SearchParameterMap params = new SearchParameterMap();
params.setLastNMax(1);
params.setLastN(true);
Map<String, String[]> requestParameters = new HashMap<>();
when(mySrd.getParameters()).thenReturn(requestParameters);
List<String> actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null));
assertEquals(4, actual.size());
}
private void executeTestCase(SearchParameterMap params, List<String> sortedPatients, List<String> sortedObservationCodes, List<String> theCategories, int expectedObservationCount) {
List<String> actual;
params.setLastN(true);
Map<String, String[]> requestParameters = new HashMap<>();
params.setLastNMax(100);
when(mySrd.getParameters()).thenReturn(requestParameters);
actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null));
assertEquals(expectedObservationCount, actual.size());
validateSorting(actual, sortedPatients, sortedObservationCodes, theCategories);
}
private void validateSorting(List<String> theObservationIds, List<String> thePatientIds, List<String> theCodes, List<String> theCategores) {
int theNextObservationIdx = 0;
// Validate patient grouping
for (String patientId : thePatientIds) {
assertEquals(patientId, observationPatientMap.get(theObservationIds.get(theNextObservationIdx)));
theNextObservationIdx = validateSortingWithinPatient(theObservationIds,theNextObservationIdx,theCodes, theCategores, patientId);
}
assertEquals(theObservationIds.size(), theNextObservationIdx);
}
private int validateSortingWithinPatient(List<String> theObservationIds, int theFirstObservationIdxForPatient, List<String> theCodes,
List<String> theCategories, String thePatientId) {
int theNextObservationIdx = theFirstObservationIdxForPatient;
for (String codeValue : theCodes) {
assertEquals(codeValue, observationCodeMap.get(theObservationIds.get(theNextObservationIdx)));
// Validate sorting within code group
theNextObservationIdx = validateSortingWithinCode(theObservationIds,theNextObservationIdx,
observationCodeMap.get(theObservationIds.get(theNextObservationIdx)), theCategories, thePatientId);
}
return theNextObservationIdx;
}
private int validateSortingWithinCode(List<String> theObservationIds, int theFirstObservationIdxForPatientAndCode, String theObservationCode,
List<String> theCategories, String thePatientId) {
int theNextObservationIdx = theFirstObservationIdxForPatientAndCode;
Date lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx));
theNextObservationIdx++;
while(theObservationCode.equals(observationCodeMap.get(theObservationIds.get(theNextObservationIdx)))
&& thePatientId.equals(observationPatientMap.get(theObservationIds.get(theNextObservationIdx)))) {
// Check that effective date is before that of the previous observation.
assertTrue(lastEffectiveDt.compareTo(observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx))) > 0);
lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx));
// Check that observation is in one of the specified categories (if applicable)
if (theCategories != null && !theCategories.isEmpty()) {
assertTrue(theCategories.contains(observationCategoryMap.get(theObservationIds.get(theNextObservationIdx))));
}
theNextObservationIdx++;
if (theNextObservationIdx >= theObservationIds.size()) {
// Have reached the end of the Observation list.
break;
}
}
return theNextObservationIdx;
}
@Test
public void testLastNSinglePatient() {
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
sortedObservationCodes.add(observationCd2);
sortedObservationCodes.add(observationCd3);
executeTestCase(params, sortedPatients,sortedObservationCodes, null,35);
params = new SearchParameterMap();
ReferenceParam patientParam = new ReferenceParam("Patient", "", patient0Id.getValue());
params.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam));
sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
sortedObservationCodes.add(observationCd2);
sortedObservationCodes.add(observationCd3);
executeTestCase(params, sortedPatients,sortedObservationCodes, null,35);
}
protected ReferenceAndListParam buildReferenceAndListParam(ReferenceParam... theReference) {
ReferenceOrListParam myReferenceOrListParam = new ReferenceOrListParam();
for (ReferenceParam referenceParam : theReference) {
myReferenceOrListParam.addOr(referenceParam);
}
return new ReferenceAndListParam().addAnd(myReferenceOrListParam);
}
@Test
public void testLastNMultiplePatients() {
// Two Subject parameters.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
sortedObservationCodes.add(observationCd2);
sortedObservationCodes.add(observationCd3);
executeTestCase(params, sortedPatients, sortedObservationCodes, null,70);
// Two Patient parameters
params = new SearchParameterMap();
ReferenceParam patientParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam patientParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(patientParam1, patientParam3));
sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient2Id.getValue());
executeTestCase(params,sortedPatients, sortedObservationCodes, null,70);
}
@Test
public void testLastNSingleCategory() {
// One category parameter.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
List<String> myCategories = new ArrayList<>();
myCategories.add(categoryCd0);
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30);
// Another category parameter.
params = new SearchParameterMap();
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
categoryParam = new TokenParam(categorySystem, categoryCd2);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
myCategories = new ArrayList<>();
myCategories.add(categoryCd2);
sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30);
}
@Test
public void testLastNMultipleCategories() {
// Two category parameters.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd0);
TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd1);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2));
List<String> myCategories = new ArrayList<>();
myCategories.add(categoryCd0);
myCategories.add(categoryCd1);
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 60);
}
@Test
public void testLastNSingleCode() {
// One code parameter.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
TokenParam code = new TokenParam(codeSystem, observationCd0);
params.add(Observation.SP_CODE, buildTokenAndListParam(code));
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
executeTestCase(params, sortedPatients, sortedObservationCodes, null, 45);
// Another code parameter.
params = new SearchParameterMap();
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
code = new TokenParam(codeSystem, observationCd2);
params.add(Observation.SP_CODE, buildTokenAndListParam(code));
sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, null, 15);
}
@Test
public void testLastNMultipleCodes() {
// Two code parameters.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0);
TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1);
params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2));
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
executeTestCase(params, sortedPatients, sortedObservationCodes, null, 75);
}
@Test
public void testLastNSinglePatientCategoryCode() {
// One patient, category and code.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
TokenParam code = new TokenParam(codeSystem, observationCd0);
params.add(Observation.SP_CODE, buildTokenAndListParam(code));
TokenParam category = new TokenParam(categorySystem, categoryCd2);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(category));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
List<String> myCategories = new ArrayList<>();
myCategories.add(categoryCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 5);
}
@Test
public void testLastNMultiplePatientsCategoriesCodes() {
// Two patients, categories and codes.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0);
TokenParam codeParam2 = new TokenParam(codeSystem, observationCd2);
params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2));
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd2);
TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd1);
TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd2);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2));
List<String> myCategories = new ArrayList<>();
myCategories.add(categoryCd1);
myCategories.add(categoryCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30);
}
protected TokenAndListParam buildTokenAndListParam(TokenParam... theToken) {
TokenOrListParam myTokenOrListParam = new TokenOrListParam();
for (TokenParam tokenParam : theToken) {
myTokenOrListParam.addOr(tokenParam);
}
return new TokenAndListParam().addAnd(myTokenOrListParam);
}
@AfterClass
public static void afterClassClearContext() {
TestUtil.clearAllStaticFieldsForUnitTest();
}
}

133
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java Normal file
View File

@ -0,0 +1,133 @@
package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.dao.SearchBuilder;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.server.RestfulServer;
import ca.uhn.fhir.util.TestUtil;
import org.hl7.fhir.r4.model.Observation;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.matchesPattern;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.when;
@RunWith(SpringJUnit4ClassRunner.class)
public class FhirResourceDaoR4SearchLastNAsyncIT extends BaseR4SearchLastN {
@Autowired
protected DaoConfig myDaoConfig;
private List<Integer> originalPreFetchThresholds;
@Before
public void before() {
RestfulServer myServer = new RestfulServer(myFhirCtx);
myServer.setPagingProvider(myDatabaseBackedPagingProvider);
when(mySrd.getServer()).thenReturn(myServer);
// Set pre-fetch sizes small so that most tests are forced to do multiple fetches.
// This will allow testing a common use case where result set is larger than first fetch size but smaller than the normal query chunk size.
originalPreFetchThresholds = myDaoConfig.getSearchPreFetchThresholds();
List<Integer> mySmallerPreFetchThresholds = new ArrayList<>();
mySmallerPreFetchThresholds.add(20);
mySmallerPreFetchThresholds.add(400);
mySmallerPreFetchThresholds.add(-1);
myDaoConfig.setSearchPreFetchThresholds(mySmallerPreFetchThresholds);
SearchBuilder.setIsTest(true);
}
@After
public void after() {
myDaoConfig.setSearchPreFetchThresholds(originalPreFetchThresholds);
SearchBuilder.setIsTest(false);
}
@Test
public void testLastNChunking() {
// Set up search parameters that will return 75 Observations.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0);
TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1);
params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2));
params.setLastN(true);
params.setLastNMax(100);
Map<String, String[]> requestParameters = new HashMap<>();
when(mySrd.getParameters()).thenReturn(requestParameters);
// Set chunk size to 50
SearchBuilder.setIsTest(true);
// Expand default fetch sizes to ensure all observations are returned in first page:
List<Integer> myBiggerPreFetchThresholds = new ArrayList<>();
myBiggerPreFetchThresholds.add(100);
myBiggerPreFetchThresholds.add(1000);
myBiggerPreFetchThresholds.add(-1);
myDaoConfig.setSearchPreFetchThresholds(myBiggerPreFetchThresholds);
myCaptureQueriesListener.clear();
List<String> results = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null));
assertEquals(75, results.size());
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
List<String> queries = myCaptureQueriesListener
.getSelectQueriesForCurrentThread()
.stream()
.map(t -> t.getSql(true, false))
.collect(Collectors.toList());
// 1 query to lookup up Search from cache, and 2 chunked queries to retrieve resources by PID.
assertEquals(3, queries.size());
// The first chunked query should have a full complement of PIDs
StringBuilder firstQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'");
for (int pidIndex = 1; pidIndex<50; pidIndex++) {
firstQueryPattern.append(" , '[0-9]+'");
}
firstQueryPattern.append("\\).*");
assertThat(queries.get(1), matchesPattern(firstQueryPattern.toString()));
// the second chunked query should be padded with "-1".
StringBuilder secondQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'");
for (int pidIndex = 1; pidIndex<25; pidIndex++) {
secondQueryPattern.append(" , '[0-9]+'");
}
for (int pidIndex = 0; pidIndex<25; pidIndex++) {
secondQueryPattern.append(" , '-1'");
}
secondQueryPattern.append("\\).*");
assertThat(queries.get(2), matchesPattern(secondQueryPattern.toString()));
}
@AfterClass
public static void afterClassClearContext() {
TestUtil.clearAllStaticFieldsForUnitTest();
}
}

578
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java
View File

@ -1,29 +1,15 @@
package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.api.dao.*;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient;
import ca.uhn.fhir.jpa.dao.BaseJpaTest;
import ca.uhn.fhir.jpa.dao.SearchBuilder;
import ca.uhn.fhir.jpa.rp.r4.ObservationResourceProvider;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener;
import ca.uhn.fhir.rest.param.*;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.util.TestUtil;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.*;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.transaction.PlatformTransactionManager;
import java.util.*;
import java.util.stream.Collectors;
@ -33,553 +19,38 @@ import static org.junit.Assert.*;
import static org.mockito.Mockito.when;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class })
public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
@Autowired
@Qualifier("myPatientDaoR4")
protected IFhirResourceDaoPatient<Patient> myPatientDao;
@Autowired
@Qualifier("myObservationDaoR4")
protected IFhirResourceDaoObservation<Observation> myObservationDao;
@Autowired
protected DaoConfig myDaoConfig;
@Autowired
protected FhirContext myFhirCtx;
@Autowired
protected PlatformTransactionManager myPlatformTransactionManager;
@Override
protected FhirContext getContext() {
return myFhirCtx;
}
@Override
protected PlatformTransactionManager getTxManager() {
return myPlatformTransactionManager;
}
@Autowired
protected CircularQueueCaptureQueriesListener myCaptureQueriesListener;
ObservationResourceProvider observationRp = new ObservationResourceProvider();
private final String observationCd0 = "code0";
private final String observationCd1 = "code1";
private final String observationCd2 = "code2";
private final String categoryCd0 = "category0";
private final String categoryCd1 = "category1";
private final String categoryCd2 = "category2";
private final String codeSystem = "http://mycode.com";
private final String categorySystem = "http://mycategory.com";
// Using static variables including the flag below so that we can initalize the database and indexes once
// (all of the tests only read from the DB and indexes and so no need to re-initialze them for each test).
private static boolean dataLoaded = false;
private static IIdType patient0Id = null;
private static IIdType patient1Id = null;
private static IIdType patient2Id = null;
private static final Map<String, String> observationPatientMap = new HashMap<>();
private static final Map<String, String> observationCategoryMap = new HashMap<>();
private static final Map<String, String> observationCodeMap = new HashMap<>();
private static final Map<String, Date> observationEffectiveMap = new HashMap<>();
@Before
public void beforeCreateTestPatientsAndObservations() {
// Using a static flag here to ensure that load is only done once. Reason for this is that we cannot
// access Autowired objects in @BeforeClass method.
if(!dataLoaded) {
Patient pt = new Patient();
pt.addName().setFamily("Lastn").addGiven("Arthur");
patient0Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
createObservationsForPatient(patient0Id);
pt = new Patient();
pt.addName().setFamily("Lastn").addGiven("Johnathan");
patient1Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
createObservationsForPatient(patient1Id);
pt = new Patient();
pt.addName().setFamily("Lastn").addGiven("Michael");
patient2Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
createObservationsForPatient(patient2Id);
dataLoaded = true;
}
observationRp.setDao(myObservationDao);
}
public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN {
@After
public void resetMaximumPageSize() {
SearchBuilder.setIsTest(false);
}
private void createObservationsForPatient(IIdType thePatientId) {
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd2, 5);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd0, 10);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd1, 5);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd2, categoryCd2, 5);
}
private void createFiveObservationsForPatientCodeCategory(IIdType thePatientId, String theObservationCode, String theCategoryCode,
Integer theTimeOffset) {
Calendar observationDate = new GregorianCalendar();
for (int idx=0; idx<5; idx++ ) {
Observation obs = new Observation();
obs.getSubject().setReferenceElement(thePatientId);
obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem);
obs.setValue(new StringType(theObservationCode + "_0"));
observationDate.add(Calendar.HOUR, -theTimeOffset+idx);
Date effectiveDtm = observationDate.getTime();
obs.setEffective(new DateTimeType(effectiveDtm));
obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem);
String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue();
observationPatientMap.put(observationId, thePatientId.getValue());
observationCategoryMap.put(observationId, theCategoryCode);
observationCodeMap.put(observationId, theObservationCode);
observationEffectiveMap.put(observationId, effectiveDtm);
}
}
private ServletRequestDetails mockSrd() {
return mySrd;
}
@Test
public void testLastNAllPatients() {
public void testLastNChunking() {
// Set up search parameters that will return 75 Observations.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
sortedObservationCodes.add(observationCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, null,90);
}
@Test
public void testLastNNoPatients() {
SearchParameterMap params = new SearchParameterMap();
params.setLastNMax(1);
List<String> sortedPatients = new ArrayList<>();
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
sortedObservationCodes.add(observationCd2);
// executeTestCase(params, sortedPatients, sortedObservationCodes, null,3);
params.setLastN(true);
Map<String, String[]> requestParameters = new HashMap<>();
when(mySrd.getParameters()).thenReturn(requestParameters);
List<String> actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null));
assertEquals(3, actual.size());
}
private void executeTestCase(SearchParameterMap params, List<String> sortedPatients, List<String> sortedObservationCodes, List<String> theCategories, int expectedObservationCount) {
List<String> actual;
params.setLastN(true);
Map<String, String[]> requestParameters = new HashMap<>();
params.setLastNMax(100);
when(mySrd.getParameters()).thenReturn(requestParameters);
actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null));
assertEquals(expectedObservationCount, actual.size());
validateSorting(actual, sortedPatients, sortedObservationCodes, theCategories);
}
private void validateSorting(List<String> theObservationIds, List<String> thePatientIds, List<String> theCodes, List<String> theCategores) {
int theNextObservationIdx = 0;
// Validate patient grouping
for (String patientId : thePatientIds) {
assertEquals(patientId, observationPatientMap.get(theObservationIds.get(theNextObservationIdx)));
theNextObservationIdx = validateSortingWithinPatient(theObservationIds,theNextObservationIdx,theCodes, theCategores, patientId);
}
assertEquals(theObservationIds.size(), theNextObservationIdx);
}
private int validateSortingWithinPatient(List<String> theObservationIds, int theFirstObservationIdxForPatient, List<String> theCodes,
List<String> theCategories, String thePatientId) {
int theNextObservationIdx = theFirstObservationIdxForPatient;
for (String codeValue : theCodes) {
assertEquals(codeValue, observationCodeMap.get(theObservationIds.get(theNextObservationIdx)));
// Validate sorting within code group
theNextObservationIdx = validateSortingWithinCode(theObservationIds,theNextObservationIdx,
observationCodeMap.get(theObservationIds.get(theNextObservationIdx)), theCategories, thePatientId);
}
return theNextObservationIdx;
}
private int validateSortingWithinCode(List<String> theObservationIds, int theFirstObservationIdxForPatientAndCode, String theObservationCode,
List<String> theCategories, String thePatientId) {
int theNextObservationIdx = theFirstObservationIdxForPatientAndCode;
Date lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx));
theNextObservationIdx++;
while(theObservationCode.equals(observationCodeMap.get(theObservationIds.get(theNextObservationIdx)))
&& thePatientId.equals(observationPatientMap.get(theObservationIds.get(theNextObservationIdx)))) {
// Check that effective date is before that of the previous observation.
assertTrue(lastEffectiveDt.compareTo(observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx))) > 0);
lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx));
// Check that observation is in one of the specified categories (if applicable)
if (theCategories != null && !theCategories.isEmpty()) {
assertTrue(theCategories.contains(observationCategoryMap.get(theObservationIds.get(theNextObservationIdx))));
}
theNextObservationIdx++;
if (theNextObservationIdx >= theObservationIds.size()) {
// Have reached the end of the Observation list.
break;
}
}
return theNextObservationIdx;
}
@Test
public void testLastNSinglePatient() {
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
sortedObservationCodes.add(observationCd2);
executeTestCase(params, sortedPatients,sortedObservationCodes, null,30);
params = new SearchParameterMap();
ReferenceParam patientParam = new ReferenceParam("Patient", "", patient0Id.getValue());
params.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam));
sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
sortedObservationCodes.add(observationCd2);
executeTestCase(params, sortedPatients,sortedObservationCodes, null,30);
}
private ReferenceAndListParam buildReferenceAndListParam(ReferenceParam... theReference) {
ReferenceOrListParam myReferenceOrListParam = new ReferenceOrListParam();
for (ReferenceParam referenceParam : theReference) {
myReferenceOrListParam.addOr(referenceParam);
}
return new ReferenceAndListParam().addAnd(myReferenceOrListParam);
}
@Test
public void testLastNMultiplePatients() {
// Two Subject parameters.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
sortedObservationCodes.add(observationCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, null,60);
// Two Patient parameters
params = new SearchParameterMap();
ReferenceParam patientParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam patientParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(patientParam1, patientParam3));
sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient2Id.getValue());
executeTestCase(params,sortedPatients, sortedObservationCodes, null,60);
}
@Test
public void testLastNSingleCategory() {
// One category parameter.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
List<String> myCategories = new ArrayList<>();
myCategories.add(categoryCd0);
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30);
// Another category parameter.
params = new SearchParameterMap();
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
categoryParam = new TokenParam(categorySystem, categoryCd2);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
myCategories = new ArrayList<>();
myCategories.add(categoryCd2);
sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30);
}
@Test
public void testLastNMultipleCategories() {
// Two category parameters.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd0);
TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd1);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2));
List<String> myCategories = new ArrayList<>();
myCategories.add(categoryCd0);
myCategories.add(categoryCd1);
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 60);
}
@Test
public void testLastNSingleCode() {
// One code parameter.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
TokenParam code = new TokenParam(codeSystem, observationCd0);
params.add(Observation.SP_CODE, buildTokenAndListParam(code));
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
executeTestCase(params, sortedPatients, sortedObservationCodes, null, 45);
// Another code parameter.
params = new SearchParameterMap();
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
code = new TokenParam(codeSystem, observationCd2);
params.add(Observation.SP_CODE, buildTokenAndListParam(code));
sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, null, 15);
}
@Test
public void testLastNMultipleCodes() {
// Two code parameters.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3));
TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0);
TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1);
params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2));
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd1);
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
sortedPatients.add(patient2Id.getValue());
executeTestCase(params, sortedPatients, sortedObservationCodes, null, 75);
}
@Test
public void testLastNSinglePatientCategoryCode() {
// One patient, category and code.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
TokenParam code = new TokenParam(codeSystem, observationCd0);
params.add(Observation.SP_CODE, buildTokenAndListParam(code));
TokenParam category = new TokenParam(categorySystem, categoryCd2);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(category));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
List<String> myCategories = new ArrayList<>();
myCategories.add(categoryCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 5);
}
@Test
public void testLastNMultiplePatientsCategoriesCodes() {
// Two patients, categories and codes.
SearchParameterMap params = new SearchParameterMap();
ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue());
ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue());
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2));
List<String> sortedPatients = new ArrayList<>();
sortedPatients.add(patient0Id.getValue());
sortedPatients.add(patient1Id.getValue());
TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0);
TokenParam codeParam2 = new TokenParam(codeSystem, observationCd2);
params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2));
List<String> sortedObservationCodes = new ArrayList<>();
sortedObservationCodes.add(observationCd0);
sortedObservationCodes.add(observationCd2);
TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd1);
TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd2);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2));
List<String> myCategories = new ArrayList<>();
myCategories.add(categoryCd1);
myCategories.add(categoryCd2);
executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30);
}
private TokenAndListParam buildTokenAndListParam(TokenParam... theToken) {
TokenOrListParam myTokenOrListParam = new TokenOrListParam();
for (TokenParam tokenParam : theToken) {
myTokenOrListParam.addOr(tokenParam);
}
return new TokenAndListParam().addAnd(myTokenOrListParam);
}
@Test
public void testLastNWithChunkedQuery() {
SearchBuilder.setIsTest(true);
Integer numberOfObservations = SearchBuilder.getMaximumPageSize()+1;
Calendar observationDate = new GregorianCalendar();
List<IIdType> myObservationIds = new ArrayList<>();
List<IIdType> myPatientIds = new ArrayList<>();
List<ReferenceParam> myPatientReferences = new ArrayList<>();
for (int idx=0; idx<numberOfObservations; idx++ ) {
Patient pt = new Patient();
pt.addName().setFamily("Lastn_" + idx).addGiven("Chunked");
IIdType patientId = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
myPatientIds.add(patientId);
ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId.getValue());
myPatientReferences.add(subjectParam);
Observation obs = new Observation();
obs.getSubject().setReferenceElement(patientId);
obs.getCode().addCoding().setCode(observationCd0).setSystem(codeSystem);
obs.setValue(new StringType(observationCd0 + "_0"));
observationDate.add(Calendar.HOUR, -1);
Date effectiveDtm = observationDate.getTime();
obs.setEffective(new DateTimeType(effectiveDtm));
obs.getCategoryFirstRep().addCoding().setCode(categoryCd0).setSystem(categorySystem);
myObservationIds.add(myObservationDao.create(obs, mockSrd()).getId());
}
SearchParameterMap params = new SearchParameterMap();
ReferenceParam[] referenceParams = new ReferenceParam[numberOfObservations];
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(myPatientReferences.toArray(referenceParams)));
TokenParam codeParam = new TokenParam(codeSystem, observationCd0);
params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
List<String> actual;
params.setLastN(true);
params.setLastNMax(100);
Map<String, String[]> requestParameters = new HashMap<>();
params.setLastNMax(1);
params.setCount(numberOfObservations);
when(mySrd.getParameters()).thenReturn(requestParameters);
myCaptureQueriesListener.clear();
actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null));
// Set chunk size to 50
SearchBuilder.setIsTest(true);
myCaptureQueriesListener.clear();
List<String> results = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null));
assertEquals(75, results.size());
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
List<String> queries = myCaptureQueriesListener
.getSelectQueriesForCurrentThread()
@ -587,22 +58,29 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
.map(t -> t.getSql(true, false))
.collect(Collectors.toList());
// First chunked query
String resultingQueryNotFormatted = queries.get(0);
assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '[0-9]+' , '[0-9]+' , '[0-9]+'\\).*"));
// Two chunked queries executed by the QueryIterator (in current thread) and two chunked queries to retrieve resources by PID.
assertEquals(4, queries.size());
// Second chunked query chunk
resultingQueryNotFormatted = queries.get(1);
assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '-1' , '-1' , '-1'\\).*"));
assertEquals(numberOfObservations, (Integer)actual.size());
for(IIdType observationId : myObservationIds) {
myObservationDao.delete(observationId);
// The first and third chunked queries should have a full complement of PIDs
StringBuilder firstQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'");
for (int pidIndex = 1; pidIndex<50; pidIndex++) {
firstQueryPattern.append(" , '[0-9]+'");
}
firstQueryPattern.append("\\).*");
assertThat(queries.get(0), matchesPattern(firstQueryPattern.toString()));
assertThat(queries.get(2), matchesPattern(firstQueryPattern.toString()));
for (IIdType patientId : myPatientIds) {
myPatientDao.delete(patientId);
// the second and fourth chunked queries should be padded with "-1".
StringBuilder secondQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'");
for (int pidIndex = 1; pidIndex<25; pidIndex++) {
secondQueryPattern.append(" , '[0-9]+'");
}
for (int pidIndex = 0; pidIndex<25; pidIndex++) {
secondQueryPattern.append(" , '-1'");
}
secondQueryPattern.append("\\).*");
assertThat(queries.get(1), matchesPattern(secondQueryPattern.toString()));
assertThat(queries.get(3), matchesPattern(secondQueryPattern.toString()));
}

20
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java
View File

@ -107,7 +107,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE);
searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam)));
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3);
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 100);
assertEquals(1, observationIdsOnly.size());
assertEquals(SINGLE_OBSERVATION_PID, observationIdsOnly.get(0));
@ -181,14 +181,14 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
SearchParameterMap searchParameterMap = new SearchParameterMap();
searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams);
//searchParameterMap.
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10);
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200);
assertEquals(100, observationIdsOnly.size());
// Filter the results by category code.
TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE);
searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam)));
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10);
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 100);
assertEquals(50, observationIdsOnly.size());
@ -277,7 +277,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
SearchParameterMap searchParameterMap = new SearchParameterMap();
searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams);
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10);
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200);
assertEquals(100, observationIdsOnly.size());
assertTrue(observationIdsOnly.contains("55"));
@ -295,7 +295,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
observation = myResourceIndexedObservationLastNDao.findForIdentifier("55");
assertNull(observation);
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10);
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200);
assertEquals(99, observationIdsOnly.size());
assertTrue(!observationIdsOnly.contains("55"));
@ -316,7 +316,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam)));
TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE);
searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam)));
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10);
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200);
assertEquals(1, observationIdsOnly.size());
assertTrue(observationIdsOnly.contains(SINGLE_OBSERVATION_PID));
@ -339,7 +339,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
assertEquals(newEffectiveDtm.getValue(), updatedObservationEntity.getEffectiveDtm());
// Repeat earlier Elasticsearch query. This time, should return no matches.
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10);
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200);
assertEquals(0, observationIdsOnly.size());
// Try again with the new patient ID.
@ -348,7 +348,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam)));
searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam)));
searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam)));
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10);
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200);
// Should see the observation returned now.
assertEquals(1, observationIdsOnly.size());
@ -398,11 +398,11 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
SearchParameterMap searchParameterMap = new SearchParameterMap();
// execute Observation ID search - Composite Aggregation
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1);
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1, 200);
assertEquals(20, observationIdsOnly.size());
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3);
observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 200);
assertEquals(38, observationIdsOnly.size());

22
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java
View File

@ -81,7 +81,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
subjectParam = new ReferenceParam("Patient", "", "9");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3);
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3, 100);
assertEquals(60, observations.size());
@ -153,7 +153,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
TokenParam codeParam2 = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-2");
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2));
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100);
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100);
assertEquals(20, observations.size());
@ -165,7 +165,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2));
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2));
observations = elasticsearchSvc.executeLastN(searchParameterMap, 100);
observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100);
assertEquals(20, observations.size());
@ -198,7 +198,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
TokenParam codeParam = new TokenParam("test-code-1");
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100);
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100);
assertEquals(5, observations.size());
@ -215,7 +215,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null);
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100);
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100);
assertEquals(10, observations.size());
}
@ -232,7 +232,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
codeParam.setModifier(TokenParamModifier.TEXT);
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100);
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100);
assertEquals(5, observations.size());
@ -248,7 +248,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1");
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100);
List<String> observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100);
assertEquals(0, observations.size());
// Invalid subject
@ -259,7 +259,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1");
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
observations = elasticsearchSvc.executeLastN(searchParameterMap, 100);
observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100);
assertEquals(0, observations.size());
// Invalid observation code
@ -270,7 +270,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-999");
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
observations = elasticsearchSvc.executeLastN(searchParameterMap, 100);
observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100);
assertEquals(0, observations.size());
// Invalid category code
@ -281,7 +281,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1");
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
observations = elasticsearchSvc.executeLastN(searchParameterMap, 100);
observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100);
assertEquals(0, observations.size());
}
@ -393,7 +393,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
@Test
public void testLastNNoParamsQuery() {
SearchParameterMap searchParameterMap = new SearchParameterMap();
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 1);
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 1, 100);
assertEquals(2, observations.size());

4
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java
View File

@ -103,13 +103,13 @@ public class LastNElasticsearchSvcSingleObservationIT {
searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam)));
// execute Observation ID search
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3);
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 100);
assertEquals(1, observationIdsOnly.size());
assertEquals(RESOURCEPID, observationIdsOnly.get(0));
// execute Observation search for all search fields
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3);
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3, 100);
validateFullObservationSearch(observations);
}

9
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java
View File

@ -37,6 +37,7 @@ public class SearchRuntimeDetails {
private boolean myLoadSynchronous;
private String myQueryString;
private SearchStatusEnum mySearchStatus;
private int myFoundIndexMatchesCount;
public SearchRuntimeDetails(RequestDetails theRequestDetails, String theSearchUuid) {
myRequestDetails = theRequestDetails;
mySearchUuid = theSearchUuid;
@ -67,6 +68,14 @@ public class SearchRuntimeDetails {
myFoundMatchesCount = theFoundMatchesCount;
}
public int getFoundIndexMatchesCount() {
return myFoundIndexMatchesCount;
}
public void setFoundIndexMatchesCount(int theFoundIndexMatchesCount) {
myFoundIndexMatchesCount = theFoundIndexMatchesCount;
}
public boolean getLoadSynchronous() {
return myLoadSynchronous;
}