diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties index dc5fe3ad700..73ba427f9c4 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties @@ -47,6 +47,7 @@ page.server_jpa.search=Search page.server_jpa.performance=Performance page.server_jpa.upgrading=Upgrade Guide page.server_jpa.diff=Diff Operation +page.server_jpa.lastn=LastN Operation section.server_jpa_empi.title=JPA Server: EMPI page.server_jpa_empi.empi=Enterprise Master Patient Index diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md new file mode 100644 index 00000000000..7f526541678 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md @@ -0,0 +1,45 @@ +# LastN Operation + +HAPI FHIR 5.1.0 introduced preliminary support for the `$lastn` operation described [here](http://hl7.org/fhir/observation-operation-lastn.html). + +This implementation of the `$lastn` operation requires an external Elasticsearch server implementation which is used to implement the indexes required by this operation. The following sections describe the current functionality supported by this operation and the configuration needed to enable this operation. + +# Functional Overview and Parameters + +As described in the [FHIR specification](http://hl7.org/fhir/observation-operation-lastn.html), the `$lastn` can be used to retrieve the most recent or last n=number of observations for one or more subjects. This implementation supports the following search parameters: + +* `subject=` or `patient=`: Identifier(s) of patient(s) to return Observation resources for. If not specified, returns most recent observations for all patients. +* `category=`: One or more category code search parameters used to filter Observations. +* `Observation.code=`: One or more `Observation.code` search parameters use to filter and group observations. If not specified, returns most recent observations for all `Observation.code` values. +* `date=`: Date search parameters used to filter Observations by `Observation.effectiveDtm`. +* `max=`: The maximum number of observations to return for each `Observation.code`. If not specified, returns only the most recent observation in each group. + +# Limitations + +Currently only Elasticsearch versions up to 6.5.4 are supported. + +Search parameters other than those listed above are currently not supported. + +The grouping of Observation resources by `Observation.code` means that the `$lastn` operation will not work in cases where `Observation.code` has more than one coding. + +# Deployment and Configuration + +The `$lastn` operation is disabled by default. The operation can be enabled by setting the DaoConfig#setLastNEnabled property (see [JavaDoc](/hapi-fhir/apidocs/hapi-fhir-jpaserver-api/ca/uhn/fhir/jpa/api/config/DaoConfig.html#setLastNEnabled(boolean))). + +In addition, the Elasticsearch client service, `ElasticsearchSvcImpl` will need to be instantiated with parameters specifying how to connect to the Elasticsearch server, for e.g.: + +```java + @Bean() + public ElasticsearchSvcImpl elasticsearchSvc() { + String elasticsearchHost = "localhost"; + String elasticsearchUserId = "elastic"; + String elasticsearchPassword = "changeme"; + int elasticsearchPort = 9301; + + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } +``` + +The Elasticsearch client service requires that security be enabled in the Elasticsearch clusters, and that an Elasticsearch user be available with permissions to create an index and to index, update and delete documents as needed. + +See the [JavaDoc](/hapi-fhir/apidocs/hapi-fhir-jpaserver-base/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.html) for more information regarding the Elasticsearch client service. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index ad463873cbe..dd65fbb430b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -311,9 +311,6 @@ public class SearchBuilder implements ISearchBuilder { throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); } } - if(myParams.getLastNMax() == null) { - throw new InvalidRequestException("Max parameter is required for $lastn operation"); - } List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults); for (String lastnResourceId : lastnResourceIds) { pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java index 4a1e3189a67..80d34042c92 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java @@ -60,6 +60,10 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider @OperationParam(name="code") TokenAndListParam theCode, + @Description(shortDefinition="The effective date of the observation") + @OperationParam(name="date") + DateAndListParam theDate, + @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -78,13 +82,16 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider SearchParameterMap paramMap = new SearchParameterMap(); paramMap.add(Observation.SP_CATEGORY, theCategory); paramMap.add(Observation.SP_CODE, theCode); + paramMap.add(Observation.SP_DATE, theDate); if (thePatient != null) { - paramMap.add("patient", thePatient); + paramMap.add(Observation.SP_PATIENT, thePatient); } if (theSubject != null) { - paramMap.add("subject", theSubject); + paramMap.add(Observation.SP_SUBJECT, theSubject); + } + if (theMax != null) { + paramMap.setLastNMax(theMax.getValue()); } - paramMap.setLastNMax(theMax.getValue()); if (theCount != null) { paramMap.setCount(theCount.getValue()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java index 2ad20aa1974..af5eb9938a2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java @@ -59,6 +59,10 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< @OperationParam(name="code") TokenAndListParam theCode, + @Description(shortDefinition="The effective date of the observation") + @OperationParam(name="date") + DateAndListParam theDate, + @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -77,13 +81,16 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< SearchParameterMap paramMap = new SearchParameterMap(); paramMap.add(Observation.SP_CATEGORY, theCategory); paramMap.add(Observation.SP_CODE, theCode); + paramMap.add(Observation.SP_DATE, theDate); if (thePatient != null) { paramMap.add(Observation.SP_PATIENT, thePatient); } if (theSubject != null) { paramMap.add(Observation.SP_SUBJECT, theSubject); } - paramMap.setLastNMax(theMax.getValue()); + if(theMax != null) { + paramMap.setLastNMax(theMax.getValue()); + } if (theCount != null) { paramMap.setCount(theCount.getValue()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java index b5cacc7c571..125a4a4c945 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java @@ -60,6 +60,10 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< @OperationParam(name="code") TokenAndListParam theCode, + @Description(shortDefinition="The effective date of the observation") + @OperationParam(name="date") + DateAndListParam theDate, + @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -78,13 +82,16 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< SearchParameterMap paramMap = new SearchParameterMap(); paramMap.add(Observation.SP_CATEGORY, theCategory); paramMap.add(Observation.SP_CODE, theCode); + paramMap.add(Observation.SP_DATE, theDate); if (thePatient != null) { paramMap.add(Observation.SP_PATIENT, thePatient); } if (theSubject != null) { paramMap.add(Observation.SP_SUBJECT, theSubject); } - paramMap.setLastNMax(theMax.getValue()); + if (theMax != null) { + paramMap.setLastNMax(theMax.getValue()); + } if (theCount != null) { paramMap.setCount(theCount.getValue()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 8893a4aaf66..5b0c177f0c1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -27,6 +27,8 @@ import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper; import ca.uhn.fhir.model.api.IQueryParameterType; +import ca.uhn.fhir.rest.param.DateParam; +import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; @@ -46,7 +48,9 @@ import org.shadehapi.elasticsearch.client.RequestOptions; import org.shadehapi.elasticsearch.client.RestHighLevelClient; import org.shadehapi.elasticsearch.common.xcontent.XContentType; import org.shadehapi.elasticsearch.index.query.BoolQueryBuilder; +import org.shadehapi.elasticsearch.index.query.MatchQueryBuilder; import org.shadehapi.elasticsearch.index.query.QueryBuilders; +import org.shadehapi.elasticsearch.index.query.RangeQueryBuilder; import org.shadehapi.elasticsearch.index.reindex.DeleteByQueryRequest; import org.shadehapi.elasticsearch.search.SearchHit; import org.shadehapi.elasticsearch.search.SearchHits; @@ -77,6 +81,7 @@ import static org.apache.commons.lang3.StringUtils.isBlank; public class ElasticsearchSvcImpl implements IElasticsearchSvc { + // Index Constants public static final String OBSERVATION_INDEX = "observation_index"; public static final String OBSERVATION_CODE_INDEX = "code_index"; public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity"; @@ -84,15 +89,34 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { public static final String OBSERVATION_INDEX_SCHEMA_FILE = "ObservationIndexSchema.json"; public static final String OBSERVATION_CODE_INDEX_SCHEMA_FILE = "ObservationCodeIndexSchema.json"; - private final RestHighLevelClient myRestHighLevelClient; - - private final ObjectMapper objectMapper = new ObjectMapper(); - + // Aggregation Constants private final String GROUP_BY_SUBJECT = "group_by_subject"; private final String GROUP_BY_SYSTEM = "group_by_system"; private final String GROUP_BY_CODE = "group_by_code"; - private final String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier"; + private final String MOST_RECENT_EFFECTIVE = "most_recent_effective"; + // Observation index document element names + private final String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier"; + private final String OBSERVATION_SUBJECT_FIELD_NAME = "subject"; + private final String OBSERVATION_CODEVALUE_FIELD_NAME = "codeconceptcodingcode"; + private final String OBSERVATION_CODESYSTEM_FIELD_NAME = "codeconceptcodingsystem"; + private final String OBSERVATION_CODEHASH_FIELD_NAME = "codeconceptcodingcode_system_hash"; + private final String OBSERVATION_CODEDISPLAY_FIELD_NAME = "codeconceptcodingdisplay"; + private final String OBSERVATION_CODE_TEXT_FIELD_NAME = "codeconcepttext"; + private final String OBSERVATION_EFFECTIVEDTM_FIELD_NAME = "effectivedtm"; + private final String OBSERVATION_CATEGORYHASH_FIELD_NAME = "categoryconceptcodingcode_system_hash"; + private final String OBSERVATION_CATEGORYVALUE_FIELD_NAME = "categoryconceptcodingcode"; + private final String OBSERVATION_CATEGORYSYSTEM_FIELD_NAME = "categoryconceptcodingsystem"; + private final String OBSERVATION_CATEGORYDISPLAY_FIELD_NAME = "categoryconceptcodingdisplay"; + private final String OBSERVATION_CATEGORYTEXT_FIELD_NAME = "categoryconcepttext"; + + // Code index document element names + private final String CODE_HASH = "codingcode_system_hash"; + private final String CODE_TEXT = "text"; + + private final RestHighLevelClient myRestHighLevelClient; + + private final ObjectMapper objectMapper = new ObjectMapper(); public ElasticsearchSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername, thePassword); @@ -171,7 +195,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { break; } SearchRequest myLastNRequest = buildObservationsSearchRequest(subject, theSearchParameterMap, theFhirContext, - createObservationSubjectAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); + createObservationSubjectAggregationBuilder(getMaxParameter(theSearchParameterMap), topHitsInclude)); try { SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext, @@ -182,7 +206,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } } else { SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, theFhirContext, - createObservationCodeAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); + createObservationCodeAggregationBuilder(getMaxParameter(theSearchParameterMap), topHitsInclude)); try { SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext, @@ -194,6 +218,14 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return searchResults; } + private int getMaxParameter(SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.getLastNMax() == null) { + return 1; + } else { + return theSearchParameterMap.getLastNMax(); + } + } + private List getSubjectReferenceCriteria(String thePatientParamName, String theSubjectParamName, SearchParameterMap theSearchParameterMap) { List subjectReferenceCriteria = new ArrayList<>(); @@ -227,8 +259,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return referenceList; } - private CompositeAggregationBuilder createObservationSubjectAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { - CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); + private CompositeAggregationBuilder createObservationSubjectAggregationBuilder(Integer theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { + CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder(OBSERVATION_SUBJECT_FIELD_NAME).field(OBSERVATION_SUBJECT_FIELD_NAME); List> compositeAggSubjectSources = new ArrayList<>(); compositeAggSubjectSources.add(subjectValuesBuilder); CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder(GROUP_BY_SUBJECT, compositeAggSubjectSources); @@ -239,14 +271,14 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } private TermsAggregationBuilder createObservationCodeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { - TermsAggregationBuilder observationCodeCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE, ValueType.STRING).field("codeconceptcodingcode"); + TermsAggregationBuilder observationCodeCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE, ValueType.STRING).field(OBSERVATION_CODEVALUE_FIELD_NAME); observationCodeCodeAggregationBuilder.order(BucketOrder.key(true)); // Top Hits Aggregation - observationCodeCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") - .sort("effectivedtm", SortOrder.DESC) + observationCodeCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits(MOST_RECENT_EFFECTIVE) + .sort(OBSERVATION_EFFECTIVEDTM_FIELD_NAME, SortOrder.DESC) .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); observationCodeCodeAggregationBuilder.size(10000); - TermsAggregationBuilder observationCodeSystemAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_SYSTEM, ValueType.STRING).field("codeconceptcodingsystem"); + TermsAggregationBuilder observationCodeSystemAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_SYSTEM, ValueType.STRING).field(OBSERVATION_CODESYSTEM_FIELD_NAME); observationCodeSystemAggregationBuilder.order(BucketOrder.key(true)); observationCodeSystemAggregationBuilder.subAggregation(observationCodeCodeAggregationBuilder); return observationCodeSystemAggregationBuilder; @@ -328,7 +360,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { private SearchHit[] getLastNMatches(Terms.Bucket theObservationCodeBucket) { Aggregations topHitObservationCodes = theObservationCodeBucket.getAggregations(); - ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + ParsedTopHits parsedTopHits = topHitObservationCodes.get(MOST_RECENT_EFFECTIVE); return parsedTopHits.getHits().getHits(); } @@ -342,6 +374,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); + addDateCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); searchSourceBuilder.query(boolQueryBuilder); } searchSourceBuilder.size(0); @@ -359,9 +392,10 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // Query BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery("subject", theSubjectParam)); + boolQueryBuilder.must(QueryBuilders.termQuery(OBSERVATION_SUBJECT_FIELD_NAME, theSubjectParam)); addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); + addDateCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); searchSourceBuilder.query(boolQueryBuilder); searchSourceBuilder.size(0); @@ -395,19 +429,19 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); } if (codeSystemHashList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode_system_hash", codeSystemHashList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYHASH_FIELD_NAME, codeSystemHashList)); } if (codeOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode", codeOnlyList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYVALUE_FIELD_NAME, codeOnlyList)); } if (systemOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingsystem", systemOnlyList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYSYSTEM_FIELD_NAME, systemOnlyList)); } if (textOnlyList.size() > 0) { BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconcepttext", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CATEGORYDISPLAY_FIELD_NAME, textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CATEGORYTEXT_FIELD_NAME, textOnlyParam)); } theBoolQueryBuilder.must(myTextBoolQueryBuilder); } @@ -493,19 +527,19 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); } if (codeSystemHashList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode_system_hash", codeSystemHashList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODEHASH_FIELD_NAME, codeSystemHashList)); } if (codeOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode", codeOnlyList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODEVALUE_FIELD_NAME, codeOnlyList)); } if (systemOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingsystem", systemOnlyList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODESYSTEM_FIELD_NAME, systemOnlyList)); } if (textOnlyList.size() > 0) { BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconcepttext", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CODEDISPLAY_FIELD_NAME, textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CODE_TEXT_FIELD_NAME, textOnlyParam)); } theBoolQueryBuilder.must(myTextBoolQueryBuilder); } @@ -513,6 +547,41 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } + private void addDateCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { + String dateParamName = LastNParameterHelper.getEffectiveParamName(theFhirContext); + if (theSearchParameterMap.containsKey(dateParamName)) { + List> andOrParams = theSearchParameterMap.get(dateParamName); + for (List nextAnd : andOrParams) { + BoolQueryBuilder myDateBoolQueryBuilder = new BoolQueryBuilder(); + for (IQueryParameterType nextOr : nextAnd) { + if (nextOr instanceof DateParam) { + DateParam myDate = (DateParam) nextOr; + createDateCriteria(myDate, myDateBoolQueryBuilder); + } + } + theBoolQueryBuilder.must(myDateBoolQueryBuilder); + } + } + } + + private void createDateCriteria(DateParam theDate, BoolQueryBuilder theBoolQueryBuilder) { + Long dateInstant = theDate.getValue().getTime(); + RangeQueryBuilder myRangeQueryBuilder = new RangeQueryBuilder(OBSERVATION_EFFECTIVEDTM_FIELD_NAME); + + ParamPrefixEnum prefix = theDate.getPrefix(); + if (prefix == ParamPrefixEnum.GREATERTHAN || prefix == ParamPrefixEnum.STARTS_AFTER) { + theBoolQueryBuilder.should(myRangeQueryBuilder.gt(dateInstant)); + } else if (prefix == ParamPrefixEnum.LESSTHAN || prefix == ParamPrefixEnum.ENDS_BEFORE) { + theBoolQueryBuilder.should(myRangeQueryBuilder.lt(dateInstant)); + } else if (prefix == ParamPrefixEnum.LESSTHAN_OR_EQUALS) { + theBoolQueryBuilder.should(myRangeQueryBuilder.lte(dateInstant)); + } else if (prefix == ParamPrefixEnum.GREATERTHAN_OR_EQUALS) { + theBoolQueryBuilder.should(myRangeQueryBuilder.gte(dateInstant)); + } else { + theBoolQueryBuilder.should(new MatchQueryBuilder(OBSERVATION_EFFECTIVEDTM_FIELD_NAME, dateInstant)); + } + } + @VisibleForTesting public List executeLastNWithAllFieldsForTest(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { return buildAndExecuteSearch(theSearchParameterMap, theFhirContext, null, t -> t, 100); @@ -604,9 +673,9 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); if (theCodeSystemHash != null) { - boolQueryBuilder.must(QueryBuilders.termQuery("codingcode_system_hash", theCodeSystemHash)); + boolQueryBuilder.must(QueryBuilders.termQuery(CODE_HASH, theCodeSystemHash)); } else { - boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("text", theText)); + boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(CODE_TEXT, theText)); } searchSourceBuilder.query(boolQueryBuilder); @@ -644,6 +713,11 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); } + @Override + public void close() throws IOException { + myRestHighLevelClient.close(); + } + private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { IndexRequest request = new IndexRequest(theIndexName); request.id(theDocumentId); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java index b9c4a662b0b..ebcb9127dc7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -25,6 +25,7 @@ import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import java.io.IOException; import java.util.List; public interface IElasticsearchSvc { @@ -75,4 +76,9 @@ public interface IElasticsearchSvc { */ void deleteObservationDocument(String theDocumentId); + /** + * Invoked when shutting down. + */ + void close() throws IOException; + } diff --git a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json index a91869e41d5..5f76e5de590 100644 --- a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json +++ b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json @@ -18,10 +18,10 @@ "type" : "keyword" }, "codeconceptcodingdisplay" : { - "type" : "keyword" + "type" : "text" }, "categoryconcepttext" : { - "type" : "keyword" + "type" : "text" }, "categoryconceptcodingcode" : { "type" : "keyword" @@ -33,7 +33,7 @@ "type" : "keyword" }, "categoryconceptcodingdisplay" : { - "type" : "keyword" + "type" : "text" }, "effectivedtm" : { "type" : "date" diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java index 9ff6bc4c491..b2e711b0c65 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java @@ -4,6 +4,9 @@ import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import javax.annotation.PreDestroy; +import java.io.IOException; + @Configuration public class TestR4ConfigWithElasticsearchClient extends TestR4ConfigWithElasticSearch { @@ -13,4 +16,9 @@ public class TestR4ConfigWithElasticsearchClient extends TestR4ConfigWithElastic return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); } + @PreDestroy + public void stopEsClient() throws IOException { + myElasticsearchSvc().close(); + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java index f990938c40a..976882aaf06 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java @@ -8,6 +8,10 @@ import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; import ca.uhn.fhir.jpa.dao.BaseJpaTest; import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.DateAndListParam; +import ca.uhn.fhir.rest.param.DateOrListParam; +import ca.uhn.fhir.rest.param.DateParam; +import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.ReferenceAndListParam; import ca.uhn.fhir.rest.param.ReferenceOrListParam; import ca.uhn.fhir.rest.param.ReferenceParam; @@ -115,6 +119,8 @@ public class BaseR4SearchLastN extends BaseJpaTest { private static final Map observationCodeMap = new HashMap<>(); private static final Map observationEffectiveMap = new HashMap<>(); + private static Calendar observationDate = new GregorianCalendar(); + @Before public void beforeCreateTestPatientsAndObservations() throws IOException { // Using a static flag to ensure that test data and elasticsearch index is only created once. @@ -154,15 +160,13 @@ public class BaseR4SearchLastN extends BaseJpaTest { private void createFiveObservationsForPatientCodeCategory(IIdType thePatientId, String theObservationCode, String theCategoryCode, Integer theTimeOffset) { - Calendar observationDate = new GregorianCalendar(); for (int idx=0; idx<5; idx++ ) { Observation obs = new Observation(); obs.getSubject().setReferenceElement(thePatientId); obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem); obs.setValue(new StringType(theObservationCode + "_0")); - observationDate.add(Calendar.HOUR, -theTimeOffset+idx); - Date effectiveDtm = observationDate.getTime(); + Date effectiveDtm = calculateObservationDateFromOffset(theTimeOffset, idx); obs.setEffective(new DateTimeType(effectiveDtm)); obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem); String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue(); @@ -173,6 +177,12 @@ public class BaseR4SearchLastN extends BaseJpaTest { } } + private Date calculateObservationDateFromOffset(Integer theTimeOffset, Integer theObservationIndex) { + int milliSecondsPerHour = 3600*1000; + // Generate a Date by subtracting a calculated number of hours from the static observationDate property. + return new Date(observationDate.getTimeInMillis() - (milliSecondsPerHour*(theTimeOffset+theObservationIndex))); + } + protected ServletRequestDetails mockSrd() { return mySrd; } @@ -204,7 +214,6 @@ public class BaseR4SearchLastN extends BaseJpaTest { public void testLastNNoPatients() { SearchParameterMap params = new SearchParameterMap(); - params.setLastNMax(1); params.setLastN(true); Map requestParameters = new HashMap<>(); @@ -548,6 +557,53 @@ public class BaseR4SearchLastN extends BaseJpaTest { return new TokenAndListParam().addAnd(myTokenOrListParam); } + @Test + public void testLastNSingleDate() { + + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + + DateParam myDateParam = new DateParam(ParamPrefixEnum.LESSTHAN, new Date(observationDate.getTimeInMillis() - (3600*1000*9))); + params.add(Observation.SP_DATE, myDateParam); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,15); + + } + + @Test + public void testLastNMultipleDates() { + + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + + DateParam lowDateParam = new DateParam(ParamPrefixEnum.LESSTHAN, new Date(observationDate.getTimeInMillis() - (3600*1000*(9)))); + DateParam highDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(observationDate.getTimeInMillis() - (3600*1000*(15)))); + DateAndListParam myDateAndListParam = new DateAndListParam(); + myDateAndListParam.addAnd(new DateOrListParam().addOr(lowDateParam)); + myDateAndListParam.addAnd(new DateOrListParam().addOr(highDateParam)); + + params.add(Observation.SP_DATE, myDateAndListParam); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,10); + + } + @AfterClass public static void afterClassClearContext() { TestUtil.clearAllStaticFieldsForUnitTest(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index b6ede9489c7..0e4346daabe 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -31,10 +31,13 @@ public class LastNElasticsearchSvcMultipleObservationsIT { private static ObjectMapper ourMapperNonPrettyPrint; + private static boolean indexLoaded = false; + private final Map>> createdPatientObservationMap = new HashMap<>(); private final FhirContext myFhirContext = FhirContext.forR4(); + static private final Calendar baseObservationDate = new GregorianCalendar(); @BeforeClass public static void beforeClass() { @@ -46,15 +49,10 @@ public class LastNElasticsearchSvcMultipleObservationsIT { @Before public void before() throws IOException { - createMultiplePatientsAndObservations(); - } - - @After - public void after() throws IOException { - elasticsearchSvc.deleteAllDocumentsForTest(ElasticsearchSvcImpl.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocumentsForTest(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX); - elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX); - elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX); + if (!indexLoaded) { + createMultiplePatientsAndObservations(); + indexLoaded = true; + } } @Test @@ -198,9 +196,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { SearchParameterMap searchParameterMap = new SearchParameterMap(); ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - TokenParam categoryParam = new TokenParam("test-heart-rate"); + TokenParam categoryParam = new TokenParam(null, "test-heart-rate"); searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - TokenParam codeParam = new TokenParam("test-code-1"); + TokenParam codeParam = new TokenParam(null,"test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); searchParameterMap.setLastNMax(100); @@ -231,11 +229,13 @@ public class LastNElasticsearchSvcMultipleObservationsIT { public void testLastNCodeCodeTextCategoryTextOnly() { SearchParameterMap searchParameterMap = new SearchParameterMap(); ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + + // Check case match searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - TokenParam categoryParam = new TokenParam("test-heart-rate display"); + TokenParam categoryParam = new TokenParam("Heart"); categoryParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - TokenParam codeParam = new TokenParam("test-code-1 display"); + TokenParam codeParam = new TokenParam("Code1"); codeParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); searchParameterMap.setLastNMax(100); @@ -244,31 +244,90 @@ public class LastNElasticsearchSvcMultipleObservationsIT { assertEquals(5, observations.size()); + // Check case not match + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + categoryParam = new TokenParam("heart"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + codeParam = new TokenParam("code1"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + + assertEquals(5, observations.size()); + + // Check hyphenated strings + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + categoryParam = new TokenParam("heart-rate"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + codeParam = new TokenParam("code1"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + + assertEquals(5, observations.size()); + + // Check partial strings + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + categoryParam = new TokenParam("hear"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + codeParam = new TokenParam("1-obs"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + + assertEquals(5, observations.size()); + } @Test public void testLastNNoMatchQueries() { - // Invalid Patient + + ReferenceParam validPatientParam = new ReferenceParam("Patient", "", "9"); + TokenParam validCategoryCodeParam = new TokenParam("http://mycodes.org/fhir/observation-category","test-heart-rate"); + TokenParam validObservationCodeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); + DateParam validDateParam = new DateParam(ParamPrefixEnum.EQUAL, new Date(baseObservationDate.getTimeInMillis() - (9*3600*1000))); + + // Ensure that valid parameters are indeed valid SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam patientParam = new ReferenceParam("Patient", "", "10"); - searchParameterMap.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); - searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); - searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.add(Observation.SP_PATIENT, buildReferenceAndListParam(validPatientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); searchParameterMap.setLastNMax(100); List observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(1, observations.size()); + + // Invalid Patient + searchParameterMap = new SearchParameterMap(); + ReferenceParam patientParam = new ReferenceParam("Patient", "", "10"); + searchParameterMap.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(0, observations.size()); // Invalid subject searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "10"); - searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); - searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); - searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(patientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); searchParameterMap.setLastNMax(100); observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); @@ -276,12 +335,11 @@ public class LastNElasticsearchSvcMultipleObservationsIT { // Invalid observation code searchParameterMap = new SearchParameterMap(); - subjectParam = new ReferenceParam("Patient", "", "9"); - searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); - searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-999"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(validPatientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-999"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); searchParameterMap.setLastNMax(100); observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); @@ -289,17 +347,112 @@ public class LastNElasticsearchSvcMultipleObservationsIT { // Invalid category code searchParameterMap = new SearchParameterMap(); - subjectParam = new ReferenceParam("Patient", "", "9"); - searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-not-a-category"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(validPatientParam)); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-not-a-category"); searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); - searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); searchParameterMap.setLastNMax(100); observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(0, observations.size()); + // Invalid date + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(validPatientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, new DateParam(ParamPrefixEnum.GREATERTHAN, baseObservationDate.getTime())); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(0, observations.size()); + + } + + @Test + public void testLastNEffectiveDates() { + Date highDate = new Date(baseObservationDate.getTimeInMillis() - (3600*1000)); + Date lowDate = new Date(baseObservationDate.getTimeInMillis() - (10*3600*1000)); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + DateParam dateParam = new DateParam(ParamPrefixEnum.EQUAL, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(1, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.GREATERTHAN_OR_EQUALS, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(10, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(9, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.STARTS_AFTER, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(9, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.LESSTHAN_OR_EQUALS, highDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(10, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.LESSTHAN, highDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(9, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.ENDS_BEFORE, highDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(9, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + DateParam startDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(baseObservationDate.getTimeInMillis() - (4*3600*1000))); + DateAndListParam dateAndListParam = new DateAndListParam(); + dateAndListParam.addAnd(new DateOrListParam().addOr(startDateParam)); + dateParam = new DateParam(ParamPrefixEnum.LESSTHAN_OR_EQUALS, highDate); + dateAndListParam.addAnd(new DateOrListParam().addOr(dateParam)); + searchParameterMap.add(Observation.SP_DATE, dateAndListParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(3, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + startDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(baseObservationDate.getTimeInMillis() - (4*3600*1000))); + searchParameterMap.add(Observation.SP_DATE, startDateParam); + dateParam = new DateParam(ParamPrefixEnum.LESSTHAN, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(0, observations.size()); + } private void createMultiplePatientsAndObservations() throws IOException { @@ -308,30 +461,30 @@ public class LastNElasticsearchSvcMultipleObservationsIT { CodeJson codeJson1 = new CodeJson(); codeJson1.setCodeableConceptText("Test Codeable Concept Field for First Code"); codeJson1.setCodeableConceptId(codeableConceptId1); - codeJson1.addCoding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display"); + codeJson1.addCoding("http://mycodes.org/fhir/observation-code", "test-code-1", "1-Observation Code1"); String codeableConceptId2 = UUID.randomUUID().toString(); CodeJson codeJson2 = new CodeJson(); codeJson2.setCodeableConceptText("Test Codeable Concept Field for Second Code"); codeJson2.setCodeableConceptId(codeableConceptId1); - codeJson2.addCoding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display"); + codeJson2.addCoding("http://mycodes.org/fhir/observation-code", "test-code-2", "2-Observation Code2"); // Create CodeableConcepts for two categories, each with three codings. // Create three codings and first category CodeableConcept List categoryConcepts1 = new ArrayList<>(); CodeJson categoryCodeableConcept1 = new CodeJson(); categoryCodeableConcept1.setCodeableConceptText("Test Codeable Concept Field for first category"); - categoryCodeableConcept1.addCoding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display"); - categoryCodeableConcept1.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display"); - categoryCodeableConcept1.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display"); + categoryCodeableConcept1.addCoding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "Test Heart Rate"); + categoryCodeableConcept1.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "Test Heartrate"); + categoryCodeableConcept1.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "Test Heart-Rate"); categoryConcepts1.add(categoryCodeableConcept1); // Create three codings and second category CodeableConcept List categoryConcepts2 = new ArrayList<>(); CodeJson categoryCodeableConcept2 = new CodeJson(); categoryCodeableConcept2.setCodeableConceptText("Test Codeable Concept Field for second category"); - categoryCodeableConcept2.addCoding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display"); - categoryCodeableConcept2.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display"); - categoryCodeableConcept2.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display"); + categoryCodeableConcept2.addCoding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "Test Vital Signs"); + categoryCodeableConcept2.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "Test Vital-Signs"); + categoryCodeableConcept2.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "Test Vitals"); categoryConcepts2.add(categoryCodeableConcept2); for (int patientCount = 0; patientCount < 10; patientCount++) { @@ -357,9 +510,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { assertTrue(elasticsearchSvc.createOrUpdateObservationCodeIndex(codeableConceptId2, codeJson2)); } - Calendar observationDate = new GregorianCalendar(); - observationDate.add(Calendar.HOUR, -10 + entryCount); - Date effectiveDtm = observationDate.getTime(); + Date effectiveDtm = new Date(baseObservationDate.getTimeInMillis() - ((10-entryCount)*3600*1000)); observationJson.setEffectiveDtm(effectiveDtm); assertTrue(elasticsearchSvc.createOrUpdateObservationIndex(identifier, observationJson)); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java index b0f913c3483..d6835c4dec2 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java @@ -7,6 +7,8 @@ import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.dstu2.resource.Observation; +import ca.uhn.fhir.rest.param.DateParam; +import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.ReferenceAndListParam; import ca.uhn.fhir.rest.param.ReferenceOrListParam; import ca.uhn.fhir.rest.param.ReferenceParam; @@ -108,6 +110,7 @@ public class LastNElasticsearchSvcSingleObservationIT { searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); + searchParameterMap.add(Observation.SP_DATE, new DateParam(ParamPrefixEnum.EQUAL, EFFECTIVEDTM)); searchParameterMap.setLastNMax(3); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java index 85a088e60b2..859a8dd1676 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java @@ -23,14 +23,14 @@ public class TestElasticsearchConfig { @Bean() - public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + public ElasticsearchSvcImpl myElasticsearchSvc() { int elasticsearchPort = embeddedElasticSearch().getHttpPort(); return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); } @Bean public EmbeddedElastic embeddedElasticSearch() { - EmbeddedElastic embeddedElastic = null; + EmbeddedElastic embeddedElastic; try { embeddedElastic = EmbeddedElastic.builder() .withElasticVersion(ELASTIC_VERSION) @@ -47,4 +47,10 @@ public class TestElasticsearchConfig { return embeddedElastic; } + @PreDestroy + public void stop() throws IOException { + myElasticsearchSvc().close(); + embeddedElasticSearch().stop(); + } + } diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java index 02e429f89b1..0c10ea5e326 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java @@ -58,17 +58,20 @@ public class LastNParameterHelper { private static boolean isLastNParameterDstu3(String theParamName) { return (theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_PATIENT) - || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)); + || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)) + || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_DATE); } private static boolean isLastNParameterR4(String theParamName) { return (theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_PATIENT) - || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CODE)); + || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CODE)) + || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_DATE); } private static boolean isLastNParameterR5(String theParamName) { return (theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_PATIENT) - || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)); + || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)) + || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_DATE); } public static String getSubjectParamName(FhirContext theContext) {