From 2e709a50c0d4f24d4bbbb21ca8904c572765e00f Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 2 Jun 2020 16:37:07 -0400 Subject: [PATCH 01/11] Enable filtering $lastn operation by Observation effective date. --- ...seJpaResourceProviderObservationDstu3.java | 9 +- .../BaseJpaResourceProviderObservationR4.java | 5 + .../BaseJpaResourceProviderObservationR5.java | 5 + .../search/lastn/ElasticsearchSvcImpl.java | 41 +++++ .../fhir/jpa/dao/r4/BaseR4SearchLastN.java | 57 +++++- ...lasticsearchSvcMultipleObservationsIT.java | 172 ++++++++++++++---- ...tNElasticsearchSvcSingleObservationIT.java | 3 + .../util/LastNParameterHelper.java | 9 +- 8 files changed, 260 insertions(+), 41 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java index 4a1e3189a67..bbbff175c47 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java @@ -60,6 +60,10 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider @OperationParam(name="code") TokenAndListParam theCode, + @Description(shortDefinition="The effective date of the observation") + @OperationParam(name="date") + DateAndListParam theDate, + @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -78,11 +82,12 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider SearchParameterMap paramMap = new SearchParameterMap(); paramMap.add(Observation.SP_CATEGORY, theCategory); paramMap.add(Observation.SP_CODE, theCode); + paramMap.add(Observation.SP_DATE, theDate); if (thePatient != null) { - paramMap.add("patient", thePatient); + paramMap.add(Observation.SP_PATIENT, thePatient); } if (theSubject != null) { - paramMap.add("subject", theSubject); + paramMap.add(Observation.SP_SUBJECT, theSubject); } paramMap.setLastNMax(theMax.getValue()); if (theCount != null) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java index 2ad20aa1974..1d9874f8803 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java @@ -59,6 +59,10 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< @OperationParam(name="code") TokenAndListParam theCode, + @Description(shortDefinition="The effective date of the observation") + @OperationParam(name="date") + DateAndListParam theDate, + @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -77,6 +81,7 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< SearchParameterMap paramMap = new SearchParameterMap(); paramMap.add(Observation.SP_CATEGORY, theCategory); paramMap.add(Observation.SP_CODE, theCode); + paramMap.add(Observation.SP_DATE, theDate); if (thePatient != null) { paramMap.add(Observation.SP_PATIENT, thePatient); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java index b5cacc7c571..530392b15f2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java @@ -60,6 +60,10 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< @OperationParam(name="code") TokenAndListParam theCode, + @Description(shortDefinition="The effective date of the observation") + @OperationParam(name="date") + DateAndListParam theDate, + @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -78,6 +82,7 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< SearchParameterMap paramMap = new SearchParameterMap(); paramMap.add(Observation.SP_CATEGORY, theCategory); paramMap.add(Observation.SP_CODE, theCode); + paramMap.add(Observation.SP_DATE, theDate); if (thePatient != null) { paramMap.add(Observation.SP_PATIENT, thePatient); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index c9118efd88f..ccc1bf75454 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -7,6 +7,8 @@ import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper; import ca.uhn.fhir.model.api.IQueryParameterType; +import ca.uhn.fhir.rest.param.DateParam; +import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; @@ -25,7 +27,9 @@ import org.shadehapi.elasticsearch.client.RequestOptions; import org.shadehapi.elasticsearch.client.RestHighLevelClient; import org.shadehapi.elasticsearch.common.xcontent.XContentType; import org.shadehapi.elasticsearch.index.query.BoolQueryBuilder; +import org.shadehapi.elasticsearch.index.query.MatchQueryBuilder; import org.shadehapi.elasticsearch.index.query.QueryBuilders; +import org.shadehapi.elasticsearch.index.query.RangeQueryBuilder; import org.shadehapi.elasticsearch.index.reindex.DeleteByQueryRequest; import org.shadehapi.elasticsearch.search.SearchHit; import org.shadehapi.elasticsearch.search.SearchHits; @@ -321,6 +325,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); + addDateCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); searchSourceBuilder.query(boolQueryBuilder); } searchSourceBuilder.size(0); @@ -341,6 +346,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { boolQueryBuilder.must(QueryBuilders.termQuery("subject", theSubjectParam)); addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); + addDateCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); searchSourceBuilder.query(boolQueryBuilder); searchSourceBuilder.size(0); @@ -492,6 +498,41 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } + private void addDateCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { + String dateParamName = LastNParameterHelper.getEffectiveParamName(theFhirContext); + if (theSearchParameterMap.containsKey(dateParamName)) { + List> andOrParams = theSearchParameterMap.get(dateParamName); + for (List nextAnd : andOrParams) { + BoolQueryBuilder myDateBoolQueryBuilder = new BoolQueryBuilder(); + for (IQueryParameterType nextOr : nextAnd) { + if (nextOr instanceof DateParam) { + DateParam myDate = (DateParam) nextOr; + createDateCriteria(myDate, myDateBoolQueryBuilder); + } + } + theBoolQueryBuilder.must(myDateBoolQueryBuilder); + } + } + } + + private void createDateCriteria(DateParam theDate, BoolQueryBuilder theBoolQueryBuilder) { + Long dateInstant = theDate.getValue().getTime(); + RangeQueryBuilder myRangeQueryBuilder = new RangeQueryBuilder("effectivedtm"); + + ParamPrefixEnum prefix = theDate.getPrefix(); + if (prefix == ParamPrefixEnum.GREATERTHAN || prefix == ParamPrefixEnum.STARTS_AFTER) { + theBoolQueryBuilder.should(myRangeQueryBuilder.gt(dateInstant)); + } else if (prefix == ParamPrefixEnum.LESSTHAN || prefix == ParamPrefixEnum.ENDS_BEFORE) { + theBoolQueryBuilder.should(myRangeQueryBuilder.lt(dateInstant)); + } else if (prefix == ParamPrefixEnum.LESSTHAN_OR_EQUALS) { + theBoolQueryBuilder.should(myRangeQueryBuilder.lte(dateInstant)); + } else if (prefix == ParamPrefixEnum.GREATERTHAN_OR_EQUALS) { + theBoolQueryBuilder.should(myRangeQueryBuilder.gte(dateInstant)); + } else { + theBoolQueryBuilder.should(new MatchQueryBuilder("effectivedtm", dateInstant)); + } + } + @VisibleForTesting List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { return buildAndExecuteSearch(theSearchParameterMap, theFhirContext, null, t -> t, 100); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java index bcfde6bac4f..048617cab80 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java @@ -8,6 +8,10 @@ import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; import ca.uhn.fhir.jpa.dao.BaseJpaTest; import ca.uhn.fhir.jpa.dao.SearchBuilder; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.DateAndListParam; +import ca.uhn.fhir.rest.param.DateOrListParam; +import ca.uhn.fhir.rest.param.DateParam; +import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.ReferenceAndListParam; import ca.uhn.fhir.rest.param.ReferenceOrListParam; import ca.uhn.fhir.rest.param.ReferenceParam; @@ -105,6 +109,8 @@ public class BaseR4SearchLastN extends BaseJpaTest { private static final Map observationCodeMap = new HashMap<>(); private static final Map observationEffectiveMap = new HashMap<>(); + private static Calendar observationDate = new GregorianCalendar(); + @Before public void beforeCreateTestPatientsAndObservations() { // Using a static flag to ensure that test data and elasticsearch index is only created once. @@ -141,15 +147,13 @@ public class BaseR4SearchLastN extends BaseJpaTest { private void createFiveObservationsForPatientCodeCategory(IIdType thePatientId, String theObservationCode, String theCategoryCode, Integer theTimeOffset) { - Calendar observationDate = new GregorianCalendar(); for (int idx=0; idx<5; idx++ ) { Observation obs = new Observation(); obs.getSubject().setReferenceElement(thePatientId); obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem); obs.setValue(new StringType(theObservationCode + "_0")); - observationDate.add(Calendar.HOUR, -theTimeOffset+idx); - Date effectiveDtm = observationDate.getTime(); + Date effectiveDtm = new Date(observationDate.getTimeInMillis() - (3600*1000*(theTimeOffset+idx))); obs.setEffective(new DateTimeType(effectiveDtm)); obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem); String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue(); @@ -535,6 +539,53 @@ public class BaseR4SearchLastN extends BaseJpaTest { return new TokenAndListParam().addAnd(myTokenOrListParam); } + @Test + public void testLastNSingleDate() { + + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + + DateParam myDateParam = new DateParam(ParamPrefixEnum.LESSTHAN, new Date(observationDate.getTimeInMillis() - (3600*1000*9))); + params.add(Observation.SP_DATE, myDateParam); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,15); + + } + + @Test + public void testLastNMultipleDates() { + + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + + DateParam lowDateParam = new DateParam(ParamPrefixEnum.LESSTHAN, new Date(observationDate.getTimeInMillis() - (3600*1000*(9)))); + DateParam highDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(observationDate.getTimeInMillis() - (3600*1000*(15)))); + DateAndListParam myDateAndListParam = new DateAndListParam(); + myDateAndListParam.addAnd(new DateOrListParam().addOr(lowDateParam)); + myDateAndListParam.addAnd(new DateOrListParam().addOr(highDateParam)); + + params.add(Observation.SP_DATE, myDateAndListParam); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,10); + + } + @AfterClass public static void afterClassClearContext() { TestUtil.clearAllStaticFieldsForUnitTest(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index e128026f737..547c0eb96f0 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -33,10 +33,13 @@ public class LastNElasticsearchSvcMultipleObservationsIT { private static ObjectMapper ourMapperNonPrettyPrint; + private static boolean indexLoaded = false; + private final Map>> createdPatientObservationMap = new HashMap<>(); private FhirContext myFhirContext = FhirContext.forR4(); + static private Calendar baseObservationDate = new GregorianCalendar(); @BeforeClass public static void beforeClass() { @@ -48,13 +51,10 @@ public class LastNElasticsearchSvcMultipleObservationsIT { @Before public void before() throws IOException { - createMultiplePatientsAndObservations(); - } - - @After - public void after() throws IOException { - elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX); + if (!indexLoaded) { + createMultiplePatientsAndObservations(); + indexLoaded = true; + } } @Test @@ -248,27 +248,41 @@ public class LastNElasticsearchSvcMultipleObservationsIT { @Test public void testLastNNoMatchQueries() { - // Invalid Patient + + ReferenceParam validPatientParam = new ReferenceParam("Patient", "", "9"); + TokenParam validCategoryCodeParam = new TokenParam("http://mycodes.org/fhir/observation-category","test-heart-rate"); + TokenParam validObservationCodeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); + DateParam validDateParam = new DateParam(ParamPrefixEnum.EQUAL, new Date(baseObservationDate.getTimeInMillis() - (9*3600*1000))); + + // Ensure that valid parameters are indeed valid SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam patientParam = new ReferenceParam("Patient", "", "10"); - searchParameterMap.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); - searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); - searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.add(Observation.SP_PATIENT, buildReferenceAndListParam(validPatientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); searchParameterMap.setLastNMax(100); List observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(1, observations.size()); + + // Invalid Patient + searchParameterMap = new SearchParameterMap(); + ReferenceParam patientParam = new ReferenceParam("Patient", "", "10"); + searchParameterMap.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(0, observations.size()); // Invalid subject searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "10"); - searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); - searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); - searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(patientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); searchParameterMap.setLastNMax(100); observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); @@ -276,12 +290,11 @@ public class LastNElasticsearchSvcMultipleObservationsIT { // Invalid observation code searchParameterMap = new SearchParameterMap(); - subjectParam = new ReferenceParam("Patient", "", "9"); - searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); - searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-999"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(validPatientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-999"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); searchParameterMap.setLastNMax(100); observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); @@ -289,17 +302,112 @@ public class LastNElasticsearchSvcMultipleObservationsIT { // Invalid category code searchParameterMap = new SearchParameterMap(); - subjectParam = new ReferenceParam("Patient", "", "9"); - searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-not-a-category"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(validPatientParam)); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-not-a-category"); searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); - searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, validDateParam); searchParameterMap.setLastNMax(100); observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(0, observations.size()); + // Invalid date + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(validPatientParam)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam)); + searchParameterMap.add(Observation.SP_DATE, new DateParam(ParamPrefixEnum.GREATERTHAN, baseObservationDate.getTime())); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(0, observations.size()); + + } + + @Test + public void testLastNEffectiveDates() { + Date highDate = new Date(baseObservationDate.getTimeInMillis() - (3600*1000)); + Date lowDate = new Date(baseObservationDate.getTimeInMillis() - (10*3600*1000)); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + DateParam dateParam = new DateParam(ParamPrefixEnum.EQUAL, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(1, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.GREATERTHAN_OR_EQUALS, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(10, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(9, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.STARTS_AFTER, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(9, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.LESSTHAN_OR_EQUALS, highDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(10, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.LESSTHAN, highDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(9, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + dateParam = new DateParam(ParamPrefixEnum.ENDS_BEFORE, highDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(9, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + DateParam startDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(baseObservationDate.getTimeInMillis() - (4*3600*1000))); + DateAndListParam dateAndListParam = new DateAndListParam(); + dateAndListParam.addAnd(new DateOrListParam().addOr(startDateParam)); + dateParam = new DateParam(ParamPrefixEnum.LESSTHAN_OR_EQUALS, highDate); + dateAndListParam.addAnd(new DateOrListParam().addOr(dateParam)); + searchParameterMap.add(Observation.SP_DATE, dateAndListParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(3, observations.size()); + + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + startDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(baseObservationDate.getTimeInMillis() - (4*3600*1000))); + searchParameterMap.add(Observation.SP_DATE, startDateParam); + dateParam = new DateParam(ParamPrefixEnum.LESSTHAN, lowDate); + searchParameterMap.add(Observation.SP_DATE, dateParam); + searchParameterMap.setLastNMax(100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + assertEquals(0, observations.size()); + } private void createMultiplePatientsAndObservations() throws IOException { @@ -359,9 +467,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX, codeableConceptId2, codeJson2Document, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); } - Calendar observationDate = new GregorianCalendar(); - observationDate.add(Calendar.HOUR, -10 + entryCount); - Date effectiveDtm = observationDate.getTime(); + Date effectiveDtm = new Date(baseObservationDate.getTimeInMillis() - ((10-entryCount)*3600*1000)); observationJson.setEffectiveDtm(effectiveDtm); String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationJson); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java index 1e2a6ecb8ed..072d171ad1d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java @@ -7,6 +7,8 @@ import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.dstu2.resource.Observation; +import ca.uhn.fhir.rest.param.DateParam; +import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.ReferenceAndListParam; import ca.uhn.fhir.rest.param.ReferenceOrListParam; import ca.uhn.fhir.rest.param.ReferenceParam; @@ -108,6 +110,7 @@ public class LastNElasticsearchSvcSingleObservationIT { searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); + searchParameterMap.add(Observation.SP_DATE, new DateParam(ParamPrefixEnum.EQUAL, EFFECTIVEDTM)); searchParameterMap.setLastNMax(3); diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java index abc1a3850b4..b035a528f8f 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java @@ -38,17 +38,20 @@ public class LastNParameterHelper { private static boolean isLastNParameterDstu3(String theParamName) { return (theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_PATIENT) - || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)); + || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)) + || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_DATE); } private static boolean isLastNParameterR4(String theParamName) { return (theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_PATIENT) - || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CODE)); + || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CODE)) + || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_DATE); } private static boolean isLastNParameterR5(String theParamName) { return (theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_PATIENT) - || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)); + || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)) + || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_DATE); } public static String getSubjectParamName(FhirContext theContext) { From bfc5f93ec8bec69224c0ba856cc6d7222d9f485b Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 2 Jun 2020 16:39:06 -0400 Subject: [PATCH 02/11] Enable filtering $lastn operation by Observation effective date. --- .../lastn/LastNElasticsearchSvcMultipleObservationsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index 547c0eb96f0..4a173f2dfbb 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -37,9 +37,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { private final Map>> createdPatientObservationMap = new HashMap<>(); - private FhirContext myFhirContext = FhirContext.forR4(); + private final FhirContext myFhirContext = FhirContext.forR4(); - static private Calendar baseObservationDate = new GregorianCalendar(); + static private final Calendar baseObservationDate = new GregorianCalendar(); @BeforeClass public static void beforeClass() { From 549993886db040f40074a80411b9d8150652aabd Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Thu, 4 Jun 2020 17:57:38 -0400 Subject: [PATCH 03/11] Fix merge conflicts. --- .../LastNElasticsearchSvcMultipleObservationsIT.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index 7e57fe3fcb6..e02cc26f229 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -55,14 +55,6 @@ public class LastNElasticsearchSvcMultipleObservationsIT { } } - @After - public void after() throws IOException { - elasticsearchSvc.deleteAllDocumentsForTest(ElasticsearchSvcImpl.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocumentsForTest(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX); - elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX); - elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX); - } - @Test public void testLastNAllPatientsQuery() { From 56cbc5149ba2ac45965c7d14c2ec71bc58971236 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Mon, 8 Jun 2020 09:24:09 -0400 Subject: [PATCH 04/11] Add support for partial code description matching. --- .../jpa/search/lastn/ElasticsearchSvcImpl.java | 8 ++++---- .../search/lastn/ObservationCodeIndexSchema.json | 4 ++-- .../jpa/search/lastn/ObservationIndexSchema.json | 6 +++--- ...astNElasticsearchSvcMultipleObservationsIT.java | 14 ++++++++------ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 19df63325fb..92055d226ae 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -412,8 +412,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { if (textOnlyList.size() > 0) { BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconcepttext", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchQuery("categoryconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchQuery("categoryconcepttext", textOnlyParam)); } theBoolQueryBuilder.must(myTextBoolQueryBuilder); } @@ -510,8 +510,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { if (textOnlyList.size() > 0) { BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconcepttext", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchQuery("codeconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchQuery("codeconcepttext", textOnlyParam)); } theBoolQueryBuilder.must(myTextBoolQueryBuilder); } diff --git a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json index acedddf3490..9387f7b901e 100644 --- a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json +++ b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json @@ -12,13 +12,13 @@ "type" : "keyword" }, "codingdisplay" : { - "type" : "keyword" + "type" : "text" }, "codingsystem" : { "type" : "keyword" }, "text" : { - "type" : "keyword" + "type" : "text" } } } diff --git a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json index a91869e41d5..5f76e5de590 100644 --- a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json +++ b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json @@ -18,10 +18,10 @@ "type" : "keyword" }, "codeconceptcodingdisplay" : { - "type" : "keyword" + "type" : "text" }, "categoryconcepttext" : { - "type" : "keyword" + "type" : "text" }, "categoryconceptcodingcode" : { "type" : "keyword" @@ -33,7 +33,7 @@ "type" : "keyword" }, "categoryconceptcodingdisplay" : { - "type" : "keyword" + "type" : "text" }, "effectivedtm" : { "type" : "date" diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index e02cc26f229..e117fd946c7 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -196,9 +196,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { SearchParameterMap searchParameterMap = new SearchParameterMap(); ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - TokenParam categoryParam = new TokenParam("test-heart-rate"); + TokenParam categoryParam = new TokenParam(null, "test-heart-rate"); searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - TokenParam codeParam = new TokenParam("test-code-1"); + TokenParam codeParam = new TokenParam(null,"test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); searchParameterMap.setLastNMax(100); @@ -230,10 +230,12 @@ public class LastNElasticsearchSvcMultipleObservationsIT { SearchParameterMap searchParameterMap = new SearchParameterMap(); ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - TokenParam categoryParam = new TokenParam("test-heart-rate display"); + // Check partial text + TokenParam categoryParam = new TokenParam("est-heart-ra"); categoryParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - TokenParam codeParam = new TokenParam("test-code-1 display"); + // Check partial text + TokenParam codeParam = new TokenParam("est-code-on"); codeParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); searchParameterMap.setLastNMax(100); @@ -414,13 +416,13 @@ public class LastNElasticsearchSvcMultipleObservationsIT { CodeJson codeJson1 = new CodeJson(); codeJson1.setCodeableConceptText("Test Codeable Concept Field for First Code"); codeJson1.setCodeableConceptId(codeableConceptId1); - codeJson1.addCoding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display"); + codeJson1.addCoding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-one display"); String codeableConceptId2 = UUID.randomUUID().toString(); CodeJson codeJson2 = new CodeJson(); codeJson2.setCodeableConceptText("Test Codeable Concept Field for Second Code"); codeJson2.setCodeableConceptId(codeableConceptId1); - codeJson2.addCoding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display"); + codeJson2.addCoding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-two display"); // Create CodeableConcepts for two categories, each with three codings. // Create three codings and first category CodeableConcept From 30dd858f17a24f4ce46f9b25e22d7bf714385211 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Mon, 8 Jun 2020 15:07:02 -0400 Subject: [PATCH 05/11] Add close method for Elasticsearch service. Fix the partial code match. --- .../search/lastn/ElasticsearchSvcImpl.java | 13 ++-- .../jpa/search/lastn/IElasticsearchSvc.java | 6 ++ .../TestR4ConfigWithElasticsearchClient.java | 8 +++ ...lasticsearchSvcMultipleObservationsIT.java | 69 +++++++++++++++---- .../lastn/config/TestElasticsearchConfig.java | 10 ++- 5 files changed, 88 insertions(+), 18 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 92055d226ae..7730e863ee4 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -412,8 +412,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { if (textOnlyList.size() > 0) { BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchQuery("categoryconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchQuery("categoryconcepttext", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("categoryconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("categoryconcepttext", textOnlyParam)); } theBoolQueryBuilder.must(myTextBoolQueryBuilder); } @@ -510,8 +510,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { if (textOnlyList.size() > 0) { BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchQuery("codeconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchQuery("codeconcepttext", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("codeconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("codeconcepttext", textOnlyParam)); } theBoolQueryBuilder.must(myTextBoolQueryBuilder); } @@ -685,6 +685,11 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); } + @Override + public void close() throws IOException { + myRestHighLevelClient.close(); + } + private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { IndexRequest request = new IndexRequest(theIndexName); request.id(theDocumentId); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java index b9c4a662b0b..ebcb9127dc7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -25,6 +25,7 @@ import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import java.io.IOException; import java.util.List; public interface IElasticsearchSvc { @@ -75,4 +76,9 @@ public interface IElasticsearchSvc { */ void deleteObservationDocument(String theDocumentId); + /** + * Invoked when shutting down. + */ + void close() throws IOException; + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java index 9ff6bc4c491..b2e711b0c65 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java @@ -4,6 +4,9 @@ import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import javax.annotation.PreDestroy; +import java.io.IOException; + @Configuration public class TestR4ConfigWithElasticsearchClient extends TestR4ConfigWithElasticSearch { @@ -13,4 +16,9 @@ public class TestR4ConfigWithElasticsearchClient extends TestR4ConfigWithElastic return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); } + @PreDestroy + public void stopEsClient() throws IOException { + myElasticsearchSvc().close(); + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index e117fd946c7..0e4346daabe 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -229,13 +229,13 @@ public class LastNElasticsearchSvcMultipleObservationsIT { public void testLastNCodeCodeTextCategoryTextOnly() { SearchParameterMap searchParameterMap = new SearchParameterMap(); ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + + // Check case match searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - // Check partial text - TokenParam categoryParam = new TokenParam("est-heart-ra"); + TokenParam categoryParam = new TokenParam("Heart"); categoryParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - // Check partial text - TokenParam codeParam = new TokenParam("est-code-on"); + TokenParam codeParam = new TokenParam("Code1"); codeParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); searchParameterMap.setLastNMax(100); @@ -244,6 +244,51 @@ public class LastNElasticsearchSvcMultipleObservationsIT { assertEquals(5, observations.size()); + // Check case not match + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + categoryParam = new TokenParam("heart"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + codeParam = new TokenParam("code1"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + + assertEquals(5, observations.size()); + + // Check hyphenated strings + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + categoryParam = new TokenParam("heart-rate"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + codeParam = new TokenParam("code1"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + + assertEquals(5, observations.size()); + + // Check partial strings + searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + categoryParam = new TokenParam("hear"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + codeParam = new TokenParam("1-obs"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); + + assertEquals(5, observations.size()); + } @Test @@ -416,30 +461,30 @@ public class LastNElasticsearchSvcMultipleObservationsIT { CodeJson codeJson1 = new CodeJson(); codeJson1.setCodeableConceptText("Test Codeable Concept Field for First Code"); codeJson1.setCodeableConceptId(codeableConceptId1); - codeJson1.addCoding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-one display"); + codeJson1.addCoding("http://mycodes.org/fhir/observation-code", "test-code-1", "1-Observation Code1"); String codeableConceptId2 = UUID.randomUUID().toString(); CodeJson codeJson2 = new CodeJson(); codeJson2.setCodeableConceptText("Test Codeable Concept Field for Second Code"); codeJson2.setCodeableConceptId(codeableConceptId1); - codeJson2.addCoding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-two display"); + codeJson2.addCoding("http://mycodes.org/fhir/observation-code", "test-code-2", "2-Observation Code2"); // Create CodeableConcepts for two categories, each with three codings. // Create three codings and first category CodeableConcept List categoryConcepts1 = new ArrayList<>(); CodeJson categoryCodeableConcept1 = new CodeJson(); categoryCodeableConcept1.setCodeableConceptText("Test Codeable Concept Field for first category"); - categoryCodeableConcept1.addCoding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display"); - categoryCodeableConcept1.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display"); - categoryCodeableConcept1.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display"); + categoryCodeableConcept1.addCoding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "Test Heart Rate"); + categoryCodeableConcept1.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "Test Heartrate"); + categoryCodeableConcept1.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "Test Heart-Rate"); categoryConcepts1.add(categoryCodeableConcept1); // Create three codings and second category CodeableConcept List categoryConcepts2 = new ArrayList<>(); CodeJson categoryCodeableConcept2 = new CodeJson(); categoryCodeableConcept2.setCodeableConceptText("Test Codeable Concept Field for second category"); - categoryCodeableConcept2.addCoding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display"); - categoryCodeableConcept2.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display"); - categoryCodeableConcept2.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display"); + categoryCodeableConcept2.addCoding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "Test Vital Signs"); + categoryCodeableConcept2.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "Test Vital-Signs"); + categoryCodeableConcept2.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "Test Vitals"); categoryConcepts2.add(categoryCodeableConcept2); for (int patientCount = 0; patientCount < 10; patientCount++) { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java index 85a088e60b2..859a8dd1676 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java @@ -23,14 +23,14 @@ public class TestElasticsearchConfig { @Bean() - public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + public ElasticsearchSvcImpl myElasticsearchSvc() { int elasticsearchPort = embeddedElasticSearch().getHttpPort(); return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); } @Bean public EmbeddedElastic embeddedElasticSearch() { - EmbeddedElastic embeddedElastic = null; + EmbeddedElastic embeddedElastic; try { embeddedElastic = EmbeddedElastic.builder() .withElasticVersion(ELASTIC_VERSION) @@ -47,4 +47,10 @@ public class TestElasticsearchConfig { return embeddedElastic; } + @PreDestroy + public void stop() throws IOException { + myElasticsearchSvc().close(); + embeddedElasticSearch().stop(); + } + } From c86e46b19651a2196df96fa9343c3ac8ccd15cf5 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Mon, 8 Jun 2020 18:08:02 -0400 Subject: [PATCH 06/11] Add documentation and remove requirement for max parameter (defaulted to 1 now). --- .../ca/uhn/hapi/fhir/docs/files.properties | 1 + .../ca/uhn/hapi/fhir/docs/server_jpa/lastn.md | 41 +++++++++++++++++++ .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 3 -- ...seJpaResourceProviderObservationDstu3.java | 4 +- .../BaseJpaResourceProviderObservationR4.java | 4 +- .../BaseJpaResourceProviderObservationR5.java | 4 +- .../search/lastn/ElasticsearchSvcImpl.java | 15 +++++-- .../lastn/ObservationCodeIndexSchema.json | 4 +- .../fhir/jpa/dao/r4/BaseR4SearchLastN.java | 1 - 9 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties index dc5fe3ad700..73ba427f9c4 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties @@ -47,6 +47,7 @@ page.server_jpa.search=Search page.server_jpa.performance=Performance page.server_jpa.upgrading=Upgrade Guide page.server_jpa.diff=Diff Operation +page.server_jpa.lastn=LastN Operation section.server_jpa_empi.title=JPA Server: EMPI page.server_jpa_empi.empi=Enterprise Master Patient Index diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md new file mode 100644 index 00000000000..dc8c65c1dfc --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md @@ -0,0 +1,41 @@ +# LastN Operation + +HAPI FHIR 5.1.0 introduced preliminary support for the `$lastn` operation described [here](http://hl7.org/fhir/observation-operation-lastn.html). + +This implementation of the `$lastn` operation requires an external Elasticsearch server implementation which is used to implement the indexes required by this operation. The following sections describe the current functionality supported by this operation and the configuration needed to enable this operation. + +# Functional Overview and Parameters + +As described in the [FHIR specification](http://hl7.org/fhir/observation-operation-lastn.html), the `$lastn` can be used to retrieve the most recent or last n=number of observations for one or more subjects. This implementation supports the following search parameters: + +* `subject=` or `patient=`: Identifier(s) of patient(s) to return Observation resources for. If not specified, returns most recent observations for all patients. +* `category=`: One or more category code search parameters used to filter Observations. +* `Observation.code=`: One or more `Observation.code` search parameters use to filter and group observations. If not specified, returns most recent observations for all `Observation.code` values. +* `date=`: Date search parameters used to filter Observations by `Observation.effectiveDtm`. +* `max=`: The maximum number of observations to return for each `Observation.code`. If not specified, returns only the most recent observation in each group. + +# Limitations + +Search parameters other than those listed above are currently not supported. + +The grouping of Observation resources by `Observation.code` means that the `$lastn` operation will not work in cases where `Observation.code` has more than one coding. + +# Deployment and Configuration + +The `$lastn` operation is disabled by default. The operation can be enabled by setting the DaoConfig#setLastNEnabled property (see [JavaDoc](/hapi-fhir/apidocs/hapi-fhir-jpaserver-api/ca/uhn/fhir/jpa/api/config/DaoConfig.html#setLastNEnabled(boolean))). + +In addition, the Elasticsearch client service, `ElasticsearchSvcImpl` will need to be instantiated with parameters specifying how to connect to the Elasticsearch server, for e.g.: + +```java + @Bean() + public ElasticsearchSvcImpl elasticsearchSvc() { + String elasticsearchHost = "localhost"; + String elasticsearchUserId = "elastic"; + String elasticsearchPassword = "changeme"; + int elasticsearchPort = 9301; + + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } +``` + +See the [JavaDoc](/hapi-fhir/apidocs/hapi-fhir-jpaserver-base/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.html) for more information regarding the Elasticsearch client service. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index ad463873cbe..dd65fbb430b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -311,9 +311,6 @@ public class SearchBuilder implements ISearchBuilder { throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); } } - if(myParams.getLastNMax() == null) { - throw new InvalidRequestException("Max parameter is required for $lastn operation"); - } List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults); for (String lastnResourceId : lastnResourceIds) { pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java index bbbff175c47..80d34042c92 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java @@ -89,7 +89,9 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider if (theSubject != null) { paramMap.add(Observation.SP_SUBJECT, theSubject); } - paramMap.setLastNMax(theMax.getValue()); + if (theMax != null) { + paramMap.setLastNMax(theMax.getValue()); + } if (theCount != null) { paramMap.setCount(theCount.getValue()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java index 1d9874f8803..af5eb9938a2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java @@ -88,7 +88,9 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< if (theSubject != null) { paramMap.add(Observation.SP_SUBJECT, theSubject); } - paramMap.setLastNMax(theMax.getValue()); + if(theMax != null) { + paramMap.setLastNMax(theMax.getValue()); + } if (theCount != null) { paramMap.setCount(theCount.getValue()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java index 530392b15f2..125a4a4c945 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java @@ -89,7 +89,9 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< if (theSubject != null) { paramMap.add(Observation.SP_SUBJECT, theSubject); } - paramMap.setLastNMax(theMax.getValue()); + if (theMax != null) { + paramMap.setLastNMax(theMax.getValue()); + } if (theCount != null) { paramMap.setCount(theCount.getValue()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 7730e863ee4..0a5619059f8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -97,7 +97,6 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { private final String GROUP_BY_CODE = "group_by_code"; private final String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier"; - public ElasticsearchSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername, thePassword); @@ -175,7 +174,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { break; } SearchRequest myLastNRequest = buildObservationsSearchRequest(subject, theSearchParameterMap, theFhirContext, - createObservationSubjectAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); + createObservationSubjectAggregationBuilder(getMaxParameter(theSearchParameterMap), topHitsInclude)); try { SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext, @@ -186,7 +185,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } } else { SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, theFhirContext, - createObservationCodeAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); + createObservationCodeAggregationBuilder(getMaxParameter(theSearchParameterMap), topHitsInclude)); try { SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext, @@ -198,6 +197,14 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return searchResults; } + private int getMaxParameter(SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.getLastNMax() == null) { + return 1; + } else { + return theSearchParameterMap.getLastNMax(); + } + } + private List getSubjectReferenceCriteria(String thePatientParamName, String theSubjectParamName, SearchParameterMap theSearchParameterMap) { List subjectReferenceCriteria = new ArrayList<>(); @@ -231,7 +238,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return referenceList; } - private CompositeAggregationBuilder createObservationSubjectAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { + private CompositeAggregationBuilder createObservationSubjectAggregationBuilder(Integer theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); List> compositeAggSubjectSources = new ArrayList<>(); compositeAggSubjectSources.add(subjectValuesBuilder); diff --git a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json index 9387f7b901e..acedddf3490 100644 --- a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json +++ b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json @@ -12,13 +12,13 @@ "type" : "keyword" }, "codingdisplay" : { - "type" : "text" + "type" : "keyword" }, "codingsystem" : { "type" : "keyword" }, "text" : { - "type" : "text" + "type" : "keyword" } } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java index 314b082edc6..390efd1e7ac 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java @@ -208,7 +208,6 @@ public class BaseR4SearchLastN extends BaseJpaTest { public void testLastNNoPatients() { SearchParameterMap params = new SearchParameterMap(); - params.setLastNMax(1); params.setLastN(true); Map requestParameters = new HashMap<>(); From 82b4864d793ab133040c396ca7dbf055ec2631f4 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Tue, 9 Jun 2020 09:52:40 -0400 Subject: [PATCH 07/11] Empi 57 filter out inactive person (#1903) * begin with failing test * test passes * pre-review cleanup * little fix * fix intermittent --- .../jpa/empi/svc/EmpiPersonFindingSvc.java | 5 +- .../fhir/jpa/empi/svc/EmpiResourceDaoSvc.java | 36 ++++++++++--- .../ca/uhn/fhir/jpa/empi/BaseEmpiR4Test.java | 5 ++ .../fhir/jpa/empi/dao/EmpiLinkDaoSvcTest.java | 4 +- .../EmpiMatchLinkSvcMultipleEidModeTest.java | 11 ++-- .../jpa/empi/svc/EmpiResourceDaoSvcTest.java | 52 +++++++++++++++++++ 6 files changed, 97 insertions(+), 16 deletions(-) create mode 100644 hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiResourceDaoSvcTest.java diff --git a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiPersonFindingSvc.java b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiPersonFindingSvc.java index 14882e3807a..c0b4cdd5eac 100644 --- a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiPersonFindingSvc.java +++ b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiPersonFindingSvc.java @@ -93,8 +93,9 @@ public class EmpiPersonFindingSvc { List eidFromResource = myEIDHelper.getExternalEid(theBaseResource); if (!eidFromResource.isEmpty()) { for (CanonicalEID eid : eidFromResource) { - IBaseResource foundPerson = myEmpiResourceDaoSvc.searchPersonByEid(eid.getValue()); - if (foundPerson != null) { + Optional oFoundPerson = myEmpiResourceDaoSvc.searchPersonByEid(eid.getValue()); + if (oFoundPerson.isPresent()) { + IAnyResource foundPerson = oFoundPerson.get(); Long pidOrNull = myIdHelperService.getPidOrNull(foundPerson); MatchedPersonCandidate mpc = new MatchedPersonCandidate(new ResourcePersistentId(pidOrNull), EmpiMatchResultEnum.MATCH); ourLog.debug("Matched {} by EID {}", foundPerson.getIdElement(), eid); diff --git a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiResourceDaoSvc.java b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiResourceDaoSvc.java index e0848e49e45..5e9b5084898 100644 --- a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiResourceDaoSvc.java +++ b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiResourceDaoSvc.java @@ -20,7 +20,9 @@ package ca.uhn.fhir.jpa.empi.svc; * #L% */ +import ca.uhn.fhir.empi.api.EmpiConstants; import ca.uhn.fhir.empi.api.IEmpiSettings; +import ca.uhn.fhir.empi.util.EmpiUtil; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome; @@ -28,6 +30,7 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; @@ -35,9 +38,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; @Service public class EmpiResourceDaoSvc { + private static final int MAX_MATCHING_PERSONS = 1000; @Autowired DaoRegistry myDaoRegistry; @Autowired @@ -74,16 +81,33 @@ public class EmpiResourceDaoSvc { return (IAnyResource) myPersonDao.readByPid(thePersonPid); } - public IAnyResource searchPersonByEid(String theEidFromResource) { + public Optional searchPersonByEid(String theEid) { SearchParameterMap map = new SearchParameterMap(); map.setLoadSynchronous(true); - map.add("identifier", new TokenParam(myEmpiConfig.getEmpiRules().getEnterpriseEIDSystem(), theEidFromResource)); + map.add("identifier", new TokenParam(myEmpiConfig.getEmpiRules().getEnterpriseEIDSystem(), theEid)); + map.add("active", new TokenParam("true")); IBundleProvider search = myPersonDao.search(map); - if (search.isEmpty()) { - return null; + + // Could add the meta tag to the query, but it's probably more efficient to filter on it afterwards since in practice + // it will always be present. + List list = search.getResources(0, MAX_MATCHING_PERSONS).stream() + .filter(EmpiUtil::isEmpiManaged) + .collect(Collectors.toList()); + + if (list.isEmpty()) { + return Optional.empty(); + } else if (list.size() > 1) { + throw new InternalErrorException("Found more than one active " + + EmpiConstants.CODE_HAPI_EMPI_MANAGED + + " Person with EID " + + theEid + + ": " + + list.get(0).getIdElement().getValue() + + ", " + + list.get(1).getIdElement().getValue() + ); } else { - return (IAnyResource) search.getResources(0, 1).get(0); + return Optional.of((IAnyResource) list.get(0)); } } - } diff --git a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/BaseEmpiR4Test.java b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/BaseEmpiR4Test.java index e28078a3ae5..de9a7aadd01 100644 --- a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/BaseEmpiR4Test.java +++ b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/BaseEmpiR4Test.java @@ -256,6 +256,11 @@ abstract public class BaseEmpiR4Test extends BaseJpaR4Test { return thePatient; } + protected Person addExternalEID(Person thePerson, String theEID) { + thePerson.addIdentifier().setSystem(myEmpiConfig.getEmpiRules().getEnterpriseEIDSystem()).setValue(theEID); + return thePerson; + } + protected Patient clearExternalEIDs(Patient thePatient) { thePatient.getIdentifier().removeIf(theIdentifier -> theIdentifier.getSystem().equalsIgnoreCase(myEmpiConfig.getEmpiRules().getEnterpriseEIDSystem())); return thePatient; diff --git a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/dao/EmpiLinkDaoSvcTest.java b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/dao/EmpiLinkDaoSvcTest.java index 5d9b8d4f57a..0209a4bb297 100644 --- a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/dao/EmpiLinkDaoSvcTest.java +++ b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/dao/EmpiLinkDaoSvcTest.java @@ -12,8 +12,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; public class EmpiLinkDaoSvcTest extends BaseEmpiR4Test { @Autowired @@ -27,7 +27,7 @@ public class EmpiLinkDaoSvcTest extends BaseEmpiR4Test { myEmpiLinkDaoSvc.save(empiLink); assertThat(empiLink.getCreated(), is(notNullValue())); assertThat(empiLink.getUpdated(), is(notNullValue())); - assertEquals(empiLink.getCreated(), empiLink.getUpdated()); + assertTrue(empiLink.getUpdated().getTime() - empiLink.getCreated().getTime() < 1000); } @Test diff --git a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiMatchLinkSvcMultipleEidModeTest.java b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiMatchLinkSvcMultipleEidModeTest.java index 04020c2bb57..3f8db3f40fb 100644 --- a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiMatchLinkSvcMultipleEidModeTest.java +++ b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiMatchLinkSvcMultipleEidModeTest.java @@ -1,15 +1,14 @@ package ca.uhn.fhir.jpa.empi.svc; import ca.uhn.fhir.empi.api.EmpiConstants; -import ca.uhn.fhir.empi.api.IEmpiLinkSvc; import ca.uhn.fhir.empi.model.CanonicalEID; import ca.uhn.fhir.empi.util.EIDHelper; -import ca.uhn.fhir.empi.util.PersonHelper; import ca.uhn.fhir.jpa.empi.BaseEmpiR4Test; import ca.uhn.fhir.jpa.entity.EmpiLink; import org.hl7.fhir.r4.model.Identifier; import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.Person; +import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -33,12 +32,12 @@ import static org.slf4j.LoggerFactory.getLogger; public class EmpiMatchLinkSvcMultipleEidModeTest extends BaseEmpiR4Test { private static final Logger ourLog = getLogger(EmpiMatchLinkSvcMultipleEidModeTest.class); @Autowired - IEmpiLinkSvc myEmpiLinkSvc; - @Autowired private EIDHelper myEidHelper; - @Autowired - private PersonHelper myPersonHelper; + @Before + public void before() { + super.loadEmpiSearchParameters(); + } @Test public void testIncomingPatientWithEIDThatMatchesPersonWithHapiEidAddsExternalEidsToPerson() { diff --git a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiResourceDaoSvcTest.java b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiResourceDaoSvcTest.java new file mode 100644 index 00000000000..ac0ee6943fb --- /dev/null +++ b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiResourceDaoSvcTest.java @@ -0,0 +1,52 @@ +package ca.uhn.fhir.jpa.empi.svc; + +import ca.uhn.fhir.jpa.empi.BaseEmpiR4Test; +import org.hl7.fhir.instance.model.api.IAnyResource; +import org.hl7.fhir.r4.model.Person; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Optional; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class EmpiResourceDaoSvcTest extends BaseEmpiR4Test { + private static final String TEST_EID = "TEST_EID"; + @Autowired + EmpiResourceDaoSvc myResourceDaoSvc; + + @Before + public void before() { + super.loadEmpiSearchParameters(); + } + + @Test + public void testSearchPersonByEidExcludesInactive() { + Person goodPerson = addExternalEID(createPerson(), TEST_EID); + myPersonDao.update(goodPerson); + + Person badPerson = addExternalEID(createPerson(), TEST_EID); + badPerson.setActive(false); + myPersonDao.update(badPerson); + + Optional foundPerson = myResourceDaoSvc.searchPersonByEid(TEST_EID); + assertTrue(foundPerson.isPresent()); + assertThat(foundPerson.get().getIdElement().toUnqualifiedVersionless().getValue(), is(goodPerson.getIdElement().toUnqualifiedVersionless().getValue())); + } + + @Test + public void testSearchPersonByEidExcludesNonEmpiManaged() { + Person goodPerson = addExternalEID(createPerson(), TEST_EID); + myPersonDao.update(goodPerson); + + Person badPerson = addExternalEID(createPerson(new Person(), false), TEST_EID); + myPersonDao.update(badPerson); + + Optional foundPerson = myResourceDaoSvc.searchPersonByEid(TEST_EID); + assertTrue(foundPerson.isPresent()); + assertThat(foundPerson.get().getIdElement().toUnqualifiedVersionless().getValue(), is(goodPerson.getIdElement().toUnqualifiedVersionless().getValue())); + } +} From 122bd9734425e5942a95ea6332546365f9f82021 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 9 Jun 2020 09:59:03 -0400 Subject: [PATCH 08/11] Changes recommended during code review. --- .../ca/uhn/hapi/fhir/docs/server_jpa/lastn.md | 4 ++ .../search/lastn/ElasticsearchSvcImpl.java | 71 ++++++++++++------- .../fhir/jpa/dao/r4/BaseR4SearchLastN.java | 8 ++- 3 files changed, 57 insertions(+), 26 deletions(-) diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md index dc8c65c1dfc..7f526541678 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/lastn.md @@ -16,6 +16,8 @@ As described in the [FHIR specification](http://hl7.org/fhir/observation-operati # Limitations +Currently only Elasticsearch versions up to 6.5.4 are supported. + Search parameters other than those listed above are currently not supported. The grouping of Observation resources by `Observation.code` means that the `$lastn` operation will not work in cases where `Observation.code` has more than one coding. @@ -38,4 +40,6 @@ In addition, the Elasticsearch client service, `ElasticsearchSvcImpl` will need } ``` +The Elasticsearch client service requires that security be enabled in the Elasticsearch clusters, and that an Elasticsearch user be available with permissions to create an index and to index, update and delete documents as needed. + See the [JavaDoc](/hapi-fhir/apidocs/hapi-fhir-jpaserver-base/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.html) for more information regarding the Elasticsearch client service. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 0a5619059f8..5b0c177f0c1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -81,6 +81,7 @@ import static org.apache.commons.lang3.StringUtils.isBlank; public class ElasticsearchSvcImpl implements IElasticsearchSvc { + // Index Constants public static final String OBSERVATION_INDEX = "observation_index"; public static final String OBSERVATION_CODE_INDEX = "code_index"; public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity"; @@ -88,14 +89,34 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { public static final String OBSERVATION_INDEX_SCHEMA_FILE = "ObservationIndexSchema.json"; public static final String OBSERVATION_CODE_INDEX_SCHEMA_FILE = "ObservationCodeIndexSchema.json"; - private final RestHighLevelClient myRestHighLevelClient; - - private final ObjectMapper objectMapper = new ObjectMapper(); - + // Aggregation Constants private final String GROUP_BY_SUBJECT = "group_by_subject"; private final String GROUP_BY_SYSTEM = "group_by_system"; private final String GROUP_BY_CODE = "group_by_code"; + private final String MOST_RECENT_EFFECTIVE = "most_recent_effective"; + + // Observation index document element names private final String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier"; + private final String OBSERVATION_SUBJECT_FIELD_NAME = "subject"; + private final String OBSERVATION_CODEVALUE_FIELD_NAME = "codeconceptcodingcode"; + private final String OBSERVATION_CODESYSTEM_FIELD_NAME = "codeconceptcodingsystem"; + private final String OBSERVATION_CODEHASH_FIELD_NAME = "codeconceptcodingcode_system_hash"; + private final String OBSERVATION_CODEDISPLAY_FIELD_NAME = "codeconceptcodingdisplay"; + private final String OBSERVATION_CODE_TEXT_FIELD_NAME = "codeconcepttext"; + private final String OBSERVATION_EFFECTIVEDTM_FIELD_NAME = "effectivedtm"; + private final String OBSERVATION_CATEGORYHASH_FIELD_NAME = "categoryconceptcodingcode_system_hash"; + private final String OBSERVATION_CATEGORYVALUE_FIELD_NAME = "categoryconceptcodingcode"; + private final String OBSERVATION_CATEGORYSYSTEM_FIELD_NAME = "categoryconceptcodingsystem"; + private final String OBSERVATION_CATEGORYDISPLAY_FIELD_NAME = "categoryconceptcodingdisplay"; + private final String OBSERVATION_CATEGORYTEXT_FIELD_NAME = "categoryconcepttext"; + + // Code index document element names + private final String CODE_HASH = "codingcode_system_hash"; + private final String CODE_TEXT = "text"; + + private final RestHighLevelClient myRestHighLevelClient; + + private final ObjectMapper objectMapper = new ObjectMapper(); public ElasticsearchSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername, thePassword); @@ -239,7 +260,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } private CompositeAggregationBuilder createObservationSubjectAggregationBuilder(Integer theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { - CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); + CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder(OBSERVATION_SUBJECT_FIELD_NAME).field(OBSERVATION_SUBJECT_FIELD_NAME); List> compositeAggSubjectSources = new ArrayList<>(); compositeAggSubjectSources.add(subjectValuesBuilder); CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder(GROUP_BY_SUBJECT, compositeAggSubjectSources); @@ -250,14 +271,14 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } private TermsAggregationBuilder createObservationCodeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { - TermsAggregationBuilder observationCodeCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE, ValueType.STRING).field("codeconceptcodingcode"); + TermsAggregationBuilder observationCodeCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE, ValueType.STRING).field(OBSERVATION_CODEVALUE_FIELD_NAME); observationCodeCodeAggregationBuilder.order(BucketOrder.key(true)); // Top Hits Aggregation - observationCodeCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") - .sort("effectivedtm", SortOrder.DESC) + observationCodeCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits(MOST_RECENT_EFFECTIVE) + .sort(OBSERVATION_EFFECTIVEDTM_FIELD_NAME, SortOrder.DESC) .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); observationCodeCodeAggregationBuilder.size(10000); - TermsAggregationBuilder observationCodeSystemAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_SYSTEM, ValueType.STRING).field("codeconceptcodingsystem"); + TermsAggregationBuilder observationCodeSystemAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_SYSTEM, ValueType.STRING).field(OBSERVATION_CODESYSTEM_FIELD_NAME); observationCodeSystemAggregationBuilder.order(BucketOrder.key(true)); observationCodeSystemAggregationBuilder.subAggregation(observationCodeCodeAggregationBuilder); return observationCodeSystemAggregationBuilder; @@ -339,7 +360,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { private SearchHit[] getLastNMatches(Terms.Bucket theObservationCodeBucket) { Aggregations topHitObservationCodes = theObservationCodeBucket.getAggregations(); - ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + ParsedTopHits parsedTopHits = topHitObservationCodes.get(MOST_RECENT_EFFECTIVE); return parsedTopHits.getHits().getHits(); } @@ -371,7 +392,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // Query BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery("subject", theSubjectParam)); + boolQueryBuilder.must(QueryBuilders.termQuery(OBSERVATION_SUBJECT_FIELD_NAME, theSubjectParam)); addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); addDateCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); @@ -408,19 +429,19 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); } if (codeSystemHashList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode_system_hash", codeSystemHashList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYHASH_FIELD_NAME, codeSystemHashList)); } if (codeOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode", codeOnlyList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYVALUE_FIELD_NAME, codeOnlyList)); } if (systemOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingsystem", systemOnlyList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYSYSTEM_FIELD_NAME, systemOnlyList)); } if (textOnlyList.size() > 0) { BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("categoryconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("categoryconcepttext", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CATEGORYDISPLAY_FIELD_NAME, textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CATEGORYTEXT_FIELD_NAME, textOnlyParam)); } theBoolQueryBuilder.must(myTextBoolQueryBuilder); } @@ -506,19 +527,19 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); } if (codeSystemHashList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode_system_hash", codeSystemHashList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODEHASH_FIELD_NAME, codeSystemHashList)); } if (codeOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode", codeOnlyList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODEVALUE_FIELD_NAME, codeOnlyList)); } if (systemOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingsystem", systemOnlyList)); + theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODESYSTEM_FIELD_NAME, systemOnlyList)); } if (textOnlyList.size() > 0) { BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("codeconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("codeconcepttext", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CODEDISPLAY_FIELD_NAME, textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CODE_TEXT_FIELD_NAME, textOnlyParam)); } theBoolQueryBuilder.must(myTextBoolQueryBuilder); } @@ -545,7 +566,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { private void createDateCriteria(DateParam theDate, BoolQueryBuilder theBoolQueryBuilder) { Long dateInstant = theDate.getValue().getTime(); - RangeQueryBuilder myRangeQueryBuilder = new RangeQueryBuilder("effectivedtm"); + RangeQueryBuilder myRangeQueryBuilder = new RangeQueryBuilder(OBSERVATION_EFFECTIVEDTM_FIELD_NAME); ParamPrefixEnum prefix = theDate.getPrefix(); if (prefix == ParamPrefixEnum.GREATERTHAN || prefix == ParamPrefixEnum.STARTS_AFTER) { @@ -557,7 +578,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } else if (prefix == ParamPrefixEnum.GREATERTHAN_OR_EQUALS) { theBoolQueryBuilder.should(myRangeQueryBuilder.gte(dateInstant)); } else { - theBoolQueryBuilder.should(new MatchQueryBuilder("effectivedtm", dateInstant)); + theBoolQueryBuilder.should(new MatchQueryBuilder(OBSERVATION_EFFECTIVEDTM_FIELD_NAME, dateInstant)); } } @@ -652,9 +673,9 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); if (theCodeSystemHash != null) { - boolQueryBuilder.must(QueryBuilders.termQuery("codingcode_system_hash", theCodeSystemHash)); + boolQueryBuilder.must(QueryBuilders.termQuery(CODE_HASH, theCodeSystemHash)); } else { - boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("text", theText)); + boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(CODE_TEXT, theText)); } searchSourceBuilder.query(boolQueryBuilder); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java index 390efd1e7ac..976882aaf06 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java @@ -166,7 +166,7 @@ public class BaseR4SearchLastN extends BaseJpaTest { obs.getSubject().setReferenceElement(thePatientId); obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem); obs.setValue(new StringType(theObservationCode + "_0")); - Date effectiveDtm = new Date(observationDate.getTimeInMillis() - (3600*1000*(theTimeOffset+idx))); + Date effectiveDtm = calculateObservationDateFromOffset(theTimeOffset, idx); obs.setEffective(new DateTimeType(effectiveDtm)); obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem); String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue(); @@ -177,6 +177,12 @@ public class BaseR4SearchLastN extends BaseJpaTest { } } + private Date calculateObservationDateFromOffset(Integer theTimeOffset, Integer theObservationIndex) { + int milliSecondsPerHour = 3600*1000; + // Generate a Date by subtracting a calculated number of hours from the static observationDate property. + return new Date(observationDate.getTimeInMillis() - (milliSecondsPerHour*(theTimeOffset+theObservationIndex))); + } + protected ServletRequestDetails mockSrd() { return mySrd; } From 64aa0feb5d1c4b55de71a7be97f864a8436ef13e Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 9 Jun 2020 11:06:01 -0400 Subject: [PATCH 09/11] Fix for issue of persisting blob data types to MS SQL databases. --- .../ca/uhn/fhir/jpa/api/config/DaoConfig.java | 43 +++++++++++++++++++ .../DatabaseBlobBinaryStorageSvcImpl.java | 13 +++++- .../DatabaseBlobBinaryStorageSvcImplTest.java | 16 +++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java index 1cd6b288bee..f68aaaa063b 100644 --- a/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java +++ b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java @@ -222,6 +222,11 @@ public class DaoConfig { */ private boolean myLastNEnabled = false; + /** + * @since 5.1.0 + */ + private boolean myPreloadBlobFromInputStream = false; + /** * Constructor */ @@ -2083,4 +2088,42 @@ public class DaoConfig { myMaximumDeleteConflictQueryCount = theMaximumDeleteConflictQueryCount; } + /** + *

+ * This determines whether $binary-access-write operations should first load the InputStream into memory before persisting the + * contents to the database. This needs to be enabled for MS SQL Server as this DB requires that the blob size be known + * in advance. + *

+ *

+ * Note that this setting should be enabled with caution as it can lead to significant demands on memory. + *

+ *

+ * The default value for this setting is {@code false}. + *

+ * + * @since 5.1.0 + */ + public boolean isPreloadBlobFromInputStream() { + return myPreloadBlobFromInputStream; + } + + /** + *

+ * This determines whether $binary-access-write operations should first load the InputStream into memory before persisting the + * contents to the database. This needs to be enabled for MS SQL Server as this DB requires that the blob size be known + * in advance. + *

+ *

+ * Note that this setting should be enabled with caution as it can lead to significant demands on memory. + *

+ *

+ * The default value for this setting is {@code false}. + *

+ * + * @since 5.1.0 + */ + public void setPreloadBlobFromInputStream(Boolean thePreloadBlobFromInputStream) { + myPreloadBlobFromInputStream = thePreloadBlobFromInputStream; + } + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImpl.java index ced2dd7d0a8..54a9e0e023c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImpl.java @@ -20,6 +20,7 @@ package ca.uhn.fhir.jpa.binstore; * #L% */ +import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.dao.data.IBinaryStorageEntityDao; import ca.uhn.fhir.jpa.model.entity.BinaryStorageEntity; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; @@ -55,10 +56,12 @@ public class DatabaseBlobBinaryStorageSvcImpl extends BaseBinaryStorageSvcImpl { private IBinaryStorageEntityDao myBinaryStorageEntityDao; @Autowired private PlatformTransactionManager myPlatformTransactionManager; + @Autowired + private DaoConfig myDaoConfig; @Override @Transactional(Transactional.TxType.SUPPORTS) - public StoredDetails storeBlob(IIdType theResourceId, String theBlobIdOrNull, String theContentType, InputStream theInputStream) { + public StoredDetails storeBlob(IIdType theResourceId, String theBlobIdOrNull, String theContentType, InputStream theInputStream) throws IOException { Date publishedDate = new Date(); HashingInputStream hashingInputStream = createHashingInputStream(theInputStream); @@ -74,7 +77,13 @@ public class DatabaseBlobBinaryStorageSvcImpl extends BaseBinaryStorageSvcImpl { Session session = (Session) myEntityManager.getDelegate(); LobHelper lobHelper = session.getLobHelper(); - Blob dataBlob = lobHelper.createBlob(countingInputStream, 0); + Blob dataBlob; + if (myDaoConfig.isPreloadBlobFromInputStream()) { + byte[] loadedStream = IOUtils.toByteArray(countingInputStream); + dataBlob = lobHelper.createBlob(loadedStream); + } else { + dataBlob = lobHelper.createBlob(countingInputStream, 0); + } entity.setBlob(dataBlob); // Save the entity diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImplTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImplTest.java index 41cf5139435..c5cde0d0fa4 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImplTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImplTest.java @@ -1,9 +1,11 @@ package ca.uhn.fhir.jpa.binstore; +import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test; import ca.uhn.fhir.jpa.model.entity.BinaryStorageEntity; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import org.hl7.fhir.r4.model.IdType; +import org.junit.After; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -32,6 +34,14 @@ public class DatabaseBlobBinaryStorageSvcImplTest extends BaseJpaR4Test { @Qualifier("databaseBlobBinaryStorageSvc") private IBinaryStorageSvc mySvc; + @Autowired + private DaoConfig myDaoConfig; + + @After + public void resetDaoConfig() { + myDaoConfig.setPreloadBlobFromInputStream(false); + } + @Test public void testStoreAndRetrieve() throws IOException { @@ -78,6 +88,12 @@ public class DatabaseBlobBinaryStorageSvcImplTest extends BaseJpaR4Test { assertArrayEquals(SOME_BYTES, mySvc.fetchBlob(resourceId, outcome.getBlobId())); } + @Test + public void testStoreAndRetrieveWithPreload() throws IOException { + myDaoConfig.setPreloadBlobFromInputStream(true); + testStoreAndRetrieve(); + } + @Test public void testStoreAndRetrieveWithManualId() throws IOException { From d6a5cee6ba960ba9531317811249ad9997dfe08a Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Tue, 9 Jun 2020 11:38:39 -0400 Subject: [PATCH 10/11] fix test (#1909) --- .../java/ca/uhn/fhir/jpa/empi/svc/EmpiMatchLinkSvcTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiMatchLinkSvcTest.java b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiMatchLinkSvcTest.java index f9b70ba31f6..b5e403d0625 100644 --- a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiMatchLinkSvcTest.java +++ b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiMatchLinkSvcTest.java @@ -18,6 +18,7 @@ import org.hl7.fhir.r4.model.Identifier; import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.Person; import org.hl7.fhir.r4.model.Practitioner; +import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -47,6 +48,11 @@ public class EmpiMatchLinkSvcTest extends BaseEmpiR4Test { @Autowired private PersonHelper myPersonHelper; + @Before + public void before() { + super.loadEmpiSearchParameters(); + } + @Test public void testAddPatientLinksToNewPersonIfNoneFound() { createPatientAndUpdateLinks(buildJanePatient()); From 97e6d967e9602bc03b559fb04d34517e8508a639 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 9 Jun 2020 11:52:51 -0400 Subject: [PATCH 11/11] Improve updated JUnit test. --- .../DatabaseBlobBinaryStorageSvcImplTest.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImplTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImplTest.java index c5cde0d0fa4..fbb52eb9b96 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImplTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/binstore/DatabaseBlobBinaryStorageSvcImplTest.java @@ -6,6 +6,7 @@ import ca.uhn.fhir.jpa.model.entity.BinaryStorageEntity; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import org.hl7.fhir.r4.model.IdType; import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -37,11 +38,18 @@ public class DatabaseBlobBinaryStorageSvcImplTest extends BaseJpaR4Test { @Autowired private DaoConfig myDaoConfig; - @After - public void resetDaoConfig() { - myDaoConfig.setPreloadBlobFromInputStream(false); + @Before + public void backupDaoConfig() { + defaultPreloadBlobFromInputStream = myDaoConfig.isPreloadBlobFromInputStream(); } + @After + public void restoreDaoConfig() { + myDaoConfig.setPreloadBlobFromInputStream(defaultPreloadBlobFromInputStream); + } + + boolean defaultPreloadBlobFromInputStream; + @Test public void testStoreAndRetrieve() throws IOException {