From 4e4c8dbd9a4d395c30445508ee077f0882a47191 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Wed, 25 Mar 2020 18:02:13 -0400 Subject: [PATCH 01/31] Initial commit of changes to support $lastn operation. --- hapi-deployable-pom/pom.xml | 2 + hapi-fhir-elasticsearch-6/pom.xml | 183 ++++++ .../src/main/java/ca/uhn/fhir/fhir/App.java | 13 + .../test/java/ca/uhn/fhir/fhir/AppTest.java | 20 + ...exedCodeCodeableConceptSearchParamDao.java | 9 + ...vationIndexedCodeCodingSearchParamDao.java | 26 + ...ObservationIndexedSearchParamLastNDao.java | 9 + .../ObservationLastNIndexPersistSvc.java | 80 +++ ...nIndexedCategoryCodeableConceptEntity.java | 40 ++ ...bservationIndexedCategoryCodingEntity.java | 28 + ...ationIndexedCodeCodeableConceptEntity.java | 65 +++ .../ObservationIndexedCodeCodingEntity.java | 44 ++ ...ervationIndexedSearchParamLastNEntity.java | 88 +++ .../jpa/dao/lastn/util/CodeSystemHash.java | 33 ++ .../dao/r4/FhirResourceDaoObservationR4.java | 92 +++ .../lastn/ElasticsearchBulkIndexSvcImpl.java | 188 ++++++ .../lastn/ElasticsearchRestClientFactory.java | 33 ++ .../search/lastn/ElasticsearchSvcImpl.java | 550 ++++++++++++++++++ .../jpa/search/lastn/IElasticsearchSvc.java | 12 + .../fhir/jpa/search/lastn/IndexConstants.java | 9 + ...ulticodeObservationsIntoElasticSearch.java | 122 ++++ .../OneMillionPatientsIntoElasticSearch.java | 128 ++++ ...ThousandObservationsIntoElasticSearch.java | 116 ++++ .../UploadSampleDatasetIntoElasticSearch.java | 153 +++++ .../lastn/config/ElasticsearchConfig.java | 26 + .../fhir/jpa/search/lastn/json/CodeJson.java | 82 +++ .../fhir/jpa/search/lastn/json/IdJson.java | 19 + .../fhir/jpa/search/lastn/json/IndexJson.java | 20 + .../search/lastn/json/ObservationJson.java | 175 ++++++ .../jpa/search/lastn/util/CodeSystemHash.java | 33 ++ .../search/lastn/util/SimpleStopWatch.java | 18 + ...bservationIndexedSearchParamLastNTest.java | 214 +++++++ ...bservationIndexedSearchParamLastNTest.java | 185 ++++++ ...ntegratedObservationIndexSearchConfig.java | 23 + .../TestObservationIndexSearchConfig.java | 88 +++ .../lastn/ElasticsearchPerformanceTests.java | 195 +++++++ .../ElasticsearchV5PerformanceTests.java | 153 +++++ ...sticsearchSvcMultipleObservationsTest.java | 323 ++++++++++ ...ElasticsearchSvcSingleObservationTest.java | 357 ++++++++++++ ...icsearchV5SvcMultipleObservationsTest.java | 312 ++++++++++ ...asticsearchV5SvcSingleObservationTest.java | 346 +++++++++++ .../lastn/config/TestElasticsearchConfig.java | 53 ++ .../config/TestElasticsearchV5Config.java | 53 ++ pom.xml | 3 +- 44 files changed, 4720 insertions(+), 1 deletion(-) create mode 100644 hapi-fhir-elasticsearch-6/pom.xml create mode 100644 hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java create mode 100644 hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchRestClientFactory.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java diff --git a/hapi-deployable-pom/pom.xml b/hapi-deployable-pom/pom.xml index 599b8fe0ee4..f33e54912df 100644 --- a/hapi-deployable-pom/pom.xml +++ b/hapi-deployable-pom/pom.xml @@ -155,6 +155,8 @@ javac.bat about.html changelog.xml + .*/favicon.ico$ + Log4j-charsets.properties diff --git a/hapi-fhir-elasticsearch-6/pom.xml b/hapi-fhir-elasticsearch-6/pom.xml new file mode 100644 index 00000000000..c63846bc972 --- /dev/null +++ b/hapi-fhir-elasticsearch-6/pom.xml @@ -0,0 +1,183 @@ + + + + 4.0.0 + + ca.uhn.hapi.fhir + hapi-fhir-elasticsearch-6 + 1.0-SNAPSHOT + + hapi-fhir-elasticsearch-6 + + http://www.example.com + + + UTF-8 + 1.7 + 1.7 + + + + + junit + junit + 4.12 + test + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.5.4 + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.dataformat + * + + + com.github.spullara.mustache.java + compiler + + + com.tdunning + t-digest + + + commons-codec + commons-codec + + + commons-logging + commons-logging + + + net.bytebuddy + byte-buddy + + + net.sf.jopt-simple + jopt-simple + + + org.apache.httpcomponents + * + + + org.apache.lucene + lucene-analyzers-common + + + org.apache.lucene + lucene-backward-codecs + + + org.apache.lucene + lucene-sandbox + + + org.elasticsearch + jna + + + org.hdrhistogram + HdrHistogram + + + org.yaml + snakeyaml + + + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + + maven-shade-plugin + 3.2.1 + + + package + + shade + + + true + shaded6 + + + com.carrotsearch.hppc + com.shadehapi.carrotsearch.hppc + + + org.apache.logging.log4j + org.shadehapi.apache.logging.log4j + + + org.apache.lucene + org.shadehapi.apache.lucene + + + org.elasticsearch + org.shadehapi.elasticsearch + + + org.joda + org.shadehapi.joda + + + + + + + + + diff --git a/hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java b/hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java new file mode 100644 index 00000000000..9b658e5bf48 --- /dev/null +++ b/hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java @@ -0,0 +1,13 @@ +package ca.uhn.fhir.fhir; + +/** + * Hello world! + * + */ +public class App +{ + public static void main( String[] args ) + { + System.out.println( "Hello World!" ); + } +} diff --git a/hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java b/hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java new file mode 100644 index 00000000000..83fcc8b98cb --- /dev/null +++ b/hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java @@ -0,0 +1,20 @@ +package ca.uhn.fhir.fhir; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Unit test for simple App. + */ +public class AppTest +{ + /** + * Rigorous Test :-) + */ + @Test + public void shouldAnswerWithTrue() + { + assertTrue( true ); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java new file mode 100644 index 00000000000..ee3ecde5d84 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.jpa.dao.data; + +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface IObservationIndexedCodeCodeableConceptSearchParamDao extends JpaRepository { +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java new file mode 100644 index 00000000000..d5d46f47e16 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java @@ -0,0 +1,26 @@ +package ca.uhn.fhir.jpa.dao.data; + +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodingEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +@Repository +public interface IObservationIndexedCodeCodingSearchParamDao extends JpaRepository { + + @Query("" + + "SELECT t.myCodeableConceptId FROM ObservationIndexedCodeCodingEntity t " + + "WHERE t.myCode = :code " + + "AND t.mySystem = :system " + + "") + String findForCodeAndSystem(@Param("code") String theCode, @Param("system") String theSystem); + + + @Query("" + + "SELECT t.myCodeableConceptId FROM ObservationIndexedCodeCodingEntity t " + + "WHERE t.myDisplay = :display" + + "") + String findForDisplay(@Param("display") String theDisplay); + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java new file mode 100644 index 00000000000..7a33dec68cd --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.jpa.dao.data; + +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface IObservationIndexedSearchParamLastNDao extends JpaRepository{ +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java new file mode 100644 index 00000000000..69caae6ff39 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java @@ -0,0 +1,80 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; +import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Observation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.*; + +@Component +public class ObservationLastNIndexPersistSvc { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myObservationIndexedCodeableConceptSearchParamDao; + + @Autowired + IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; + + public void indexObservation(Observation theObservation) { + ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + String resourcePID = theObservation.getId(); + indexedObservation.setIdentifier(resourcePID); + // TODO: Need to improve this as there are multiple use cases involving subject. + String subjectId = "Patient/" + theObservation.getSubject().getReference(); + indexedObservation.setSubject(subjectId); + Date effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); + indexedObservation.setEffectiveDtm(effectiveDtm); + + // Build CodeableConcept entities for Observation.Category + Set categoryConcepts = new HashSet<>(); + for(CodeableConcept categoryCodeableConcept : theObservation.getCategory()) { + // Build Coding entities for each category CodeableConcept + Set categoryCodingEntities = new HashSet<>(); + ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConceptEntity = new ObservationIndexedCategoryCodeableConceptEntity(categoryCodeableConcept.getText()); + for(Coding categoryCoding : categoryCodeableConcept.getCoding()){ + categoryCodingEntities.add(new ObservationIndexedCategoryCodingEntity(categoryCoding.getSystem(), categoryCoding.getCode(), categoryCoding.getDisplay())); + } + categoryCodeableConceptEntity.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); + categoryConcepts.add(categoryCodeableConceptEntity); + } + indexedObservation.setCategoryCodeableConcepts(categoryConcepts); + + // Build CodeableConcept entity for Observation.Code. + CodeableConcept codeCodeableConcept = theObservation.getCode(); + String observationCodeNormalizedId = null; + + // Determine if a Normalized ID was created previously for Observation Code + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + if (codeCoding.hasCode() && codeCoding.hasSystem()) { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); + } else { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); + } + } + // Generate a new a normalized ID if necessary + if (observationCodeNormalizedId == null) { + observationCodeNormalizedId = UUID.randomUUID().toString(); + } + + // Create/update normalized Observation Code index record + ObservationIndexedCodeCodeableConceptEntity codeableConceptField = new ObservationIndexedCodeCodeableConceptEntity(codeCodeableConcept.getText(), observationCodeNormalizedId); + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); + } + myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); + + indexedObservation.setObservationCode(codeableConceptField); + indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); + myResourceIndexedObservationLastNDao.save(indexedObservation); + + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java new file mode 100644 index 00000000000..7d5d98525af --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java @@ -0,0 +1,40 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import org.hibernate.search.annotations.DocumentId; +import org.hibernate.search.annotations.Field; +import org.hibernate.search.annotations.IndexedEmbedded; + +import javax.persistence.*; +import java.util.Set; + +@Embeddable +public class ObservationIndexedCategoryCodeableConceptEntity { + + @Id + @DocumentId(name = "category_concept_id") + @SequenceGenerator(name = "SEQ_CATEGORY_CONCEPT", sequenceName = "SEQ_CATEGORY_CONCEPT") + @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CATEGORY_CONCEPT") + @Column(name = "CATEGORY_CONCEPT_ID") + private Long myId; + + @Field(name = "text") + @Column(name = "CODEABLE_CONCEPT_TEXT", nullable = true) + private String myCodeableConceptText; + + @IndexedEmbedded(depth=2, prefix = "coding") + @OneToMany(mappedBy = "myCodeableConceptId", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) + private Set myObservationIndexedCategoryCodingEntitySet; + + public ObservationIndexedCategoryCodeableConceptEntity(String theCodeableConceptText) { + setCodeableConceptText(theCodeableConceptText); + } + + public void setObservationIndexedCategoryCodingEntitySet(Set theObservationIndexedCategoryCodingEntitySet) { + myObservationIndexedCategoryCodingEntitySet = theObservationIndexedCategoryCodingEntitySet; + } + + public void setCodeableConceptText(String theCodeableConceptText) { + myCodeableConceptText = theCodeableConceptText; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java new file mode 100644 index 00000000000..9bc95ed657c --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java @@ -0,0 +1,28 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import ca.uhn.fhir.jpa.dao.lastn.util.CodeSystemHash; +import org.hibernate.search.annotations.Analyze; +import org.hibernate.search.annotations.Field; + +import javax.persistence.*; + +@Embeddable +public class ObservationIndexedCategoryCodingEntity { + + @Field (name = "code", analyze = Analyze.NO) + private String myCode; + @Field (name = "system", analyze = Analyze.NO) + private String mySystem; + @Field (name = "code_system_hash", analyze = Analyze.NO) + private String myCodeSystemHash; + @Field (name = "display") + private String myDisplay; + + public ObservationIndexedCategoryCodingEntity(String theSystem, String theCode, String theDisplay) { + myCode = theCode; + mySystem = theSystem; + myCodeSystemHash = String.valueOf(CodeSystemHash.hashCodeSystem(theSystem, theCode)); + myDisplay = theDisplay; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java new file mode 100644 index 00000000000..6f9b559a476 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java @@ -0,0 +1,65 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import org.hibernate.search.annotations.DocumentId; +import org.hibernate.search.annotations.Field; +import org.hibernate.search.annotations.Indexed; +import org.hibernate.search.annotations.IndexedEmbedded; + +import javax.persistence.*; +import java.util.HashSet; +import java.util.Set; + +@Entity +@Indexed(index = "code_index") +@Embeddable +@Table(name = "HFJ_SPIDX_LASTN_CODEABLE_CONCEPT") +public class ObservationIndexedCodeCodeableConceptEntity { + + @Id + @DocumentId(name = "codeable_concept_id") + @Column(name="CODEABLE_CONCEPT_ID") + private String myCodeableConceptId; + + @Field(name = "text") + @Column(name = "CODEABLE_CONCEPT_TEXT", nullable = true) + private String myCodeableConceptText; + + @IndexedEmbedded(depth=2, prefix = "coding") + @OneToMany(mappedBy = "myCodeableConceptId", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) + private Set myObservationIndexedCodeCodingEntitySet; + + public ObservationIndexedCodeCodeableConceptEntity() { + + } + + public ObservationIndexedCodeCodeableConceptEntity(String theCodeableConceptText, String theCodeableConceptId) { + setCodeableConceptText(theCodeableConceptText); + setCodeableConceptId(theCodeableConceptId); + } + + public void addCoding(ObservationIndexedCodeCodingEntity theObservationIndexedCodeCodingEntity) { + if (myObservationIndexedCodeCodingEntitySet == null) { + myObservationIndexedCodeCodingEntitySet = new HashSet<>(); + } + myObservationIndexedCodeCodingEntitySet.add(theObservationIndexedCodeCodingEntity); + } + + public String getCodeableConceptId() { + return myCodeableConceptId; + } + + public void setCodeableConceptId(String theCodeableConceptId) { + myCodeableConceptId = theCodeableConceptId; + } + + public String getCodeableConceptText() { + return myCodeableConceptText; + } + + public void setCodeableConceptText(String theCodeableConceptText) { + myCodeableConceptText = theCodeableConceptText; + } + + + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java new file mode 100644 index 00000000000..69183faf393 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java @@ -0,0 +1,44 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import ca.uhn.fhir.jpa.dao.lastn.util.CodeSystemHash; +import org.hibernate.search.annotations.Analyze; +import org.hibernate.search.annotations.Field; + +import javax.persistence.*; + +@Entity +@Embeddable +@Table(name = "HFJ_SPIDX_LASTN_CODING") +public class ObservationIndexedCodeCodingEntity { + + @Id + @SequenceGenerator(name = "SEQ_CODING_FIELD", sequenceName = "SEQ_CODING_FIELD") + @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CODING_FIELD") + private Long myId; + + @Column(name="CODEABLE_CONCEPT_ID") + private String myCodeableConceptId; + + @Field (name = "code", analyze = Analyze.NO) + private String myCode; + + @Field (name = "system", analyze = Analyze.NO) + private String mySystem; + + @Field (name = "code_system_hash", analyze = Analyze.NO) + private String myCodeSystemHash; + + @Field (name = "display") + private String myDisplay; + + public ObservationIndexedCodeCodingEntity() {} + + public ObservationIndexedCodeCodingEntity(String theSystem, String theCode, String theDisplay, String theCodeableConceptId) { + myCode = theCode; + mySystem = theSystem; + myCodeSystemHash = String.valueOf(CodeSystemHash.hashCodeSystem(theSystem, theCode)); + myDisplay = theDisplay; + myCodeableConceptId = theCodeableConceptId; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java new file mode 100644 index 00000000000..70f8996089a --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java @@ -0,0 +1,88 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import org.hibernate.search.annotations.*; + +import javax.persistence.*; +import java.util.*; + +@Entity +@Table(name = "HFJ_LASTN_OBSERVATION") +@Indexed(index = "observation_index") +public class ObservationIndexedSearchParamLastNEntity { + + @Id + @SequenceGenerator(name = "SEQ_LASTN", sequenceName = "SEQ_LASTN") + @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_LASTN") + @Column(name = "LASTN_ID") + private Long myId; + + @Field(name = "subject", analyze = Analyze.NO) + @Column(name = "LASTN_SUBJECT_ID", nullable = false) + private String mySubject; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_OBSERVATION_CODE_FK")) + @IndexedEmbedded(depth = 2, prefix = "codeconcept") + private ObservationIndexedCodeCodeableConceptEntity myObservationCode; + + @Field(name = "codeconceptid", analyze = Analyze.NO) + @Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false) + private String myCodeNormalizedId; + + @IndexedEmbedded(depth = 2, prefix = "categoryconcept") + @Transient + private Set myCategoryCodeableConcepts; + + @Field(name = "effectivedtm", analyze = Analyze.NO) + @Temporal(TemporalType.TIMESTAMP) + @Column(name = "LASTN_EFFECTIVE_DATETIME", nullable = true) + private Date myEffectiveDtm; + + @DocumentId(name = "identifier") + @Column(name = "RESOURCE_IDENTIFIER", nullable = false) + private String myIdentifier; + + public ObservationIndexedSearchParamLastNEntity() {} + + public String getSubject() { + return mySubject; + } + + public void setSubject(String theSubject) { + mySubject = theSubject; + } + + public String getIdentifier() { + return myIdentifier; + } + + public void setIdentifier(String theIdentifier) { + myIdentifier = theIdentifier; + } + + public void setEffectiveDtm(Date theEffectiveDtm) { + myEffectiveDtm = theEffectiveDtm; + } + + public Date getEffectiveDtm() { + return myEffectiveDtm; + } + + public void setCodeNormalizedId(String theCodeNormalizedId) { + myCodeNormalizedId = theCodeNormalizedId; + } + + public String getCodeNormalizedId() { + return myCodeNormalizedId; + } + + + public void setObservationCode(ObservationIndexedCodeCodeableConceptEntity theObservationCode) { + myObservationCode = theObservationCode; + } + + public void setCategoryCodeableConcepts(Set theCategoryCodeableConcepts) { + myCategoryCodeableConcepts = theCategoryCodeableConcepts; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java new file mode 100644 index 00000000000..6cac1cd3627 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java @@ -0,0 +1,33 @@ +package ca.uhn.fhir.jpa.dao.lastn.util; + +import ca.uhn.fhir.util.UrlUtil; +import com.google.common.base.Charsets; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; + +public class CodeSystemHash { + private static final HashFunction HASH_FUNCTION = Hashing.murmur3_128(0); + private static final byte[] DELIMITER_BYTES = "|".getBytes(Charsets.UTF_8); + + static public long hashCodeSystem( String system, String code ) { + Hasher hasher = HASH_FUNCTION.newHasher(); + addStringToHasher(hasher, system); + addStringToHasher(hasher, code); + + HashCode hashCode = hasher.hash(); + return hashCode.asLong(); + } + + static private void addStringToHasher(Hasher hasher, String next) { + if (next == null) { + hasher.putByte((byte) 0); + } else { + next = UrlUtil.escapeUrlParam(next); + byte[] bytes = next.getBytes(Charsets.UTF_8); + hasher.putBytes(bytes); + } + hasher.putBytes(DELIMITER_BYTES); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java new file mode 100644 index 00000000000..91bc3679bf7 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -0,0 +1,92 @@ +package ca.uhn.fhir.jpa.dao.r4; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; +import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; +import ca.uhn.fhir.model.api.IResource; +import ca.uhn.fhir.rest.api.CacheControlDirective; +import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.param.DateRangeParam; +import ca.uhn.fhir.rest.param.StringAndListParam; +import ca.uhn.fhir.rest.param.StringParam; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.hl7.fhir.r4.model.Observation; + +import javax.servlet.http.HttpServletRequest; +import java.util.Collections; +import java.util.Date; + +public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDao { + + private IBundleProvider doLastNOperation(IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequest) { + SearchParameterMap paramMap = new SearchParameterMap(); + if (theCount != null) { + paramMap.setCount(theCount.getValue()); + } + if (theContent != null) { + paramMap.add(Constants.PARAM_CONTENT, theContent); + } + if (theNarrative != null) { + paramMap.add(Constants.PARAM_TEXT, theNarrative); + } + paramMap.setIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive())); + paramMap.setEverythingMode(theId != null ? EverythingModeEnum.PATIENT_INSTANCE : EverythingModeEnum.PATIENT_TYPE); + paramMap.setSort(theSort); + paramMap.setLastUpdated(theLastUpdated); + if (theId != null) { + paramMap.add("_id", new StringParam(theId.getIdPart())); + } + + if (!isPagingProviderDatabaseBacked(theRequest)) { + paramMap.setLoadSynchronous(true); + } + + return mySearchCoordinatorSvc.registerSearch(this, paramMap, getResourceName(), new CacheControlDirective().parse(theRequest.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequest); + } + + public IBundleProvider observationsLastN(HttpServletRequest theServletRequest, IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequestDetails) { + return doLastNOperation(theId, theCount, theLastUpdated, theSort, theContent, theNarrative, theFilter, theRequestDetails); + } + + @Override + public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, + boolean theUpdateVersion, Date theUpdateTime, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { + ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theUpdateTime, theForceUpdate, theCreateNewHistoryEntry); + + if(thePerformIndexing) { + // Update indexes here for LastN operation. + Observation observation = (Observation)theResource; + + } + + return retVal; + } + + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java new file mode 100644 index 00000000000..2444195c631 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java @@ -0,0 +1,188 @@ +package ca.uhn.fhir.jpa.search.lastn; +/* +import org.shadehapi.elasticsearch.action.DocWriteRequest; +import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.shadehapi.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.shadehapi.elasticsearch.action.bulk.BulkItemResponse; +import org.shadehapi.elasticsearch.action.bulk.BulkRequest; +import org.shadehapi.elasticsearch.action.bulk.BulkResponse; +import org.shadehapi.elasticsearch.action.index.IndexRequest; +import org.shadehapi.elasticsearch.client.RequestOptions; +import org.shadehapi.elasticsearch.client.RestHighLevelClient; +import org.shadehapi.elasticsearch.common.xcontent.XContentType; +*/ +import java.io.IOException; + +public class ElasticsearchBulkIndexSvcImpl { + +// RestHighLevelClient myRestHighLevelClient; + +// BulkRequest myBulkRequest = null; + + public ElasticsearchBulkIndexSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { + +// myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername,thePassword); + + try { + createObservationIndexIfMissing(); + createCodeIndexIfMissing(); + } catch (IOException theE) { + throw new RuntimeException("Failed to create document index", theE); + } + } + + public void createObservationIndexIfMissing() throws IOException { + if(indexExists(IndexConstants.OBSERVATION_INDEX)) { + return; + } + String observationMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeconceptid\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"norms\" : true\n" + + " },\n" + + " \"codeconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codeconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"effectivedtm\" : {\n" + + " \"type\" : \"date\"\n" + + " },\n" + + " \"identifier\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"subject\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if(!createIndex(IndexConstants.OBSERVATION_INDEX, observationMapping)) { + throw new RuntimeException("Failed to create observation index"); + } + + } + + public void createCodeIndexIfMissing() throws IOException { + if(indexExists(IndexConstants.CODE_INDEX)) { + return; + } + String codeMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeable_concept_id\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"codeable_concept_text\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if (!createIndex(IndexConstants.CODE_INDEX, codeMapping)) { + throw new RuntimeException("Failed to create code index"); + } + + } + + public boolean createIndex(String theIndexName, String theMapping) throws IOException { +/* CreateIndexRequest request = new CreateIndexRequest(theIndexName); + request.source(theMapping, XContentType.JSON); + CreateIndexResponse createIndexResponse = myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT); + return createIndexResponse.isAcknowledged(); + */ + return false; + } + + public boolean indexExists(String theIndexName) throws IOException { +/* GetIndexRequest request = new GetIndexRequest(); + request.indices(theIndexName); + return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); + + */ + return false; + } + + public void addToBulkIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { +/* IndexRequest request = new IndexRequest(theIndexName); + request.id(theDocumentId); + request.type(theDocumentType); + + request.source(theObservationDocument, XContentType.JSON); + + if (myBulkRequest == null) { + myBulkRequest = new BulkRequest(); + } + myBulkRequest.add(request); */ + } + + public void executeBulkIndex() throws IOException { +/* if (myBulkRequest == null) { + throw new RuntimeException(("No index requests have been added to the bulk request")); + } + BulkResponse bulkResponse = myRestHighLevelClient.bulk(myBulkRequest, RequestOptions.DEFAULT); + for (BulkItemResponse bulkItemResponse : bulkResponse) { + if (bulkItemResponse.getOpType() != DocWriteRequest.OpType.CREATE && bulkItemResponse.getOpType() != DocWriteRequest.OpType.INDEX) { + throw new RuntimeException("Unexpected response for bulk index request: " + bulkItemResponse.getOpType()); + } + } + myBulkRequest = null; */ + } + + public boolean bulkRequestPending() { +// return (myBulkRequest != null); + return false; + } + + public void closeClient() throws IOException { +// myRestHighLevelClient.close(); + } + + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchRestClientFactory.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchRestClientFactory.java new file mode 100644 index 00000000000..38c2c5c4f6d --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchRestClientFactory.java @@ -0,0 +1,33 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.shadehapi.elasticsearch.client.RestClient; +import org.shadehapi.elasticsearch.client.RestClientBuilder; +import org.shadehapi.elasticsearch.client.RestHighLevelClient; + +public class ElasticsearchRestClientFactory { + + static public RestHighLevelClient createElasticsearchHighLevelRestClient(String theHostname, int thePort, String theUsername, String thePassword) { + final CredentialsProvider credentialsProvider = + new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(theUsername, thePassword)); + + RestClientBuilder clientBuilder = RestClient.builder( + new HttpHost(theHostname, thePort)) + .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider)); + + Header[] defaultHeaders = new Header[]{new BasicHeader("Content-Type", "application/json")}; + clientBuilder.setDefaultHeaders(defaultHeaders); + + return new RestHighLevelClient(clientBuilder); + + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java new file mode 100644 index 00000000000..62cc30fd6a1 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -0,0 +1,550 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.api.IQueryParameterType; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +//import org.assertj.core.util.VisibleForTesting; +import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.shadehapi.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.shadehapi.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.shadehapi.elasticsearch.action.DocWriteResponse; +import org.shadehapi.elasticsearch.action.index.IndexRequest; +import org.shadehapi.elasticsearch.action.index.IndexResponse; +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.shadehapi.elasticsearch.action.support.master.AcknowledgedResponse; +import org.shadehapi.elasticsearch.client.RequestOptions; +import org.shadehapi.elasticsearch.client.RestHighLevelClient; +import org.shadehapi.elasticsearch.common.xcontent.XContentType; +import org.shadehapi.elasticsearch.index.query.BoolQueryBuilder; +import org.shadehapi.elasticsearch.index.query.QueryBuilders; +import org.shadehapi.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.shadehapi.elasticsearch.search.SearchHit; +import org.shadehapi.elasticsearch.search.SearchHits; +import org.shadehapi.elasticsearch.search.aggregations.AggregationBuilder; +import org.shadehapi.elasticsearch.search.aggregations.AggregationBuilders; +import org.shadehapi.elasticsearch.search.aggregations.Aggregations; +import org.shadehapi.elasticsearch.search.aggregations.bucket.composite.*; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.shadehapi.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits; +import org.shadehapi.elasticsearch.search.aggregations.support.ValueType; +import org.shadehapi.elasticsearch.search.builder.SearchSourceBuilder; +import org.shadehapi.elasticsearch.search.sort.SortOrder; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.commons.lang3.StringUtils.isBlank; + +//@Component +public class ElasticsearchSvcImpl implements IElasticsearchSvc { + + RestHighLevelClient myRestHighLevelClient; + + ObjectMapper objectMapper = new ObjectMapper(); + + + public ElasticsearchSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { + + myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername,thePassword); + + try { + createObservationIndexIfMissing(); + createCodeIndexIfMissing(); + } catch (IOException theE) { + throw new RuntimeException("Failed to create document index", theE); + } + } + + public void createObservationIndexIfMissing() throws IOException { + if(indexExists(IndexConstants.OBSERVATION_INDEX)) { + return; + } + String observationMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeconceptid\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"norms\" : true\n" + + " },\n" + + " \"codeconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codeconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"effectivedtm\" : {\n" + + " \"type\" : \"date\"\n" + + " },\n" + + " \"identifier\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"subject\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if(!createIndex(IndexConstants.OBSERVATION_INDEX, observationMapping)) { + throw new RuntimeException("Failed to create observation index"); + } + + } + + public void createCodeIndexIfMissing() throws IOException { + if(indexExists(IndexConstants.CODE_INDEX)) { + return; + } + String codeMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeable_concept_id\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"codeable_concept_text\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if (!createIndex(IndexConstants.CODE_INDEX, codeMapping)) { + throw new RuntimeException("Failed to create code index"); + } + + } + + public boolean createIndex(String theIndexName, String theMapping) throws IOException { + CreateIndexRequest request = new CreateIndexRequest(theIndexName); + request.source(theMapping, XContentType.JSON); + CreateIndexResponse createIndexResponse = myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT); + return createIndexResponse.isAcknowledged(); + + } + + public boolean performIndex(String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) throws IOException { + + IndexResponse indexResponse = myRestHighLevelClient.index(createIndexRequest(theIndexName, theDocumentId,theIndexDocument,theDocumentType), + RequestOptions.DEFAULT); + + return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); + } + + private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { + IndexRequest request = new IndexRequest(theIndexName); + request.id(theDocumentId); + request.type(theDocumentType); + + request.source(theObservationDocument, XContentType.JSON); + return request; + } + + public boolean indexExists(String theIndexName) throws IOException { + GetIndexRequest request = new GetIndexRequest(); + request.indices(theIndexName); + return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); + } + + SearchRequest buildObservationCodesSearchRequest(int theMaxResultSetSize) { + SearchRequest searchRequest = new SearchRequest(IndexConstants.CODE_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + // Query + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.size(theMaxResultSetSize); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + private CompositeAggregationBuilder createCompositeAggregationBuilder(int theMaximumResultSetSize, int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { + TermsAggregationBuilder observationCodeAggregationBuilder = new TermsAggregationBuilder("group_by_code", ValueType.STRING).field("codeconceptid"); + // Top Hits Aggregation + observationCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + .sort("effectivedtm", SortOrder.DESC) + .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); + observationCodeAggregationBuilder.size(theMaximumResultSetSize); + CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); + List> compositeAggSubjectSources = new ArrayList(); + compositeAggSubjectSources.add(subjectValuesBuilder); + CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder("group_by_subject", compositeAggSubjectSources); + compositeAggregationSubjectBuilder.subAggregation(observationCodeAggregationBuilder); + + return compositeAggregationSubjectBuilder; + } + + private TermsAggregationBuilder createTermsAggregationBuilder(int theMaximumResultSetSize, int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { + TermsAggregationBuilder observationCodeAggregationBuilder = new TermsAggregationBuilder("group_by_code", ValueType.STRING).field("codeconceptid"); + // Top Hits Aggregation + observationCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + .sort("effectivedtm", SortOrder.DESC).fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); + observationCodeAggregationBuilder.size(theMaximumResultSetSize); + TermsAggregationBuilder subjectsBuilder = new TermsAggregationBuilder("group_by_subject", ValueType.STRING).field("subject"); + subjectsBuilder.subAggregation(observationCodeAggregationBuilder); + subjectsBuilder.size(theMaximumResultSetSize); + return subjectsBuilder; + } + + public SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { + return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + } + + public List buildObservationCompositeResults(SearchResponse theSearchResponse) throws IOException { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedComposite aggregatedSubjects = responseAggregations.get("group_by_subject"); + List subjectBuckets = aggregatedSubjects.getBuckets(); + List codes = new ArrayList<>(); + for(ParsedComposite.ParsedBucket subjectBucket : subjectBuckets) { + Aggregations observationCodeAggregations = subjectBucket.getAggregations(); + ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); + List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); + for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { + Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + SearchHit[] topHits = parsedTopHits.getHits().getHits(); + for (SearchHit topHit : topHits) { + String sources = topHit.getSourceAsString(); + ObservationJson code = objectMapper.readValue(sources, ObservationJson.class); + codes.add(code); + } + } + } + return codes; + } + + public List buildObservationTermsResults(SearchResponse theSearchResponse) throws IOException { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedTerms aggregatedSubjects = responseAggregations.get("group_by_subject"); + List subjectBuckets = aggregatedSubjects.getBuckets(); + List codes = new ArrayList<>(); + for (Terms.Bucket subjectBucket : subjectBuckets) { + Aggregations observationCodeAggregations = subjectBucket.getAggregations(); + ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); + List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); + for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { + Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + SearchHit[] topHits = parsedTopHits.getHits().getHits(); + for (SearchHit topHit : topHits) { + String sources = topHit.getSourceAsString(); + ObservationJson code = objectMapper.readValue(sources,ObservationJson.class); + codes.add(code); + } + } + } + return codes; + } + + public List buildCodeResult(SearchResponse theSearchResponse) throws JsonProcessingException { + SearchHits codeHits = theSearchResponse.getHits(); + List codes = new ArrayList<>(); + for (SearchHit codeHit : codeHits) { + CodeJson code = objectMapper.readValue(codeHit.getSourceAsString(), CodeJson.class); + codes.add(code); + } + return codes; + } + +// @VisibleForTesting + public SearchRequest buildObservationAllFieldsCompositeSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { + + return buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, null)); + } + + public SearchRequest buildObservationCompositeSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { + // Return only identifiers + String[] topHitsInclude = {"identifier","subject","effectivedtm","codeconceptid"}; + return buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, topHitsInclude)); + } + +// @VisibleForTesting + public SearchRequest buildObservationAllFieldsTermsSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { + return buildObservationsSearchRequest(theSearchParameterMap, createTermsAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, null)); + } + + public SearchRequest buildObservationTermsSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { + // Return only identifiers + String[] topHitsInclude = {"identifier","subject","effectivedtm","codeconceptid"}; + return buildObservationsSearchRequest(theSearchParameterMap, createTermsAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, topHitsInclude)); + } + + private SearchRequest buildObservationsSearchRequest(SearchParameterMap theSearchParameterMap, AggregationBuilder theAggregationBuilder) { + SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + // Query + if(!searchParamsHaveLastNCriteria(theSearchParameterMap)) { + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + } else { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + addSubjectsCriteria(boolQueryBuilder,theSearchParameterMap); + addCategoriesCriteria(boolQueryBuilder,theSearchParameterMap); + addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap); + searchSourceBuilder.query(boolQueryBuilder); + } + searchSourceBuilder.size(0); + + // Aggregation by order codes + searchSourceBuilder.aggregation(theAggregationBuilder); + searchRequest.source(searchSourceBuilder); + + return searchRequest; + } + + private Boolean searchParamsHaveLastNCriteria(SearchParameterMap theSearchParameterMap) { + return theSearchParameterMap != null && + (theSearchParameterMap.containsKey("patient") || theSearchParameterMap.containsKey("subject") || + theSearchParameterMap.containsKey("category") || theSearchParameterMap.containsKey("code")); + } + + private void addSubjectsCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.containsKey("patient") || theSearchParameterMap.containsKey("subject")) { + ArrayList subjectReferenceCriteria = new ArrayList<>(); + List> andOrParams = new ArrayList<>(); + if (theSearchParameterMap.get("patient") != null) { + andOrParams.addAll(theSearchParameterMap.get("patient")); + } + if (theSearchParameterMap.get("subject") != null) { + andOrParams.addAll(theSearchParameterMap.get("subject")); + } + for (List nextAnd : andOrParams) { + subjectReferenceCriteria.addAll(getReferenceValues(nextAnd)); + } + if (subjectReferenceCriteria.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("subject", subjectReferenceCriteria)); + } + } + + } + + private List getReferenceValues(List referenceParams) { + ArrayList referenceList = new ArrayList<>(); + + for (IQueryParameterType nextOr : referenceParams) { + + if (nextOr instanceof ReferenceParam) { + ReferenceParam ref = (ReferenceParam) nextOr; + if (isBlank(ref.getChain())) { + if (ref.getResourceType() != null) { + referenceList.add(ref.getResourceType() + "/" + ref.getValue()); + } else { + referenceList.add(ref.getValue()); + } + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting ReferenceParam): " + nextOr.getClass()); + } + } + return referenceList; + } + + private void addCategoriesCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.containsKey("category")) { + ArrayList codeSystemHashList = new ArrayList<>(); + ArrayList codeOnlyList = new ArrayList<>(); + ArrayList systemOnlyList = new ArrayList<>(); + ArrayList textOnlyList = new ArrayList<>(); + List> andOrParams = theSearchParameterMap.get("category"); + for (List nextAnd : andOrParams) { + codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); + codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); + systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); + textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); + } + if (codeSystemHashList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode_system_hash", codeSystemHashList)); + } + if (codeOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode", codeOnlyList)); + } + if (systemOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingsystem", systemOnlyList)); + } + if (textOnlyList.size() > 0) { + BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); + for (String textOnlyParam : textOnlyList) { + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconcepttext", textOnlyParam)); + } + theBoolQueryBuilder.must(myTextBoolQueryBuilder); + } + } + + } + + private List getCodingCodeSystemValues(List codeParams) { + ArrayList codeSystemHashList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams ) { + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.getSystem() != null && ref.getValue() != null) { + codeSystemHashList.add(String.valueOf(CodeSystemHash.hashCodeSystem(ref.getSystem(), ref.getValue()))); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return codeSystemHashList; + } + + private List getCodingCodeOnlyValues(List codeParams ) { + ArrayList codeOnlyList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams) { + + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.getValue() != null && ref.getSystem() == null && !ref.isText()) { + codeOnlyList.add(ref.getValue()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return codeOnlyList; + } + + private List getCodingSystemOnlyValues(List codeParams ) { + ArrayList systemOnlyList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams) { + + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.getValue() == null && ref.getSystem() != null) { + systemOnlyList.add(ref.getSystem()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return systemOnlyList; + } + + private List getCodingTextOnlyValues(List codeParams ) { + ArrayList textOnlyList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams ) { + + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.isText() && ref.getValue() != null) { + textOnlyList.add(ref.getValue()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return textOnlyList; + } + + private void addObservationCodeCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.containsKey("code")) { + ArrayList codeSystemHashList = new ArrayList<>(); + ArrayList codeOnlyList = new ArrayList<>(); + ArrayList systemOnlyList = new ArrayList<>(); + ArrayList textOnlyList = new ArrayList<>(); + List> andOrParams = theSearchParameterMap.get("code"); + for (List nextAnd : andOrParams) { + codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); + codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); + systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); + textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); + } + if (codeSystemHashList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode_system_hash", codeSystemHashList)); + } + if (codeOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode", codeOnlyList)); + } + if (systemOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingsystem", systemOnlyList)); + } + if (textOnlyList.size() > 0) { + BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); + for (String textOnlyParam : textOnlyList) { + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconcepttext", textOnlyParam)); + } + theBoolQueryBuilder.must(myTextBoolQueryBuilder); + } + } + + } + +// @VisibleForTesting + public boolean deleteIndex(String theIndexName) throws IOException { + DeleteIndexRequest request = new DeleteIndexRequest(theIndexName); + AcknowledgedResponse deleteIndexResponse = myRestHighLevelClient.indices().delete(request, RequestOptions.DEFAULT); + + return deleteIndexResponse.isAcknowledged(); + } + + + +// @VisibleForTesting + public void deleteAllDocuments(String theIndexName) throws IOException { + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(theIndexName); + deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); + myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); + } + +/* @Override + public IBundleProvider lastN(HttpServletRequest theServletRequest, RequestDetails theRequestDetails) { + + return null; + } + + @Override + public IBundleProvider uniqueCodes(HttpServletRequest theServletRequest, RequestDetails theRequestDetails) { + return null; + } + + */ +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java new file mode 100644 index 00000000000..4c2be1cb2b5 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -0,0 +1,12 @@ +package ca.uhn.fhir.jpa.search.lastn; + +//import ca.uhn.fhir.rest.api.server.IBundleProvider; + +public interface IElasticsearchSvc { + + +// IBundleProvider lastN(javax.servlet.http.HttpServletRequest theServletRequest, ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails); + +// IBundleProvider uniqueCodes(javax.servlet.http.HttpServletRequest theServletRequest, ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails); + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java new file mode 100644 index 00000000000..94276924896 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.jpa.search.lastn; + +public class IndexConstants { + + public static final String OBSERVATION_INDEX = "observation_index"; + public static final String CODE_INDEX = "code_index"; + public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; + public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity"; +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java new file mode 100644 index 00000000000..10c0453431e --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java @@ -0,0 +1,122 @@ +package ca.uhn.fhir.jpa.search.lastn.cli; + +import ca.uhn.fhir.jpa.search.lastn.json.IdJson; +import ca.uhn.fhir.jpa.search.lastn.json.IndexJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +public class MulticodeObservationsIntoElasticSearch { + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(MulticodeObservationsIntoElasticSearch.class); + + private static final ObjectMapper ourMapperNonPrettyPrint; + + static { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + public static void main(String[] theArgs) { + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subject = "Patient/"+UUID.randomUUID().toString(); + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + String nextResourceId = UUID.randomUUID().toString(); + + IdJson id = new IdJson(nextResourceId); + IndexJson documentIndex = new IndexJson(id); + + ObservationJson observationDocument = new ObservationJson(); + observationDocument.setIdentifier(nextResourceId); + observationDocument.setSubject(subject); + // Add three CodeableConcepts for category + List category = new ArrayList<>(); + // Create three codings and first category CodeableConcept + Coding categoryCoding1_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test heart-rate"); + Coding categoryCoding1_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test alternate heart-rate"); + Coding categoryCoding1_3 = new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test second alternate heart-rate"); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept(); + categoryCodeableConcept1.getCoding().add(categoryCoding1_1); + categoryCodeableConcept1.getCoding().add(categoryCoding1_2); + categoryCodeableConcept1.getCoding().add(categoryCoding1_3); + categoryCodeableConcept1.setText("Heart Rate Codeable Concept"); + category.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + Coding categoryCoding2_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test vital signs"); + Coding categoryCoding2_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test alternate vital signs"); + Coding categoryCoding2_3 = new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test second alternate vital signs"); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept(); + categoryCodeableConcept2.getCoding().add(categoryCoding2_1); + categoryCodeableConcept2.getCoding().add(categoryCoding2_2); + categoryCodeableConcept2.getCoding().add(categoryCoding2_3); + categoryCodeableConcept2.setText("Vital Signs Codeable Concept"); + category.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + Coding categoryCoding3_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test vital signs panel"); + Coding categoryCoding3_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test alternate vital signs panel"); + Coding categoryCoding3_3 = new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test second alternate vital signs panel"); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept(); + categoryCodeableConcept3.getCoding().add(categoryCoding3_1); + categoryCodeableConcept3.getCoding().add(categoryCoding3_2); + categoryCodeableConcept3.getCoding().add(categoryCoding3_3); + categoryCodeableConcept3.setText("Vital Signs Panel Codeable Concept"); + category.add(categoryCodeableConcept3); + observationDocument.setCategories(category); + + Coding codeCoding1 = new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test observation code"); + Coding codeCoding2 = new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test observation code"); + Coding codeCoding3 = new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test observation code"); + CodeableConcept code = new CodeableConcept(); + code.getCoding().add(codeCoding1); + code.getCoding().add(codeCoding2); + code.getCoding().add(codeCoding3); + code.setText("Observation code CodeableConcept"); + observationDocument.setCode(code); + observationDocument.setCode_concept_id("multicode_test_normalized_code"); + + Date effectiveDtm = new Date(); + observationDocument.setEffectiveDtm(effectiveDtm); + + StringWriter stringWriter = new StringWriter(); + + File outputFile = new File("Observations_multiplecodes.json"); + try { + FileOutputStream outputStream = new FileOutputStream(outputFile, true); + ourMapperNonPrettyPrint.writeValue(stringWriter, documentIndex); + stringWriter.append('\n'); + ourMapperNonPrettyPrint.writeValue(stringWriter, observationDocument); + ourMapperNonPrettyPrint.writeValue(outputStream, documentIndex); + outputStream.write('\n'); + ourMapperNonPrettyPrint.writeValue(outputStream, observationDocument); + outputStream.write('\n'); + outputStream.flush(); + outputStream.close(); + } catch (IOException theE) { + theE.printStackTrace(); + } + } + + } + + ourLog.info("Upload complete"); + + } + +} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java new file mode 100644 index 00000000000..300fbba9315 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java @@ -0,0 +1,128 @@ +package ca.uhn.fhir.jpa.search.lastn.cli; + +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchBulkIndexSvcImpl; +import ca.uhn.fhir.jpa.search.lastn.IndexConstants; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.SimpleStopWatch; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.commons.lang3.RandomStringUtils; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.io.IOException; +import java.util.*; + +public class OneMillionPatientsIntoElasticSearch { +// private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(OneMillionPatientsIntoElasticSearch.class); + + private static final ObjectMapper ourMapperNonPrettyPrint; + + static { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + public static void main(String[] theArgs) throws IOException { + + ElasticsearchBulkIndexSvcImpl elasticsearchSvc = new ElasticsearchBulkIndexSvcImpl("localhost",9301, "elastic", "changeme"); + try { + + SimpleStopWatch stopwatch = new SimpleStopWatch(); + + List observationCodeIds = new ArrayList<>(); + List observationCodes = new ArrayList<>(); + + for (int codeCount = 0; codeCount < 1000; codeCount++) { + String code = RandomStringUtils.random(10,true,true); + Coding codeCoding = new Coding("http://mycodes.org/fhir/observation-code", code, "test observation code " + code); + CodeableConcept codeConcept = new CodeableConcept(); + codeConcept.getCoding().add(codeCoding); + codeConcept.setText("test observation code concept " + code); + String codeableConceptId = UUID.randomUUID().toString(); + observationCodeIds.add(codeableConceptId); + observationCodes.add(codeConcept); + CodeJson codeDocument = new CodeJson(codeConcept, codeableConceptId); + String printedObservationDocument = ourMapperNonPrettyPrint.writeValueAsString(codeDocument); + System.out.println(printedObservationDocument); + System.out.println(codeableConceptId); + elasticsearchSvc.addToBulkIndexRequest("code_index", codeableConceptId, printedObservationDocument, IndexConstants.CODE_DOCUMENT_TYPE); + if ((codeCount+1)%250 == 0) { + elasticsearchSvc.executeBulkIndex(); + long elapsedTime = stopwatch.getElapsedTime(); + stopwatch.restart(); + System.out.println("Elapsed processing time = " + elapsedTime/1000 + "s"); + System.out.println("Average processing time/code = " + elapsedTime/5000 + "ms"); + } + } + + for (int patientCount = 0; patientCount < 1000000 ; patientCount++) { + String subject = "Patient/"+UUID.randomUUID().toString(); + ArrayList observationCodesSubSet = new ArrayList<>(); + ArrayList observationCodeIdsSubSet = new ArrayList<>(); + for (int observationCount = 0; observationCount < 15 ; observationCount++) { + int codeIndex = (int) (1000 * Math.random()); + observationCodesSubSet.add(observationCodes.get(codeIndex)); + observationCodeIdsSubSet.add(observationCodeIds.get(codeIndex)); + } + int repeatedCodeIndex = (int) (1000 * Math.random()); + CodeableConcept repeatedCoding = observationCodes.get(repeatedCodeIndex); + for (int observationCount = 0; observationCount < 10 ; observationCount++ ) { + observationCodesSubSet.add(repeatedCoding); + observationCodeIdsSubSet.add(observationCodeIds.get(repeatedCodeIndex)); + } + int entryCount = 0; + for (int codingCount=0; codingCount category = new ArrayList<>(); + Coding categoryCoding = new Coding("http://mycodes.org/fhir/category-code", "test-category-code", "test category display"); + CodeableConcept categoryCodeableConcept = new CodeableConcept(); + categoryCodeableConcept.getCoding().add(categoryCoding); + categoryCodeableConcept.setText("Test Category CodeableConcept Text"); + category.add(categoryCodeableConcept); + observationDocument.setCategories(category); + observationDocument.setCode(observationCodesSubSet.get(codingCount)); + observationDocument.setCode_concept_id(observationCodeIdsSubSet.get(codingCount)); + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + entryCount++; + Date effectiveDtm = observationDate.getTime(); + observationDocument.setEffectiveDtm(effectiveDtm); + + String printedObservationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationDocument); + elasticsearchSvc.addToBulkIndexRequest("observation_index", nextResourceId, printedObservationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE ); + } + if ((patientCount+1)%100 == 0) { + System.out.println("Entries created = " + (patientCount+1)*25); + } + if ((patientCount+1)%250 == 0) { + elasticsearchSvc.executeBulkIndex(); + long elapsedTime = stopwatch.getElapsedTime(); + stopwatch.restart(); + System.out.println("Elapsed processing time = " + elapsedTime/1000 + "s"); + System.out.println("Average processing time/observation = " + elapsedTime/5000 + "ms"); + } + + + } + + if (elasticsearchSvc.bulkRequestPending()) { + elasticsearchSvc.executeBulkIndex(); + } + + // ourLog.info("Upload complete"); + } finally { + elasticsearchSvc.closeClient(); + } + + } + +} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java new file mode 100644 index 00000000000..3a6d2218a3a --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java @@ -0,0 +1,116 @@ +package ca.uhn.fhir.jpa.search.lastn.cli; + +import ca.uhn.fhir.jpa.search.lastn.json.IdJson; +import ca.uhn.fhir.jpa.search.lastn.json.IndexJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +public class OneThousandObservationsIntoElasticSearch { + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(OneThousandObservationsIntoElasticSearch.class); + + private static final ObjectMapper ourMapperNonPrettyPrint; + + static { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + public static void main(String[] theArgs) { + + for (int patientCount = 0; patientCount < 3 ; patientCount++) { + + String subject = "Patient/"+UUID.randomUUID().toString(); + + for ( int entryCount = 0; entryCount < 1100 ; entryCount++ ) { + String nextResourceId = UUID.randomUUID().toString(); + + IdJson id = new IdJson(nextResourceId); + IndexJson documentIndex = new IndexJson(id); + + ObservationJson observationDocument = new ObservationJson(); + observationDocument.setIdentifier(nextResourceId); + observationDocument.setSubject(subject); + // Add three CodeableConcepts for category + List category = new ArrayList<>(); + // Create three codings and first category CodeableConcept + Coding categoryCoding1_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test heart-rate"); + Coding categoryCoding1_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test alternate heart-rate"); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept(); + categoryCodeableConcept1.getCoding().add(categoryCoding1_1); + categoryCodeableConcept1.getCoding().add(categoryCoding1_2); + categoryCodeableConcept1.setText("Heart Rate CodeableConcept"); + category.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + Coding categoryCoding2_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test vital signs"); + Coding categoryCoding2_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test alternate vital signs"); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept(); + categoryCodeableConcept2.getCoding().add(categoryCoding2_1); + categoryCodeableConcept2.getCoding().add(categoryCoding2_2); + categoryCodeableConcept2.setText("Vital Signs CodeableConcept"); + category.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + Coding categoryCoding3_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test vital signs panel"); + Coding categoryCoding3_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test alternate vital signs panel"); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept(); + categoryCodeableConcept3.getCoding().add(categoryCoding3_1); + categoryCodeableConcept3.getCoding().add(categoryCoding3_2); + categoryCodeableConcept3.setText("Vital Signs Panel CodeableConcept"); + category.add(categoryCodeableConcept3); + observationDocument.setCategories(category); + + Coding codeCoding1 = new Coding("http://mycodes.org/fhir/observation-code", "test-code_" + entryCount, "test observation code"); + Coding codeCoding2 = new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code_" + entryCount, "test observation code"); + Coding codeCoding3 = new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code" + entryCount, "test observation code"); + CodeableConcept code = new CodeableConcept(); + code.getCoding().add(codeCoding1); + code.getCoding().add(codeCoding2); + code.getCoding().add(codeCoding3); + code.setText("Observation code CodeableConcept " + entryCount); + observationDocument.setCode(code); + observationDocument.setCode_concept_id("multicode_test_normalized_code_" + entryCount); + + Date effectiveDtm = new Date(); + observationDocument.setEffectiveDtm(effectiveDtm); + + StringWriter stringWriter = new StringWriter(); + + File outputFile = new File("one_thousand_observations.json"); + try { + FileOutputStream outputStream = new FileOutputStream(outputFile, true); + ourMapperNonPrettyPrint.writeValue(stringWriter, documentIndex); + stringWriter.append('\n'); + ourMapperNonPrettyPrint.writeValue(stringWriter, observationDocument); + ourMapperNonPrettyPrint.writeValue(outputStream, documentIndex); + outputStream.write('\n'); + ourMapperNonPrettyPrint.writeValue(outputStream, observationDocument); + outputStream.write('\n'); + outputStream.flush(); + outputStream.close(); + } catch (IOException theE) { + theE.printStackTrace(); + } + } + + } + + ourLog.info("Upload complete"); + + } + +} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java new file mode 100644 index 00000000000..bfe2e82bd8f --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java @@ -0,0 +1,153 @@ +package ca.uhn.fhir.jpa.search.lastn.cli; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.parser.IParser; +import ca.uhn.fhir.parser.LenientErrorHandler; +import ca.uhn.fhir.jpa.search.lastn.json.IdJson; +import ca.uhn.fhir.jpa.search.lastn.json.IndexJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.base.Charsets; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.io.IOUtils; +import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; + +import java.io.*; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +public class UploadSampleDatasetIntoElasticSearch { + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(UploadSampleDatasetIntoElasticSearch.class); + + private static final ObjectMapper ourMapperNonPrettyPrint; + + static { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + public static void main(String[] theArgs) { + + FhirContext myFhirCtx = FhirContext.forR4(); + myFhirCtx.getRestfulClientFactory().setSocketTimeout(120000); + + PathMatchingResourcePatternResolver provider = new PathMatchingResourcePatternResolver(); + final Resource[] bundleResources; + try { + bundleResources = provider.getResources("*.json.bz2"); + } catch (IOException e) { + throw new RuntimeException("Unexpected error during transmission: " + e.toString(), e); + } + + AtomicInteger index = new AtomicInteger(); + + Arrays.stream(bundleResources).forEach( + resource -> { + index.incrementAndGet(); + + InputStream resIs = null; + String nextBundleString; + try { + resIs = resource.getInputStream(); + resIs = new BZip2CompressorInputStream(resIs); + nextBundleString = IOUtils.toString(resIs, Charsets.UTF_8); + } catch (IOException e) { + ourLog.error("Failure reading: {}", resource.getFilename(), e); + return; + } finally { + try { + if (resIs != null) { + resIs.close(); + } + } catch (final IOException ioe) { + // ignore + } + } + + ourLog.info("Uploading {}/{} - {} ({} bytes)", index, bundleResources.length, resource.getFilename(), nextBundleString.length()); + + /* + * SMART demo apps rely on the use of LOINC 3141-9 (Body Weight Measured) + * instead of LOINC 29463-7 (Body Weight) + */ + nextBundleString = nextBundleString.replace("\"29463-7\"", "\"3141-9\""); + + IParser parser = myFhirCtx.newJsonParser(); + parser.setParserErrorHandler(new LenientErrorHandler(false)); + Bundle bundle = parser.parseResource(Bundle.class, nextBundleString); + + for (BundleEntryComponent nextEntry : bundle.getEntry()) { + + /* + * Synthea gives resources UUIDs with urn:uuid: prefix, which is only + * used for placeholders. We're going to use these as the actual resource + * IDs, so we strip the prefix. + */ + String nextResourceId = nextEntry.getFullUrl(); + if (nextResourceId == null) { + nextResourceId = UUID.randomUUID().toString(); + } + + nextResourceId = nextResourceId.replace("urn:uuid:", ""); + nextEntry.getResource().setId(nextResourceId); + nextEntry.setFullUrl(nextResourceId); + + if (nextEntry.getResource().getResourceType().equals(ResourceType.Observation)) { + + IdJson id = new IdJson(nextResourceId); + IndexJson documentIndex = new IndexJson(id); + + org.hl7.fhir.r4.model.Observation observation = (Observation) nextEntry.getResource(); + ObservationJson observationDocument = new ObservationJson(); + observationDocument.setIdentifier(nextResourceId); + String subject = "Patient/"+observation.getSubject().getReference(); + observationDocument.setSubject(subject); + List category = observation.getCategory(); + observationDocument.setCategories(category); + observationDocument.setCode_concept_id(category.get(0).getCodingFirstRep().getSystem() + "/" + category.get(0).getCodingFirstRep().getCode()); + CodeableConcept code = observation.getCode(); + observationDocument.setCode(code); + Date effectiveDtm = observation.getEffectiveDateTimeType().getValue(); + observationDocument.setEffectiveDtm(effectiveDtm); + + StringWriter stringWriter = new StringWriter(); + File outputFile = new File("Observations.json"); + try { + FileOutputStream outputStream = new FileOutputStream(outputFile, true); + ourMapperNonPrettyPrint.writeValue(stringWriter, documentIndex); + stringWriter.append('\n'); + ourMapperNonPrettyPrint.writeValue(stringWriter, observationDocument); + ourMapperNonPrettyPrint.writeValue(outputStream, documentIndex); + outputStream.write('\n'); + ourMapperNonPrettyPrint.writeValue(outputStream, observationDocument); + outputStream.write('\n'); + outputStream.flush(); + outputStream.close(); + } catch (IOException theE) { + theE.printStackTrace(); + } + System.out.println(stringWriter.toString()); + + } + + } + + } + ); + + ourLog.info("Upload complete"); + + } + +} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java new file mode 100644 index 00000000000..3ede8b1d67f --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java @@ -0,0 +1,26 @@ +package ca.uhn.fhir.jpa.search.lastn.config; + +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import java.io.IOException; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") +@EnableTransactionManagement +public class ElasticsearchConfig { + + private final String elasticsearchHost = "127.0.0.1"; + private final Integer elasticsearchPort = 9301; + private final String elasticsearchUserId = "elastic"; + private final String elasticsearchPassword = "changeme"; + + @Bean() + public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java new file mode 100644 index 00000000000..032279866e2 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java @@ -0,0 +1,82 @@ +package ca.uhn.fhir.jpa.search.lastn.json; + +/* + * #%L + * Smile CDR - CDR + * %% + * Copyright (C) 2016 - 2019 Simpatico Intelligent Systems Inc + * %% + * All rights reserved. + * #L% + */ + +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.util.ArrayList; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class CodeJson { + + @JsonProperty(value = "codeable_concept_id", required = false) + private String myCodeableConceptId; + + @JsonProperty(value = "codeable_concept_text", required = false) + private String myCodeableConceptText; + + @JsonProperty(value = "codingcode", required = false) + private List myCoding_code = new ArrayList<>(); + + @JsonProperty(value = "codingcode_system_hash", required = true) + private List myCoding_code_system_hash = new ArrayList<>(); + + @JsonProperty(value = "codingdisplay", required = false) + private List myCoding_display = new ArrayList<>(); + + @JsonProperty(value = "codingsystem", required = false) + private List myCoding_system = new ArrayList<>(); + + public CodeJson(){ + } + + public CodeJson(CodeableConcept theCodeableConcept, String theCodeableConceptId) { + myCodeableConceptText = theCodeableConcept.getText(); + myCodeableConceptId = theCodeableConceptId; + for (Coding theCoding : theCodeableConcept.getCoding()) { + myCoding_code.add(theCoding.getCode()); + myCoding_system.add(theCoding.getSystem()); + myCoding_display.add(theCoding.getDisplay()); + myCoding_code_system_hash.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCoding.getSystem(), theCoding.getCode()))); + } + } + + public String getCodeableConceptId() { + return myCodeableConceptId; + } + + public String getCodeableConceptText() { + return myCodeableConceptText; + } + + public List getCoding_code() { + return myCoding_code; + } + + public List getCoding_code_system_hash() { + return myCoding_code_system_hash; + } + + public List getCoding_display() { + return myCoding_display; + } + + public List getCoding_system() { + return myCoding_system; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java new file mode 100644 index 00000000000..1df4410f054 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java @@ -0,0 +1,19 @@ +package ca.uhn.fhir.jpa.search.lastn.json; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class IdJson { + + @JsonProperty(value = "_id", required = true) + private String myId; + + public IdJson(String theId) { + myId = theId; + } + + public String getId() { return myId; } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java new file mode 100644 index 00000000000..dc1880124b4 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java @@ -0,0 +1,20 @@ +package ca.uhn.fhir.jpa.search.lastn.json; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class IndexJson { + + @JsonProperty(value = "index", required = true) + private IdJson myIndex; + + public IndexJson(IdJson theIndex) { + myIndex = theIndex; + } + + public IdJson getId() { return myIndex; } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java new file mode 100644 index 00000000000..959dba5c5e7 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java @@ -0,0 +1,175 @@ +package ca.uhn.fhir.jpa.search.lastn.json; + +/* + * #%L + * Smile CDR - CDR + * %% + * Copyright (C) 2016 - 2019 Simpatico Intelligent Systems Inc + * %% + * All rights reserved. + * #L% + */ + +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class ObservationJson { + + @JsonProperty(value = "identifier", required = true) + private String myIdentifier; + + @JsonProperty(value = "subject", required = true) + private String mySubject; + + @JsonProperty(value = "categoryconcepttext", required = false) + private List myCategory_concept_text = new ArrayList<>(); + + @JsonProperty(value = "categoryconceptcodingcode", required = false) + private List> myCategory_coding_code = new ArrayList<>(); + + @JsonProperty(value = "categoryconceptcodingcode_system_hash", required = false) + private List> myCategory_coding_code_system_hash = new ArrayList<>(); + + @JsonProperty(value = "categoryconceptcodingdisplay", required = false) + private List> myCategory_coding_display = new ArrayList<>(); + + @JsonProperty(value = "categoryconceptcodingsystem", required = false) + private List> myCategory_coding_system = new ArrayList<>(); + + @JsonProperty(value = "codeconceptid", required = false) + private String myCode_concept_id; + + @JsonProperty(value = "codeconcepttext", required = false) + private String myCode_concept_text; + + @JsonProperty(value = "codeconceptcodingcode", required = false) + private List myCode_coding_code = new ArrayList<>(); + + @JsonProperty(value = "codeconceptcodingcode_system_hash", required = false) + private List myCode_coding_code_system_hash = new ArrayList<>(); + + @JsonProperty(value = "codeconceptcodingdisplay", required = false) + private List myCode_coding_display = new ArrayList<>(); + + @JsonProperty(value = "codeconceptcodingsystem", required = false) + private List myCode_coding_system = new ArrayList<>(); + + @JsonProperty(value = "effectivedtm", required = true) + private Date myEffectiveDtm; + + public ObservationJson() {} + + public void setIdentifier(String theIdentifier) { + myIdentifier = theIdentifier; + } + + public void setSubject(String theSubject) { + mySubject = theSubject; + } + + public void setCategories(List theCategories) { + for (CodeableConcept theConcept : theCategories) { + myCategory_concept_text.add(theConcept.getText()); + List coding_code_system_hashes = new ArrayList<>(); + List coding_codes = new ArrayList<>(); + List coding_displays = new ArrayList<>(); + List coding_systems = new ArrayList<>(); + for (Coding theCategoryCoding : theConcept.getCoding()) { + coding_code_system_hashes.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCategoryCoding.getSystem(), theCategoryCoding.getCode()))); + coding_codes.add(theCategoryCoding.getCode()); + coding_displays.add(theCategoryCoding.getDisplay()); + coding_systems.add(theCategoryCoding.getSystem()); + } + myCategory_coding_code_system_hash.add(coding_code_system_hashes); + myCategory_coding_code.add(coding_codes); + myCategory_coding_display.add(coding_displays); + myCategory_coding_system.add(coding_systems); + } + } + + public List getCategory_concept_text() { + return myCategory_concept_text; + } + + public List> getCategory_coding_code_system_hash() { + return myCategory_coding_code_system_hash; + } + + public List> getCategory_coding_code() { + return myCategory_coding_code; + } + + public List> getCategory_coding_display() { + return myCategory_coding_display; + } + + public List> getCategory_coding_system() { + return myCategory_coding_system; + } + + public void setCode(CodeableConcept theCode) { + myCode_concept_text = theCode.getText(); + for(Coding theCodeCoding : theCode.getCoding()) { + myCode_coding_code_system_hash.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCodeCoding.getSystem(), theCodeCoding.getCode()))); + myCode_coding_code.add(theCodeCoding.getCode()); + myCode_coding_display.add(theCodeCoding.getDisplay()); + myCode_coding_system.add(theCodeCoding.getSystem()); + } + + } + + public String getCode_concept_text() { + return myCode_concept_text; + } + + public List getCode_coding_code_system_hash() { + return myCode_coding_code_system_hash; + } + + public List getCode_coding_code() { + return myCode_coding_code; + } + + public List getCode_coding_display() { + return myCode_coding_display; + } + + public List getCode_coding_system() { + return myCode_coding_system; + } + + public void setCode_concept_id(String theCodeId) { + myCode_concept_id = theCodeId; + } + + public String getCode_concept_id() { + return myCode_concept_id; + } + + public void setEffectiveDtm(Date theEffectiveDtm) { + myEffectiveDtm = theEffectiveDtm; + } + + public Date getEffectiveDtm() { + return myEffectiveDtm; + } + + public String getSubject() { + return mySubject; + } + + public String getIdentifier() { + return myIdentifier; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java new file mode 100644 index 00000000000..9995b1d3ec0 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java @@ -0,0 +1,33 @@ +package ca.uhn.fhir.jpa.search.lastn.util; + +import ca.uhn.fhir.util.UrlUtil; +import com.google.common.base.Charsets; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; + +public class CodeSystemHash { + private static final HashFunction HASH_FUNCTION = Hashing.murmur3_128(0); + private static final byte[] DELIMITER_BYTES = "|".getBytes(Charsets.UTF_8); + + static public long hashCodeSystem( String system, String code ) { + Hasher hasher = HASH_FUNCTION.newHasher(); + addStringToHasher(hasher, system); + addStringToHasher(hasher, code); + + HashCode hashCode = hasher.hash(); + return hashCode.asLong(); + } + + static private void addStringToHasher(Hasher hasher, String next) { + if (next == null) { + hasher.putByte((byte) 0); + } else { + next = UrlUtil.escapeUrlParam(next); + byte[] bytes = next.getBytes(Charsets.UTF_8); + hasher.putBytes(bytes); + } + hasher.putBytes(DELIMITER_BYTES); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java new file mode 100644 index 00000000000..20381a5e2fe --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java @@ -0,0 +1,18 @@ +package ca.uhn.fhir.jpa.search.lastn.util; + +public class SimpleStopWatch { + private long myStarted = System.currentTimeMillis(); + + public SimpleStopWatch() { + + } + + public long getElapsedTime() { + return System.currentTimeMillis() - myStarted; + } + + public void restart() { + myStarted = System.currentTimeMillis(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java new file mode 100644 index 00000000000..32a1324bea5 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java @@ -0,0 +1,214 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.jpa.dao.lastn.config.TestIntegratedObservationIndexSearchConfig; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.hl7.fhir.r4.model.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestIntegratedObservationIndexSearchConfig.class }) +public class IntegratedObservationIndexedSearchParamLastNTest { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; + + @Autowired + ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; + + @Autowired + private ElasticsearchSvcImpl elasticsearchSvc; + + final String RESOURCEPID = "123"; + final String SUBJECTID = "4567"; + final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; + final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; + + final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; + final String CODEFIRSTCODINGCODE = "test-code"; + + @Before + public void before() throws IOException { + + myResourceIndexedObservationLastNDao.deleteAll(); + myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); + + } + + @After + public void after() { + + myResourceIndexedObservationLastNDao.deleteAll(); + myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); + + } + + @Test + public void testIndexObservationSingle() throws IOException { + + Observation myObservation = new Observation(); + String resourcePID = "123"; + myObservation.setId(resourcePID); + Reference subjectId = new Reference("4567"); + myObservation.setSubject(subjectId); + Date effectiveDtm = new Date(); + myObservation.setEffective(new DateTimeType(effectiveDtm)); + + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for for second category"); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText("Test Codeable Concept Field for third category"); + category3.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test-vitals-panel display")); + category3.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test-alt-vitals-panel display")); + category3.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test-2nd-alt-vitals-panel display")); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + myObservation.setCategory(categoryConcepts); + + // Create CodeableConcept for Code with three codings. + String observationCodeText = "Test Codeable Concept Field for Code"; + CodeableConcept codeableConceptField = new CodeableConcept().setText(observationCodeText); + codeableConceptField.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test-code display")); + codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); + codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); + myObservation.setCode(codeableConceptField); + + myObservationLastNIndexPersistSvc.indexObservation(myObservation); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); + searchParameterMap.add("code", codeParam); + + // execute Observation ID search - Terms Aggregation + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(1, observationIdsOnly.size()); + ObservationJson observationIdOnly = observationIdsOnly.get(0); + assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + // execute Observation ID search - Composite Aggregation +// searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); +// responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); +// observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + +// assertEquals(1, observationIdsOnly.size()); +// observationIdOnly = observationIdsOnly.get(0); +// assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + } + + +// @Test + public void testIndexObservationMultiple() { + + // Create two CodeableConcept values each for a Code with three codings. + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + + // Create two CodeableConcept entities for category, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subjectId = String.valueOf(patientCount); + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + + Observation observation = new Observation(); + observation.setId(String.valueOf(entryCount + patientCount*10)); + Reference subject = new Reference(subjectId); + observation.setSubject(subject); + + if (entryCount%2 == 1) { + observation.setCategory(categoryConcepts1); + observation.setCode(codeableConceptField1); + } else { + observation.setCategory(categoryConcepts2); + observation.setCode(codeableConceptField2); + } + + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observation.setEffective(new DateTimeType(effectiveDtm)); + + myObservationLastNIndexPersistSvc.indexObservation(observation); + + } + + } + + assertEquals(100, myResourceIndexedObservationLastNDao.count()); + assertEquals(2, myCodeableConceptIndexedSearchParamNormalizedDao.count()); + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java new file mode 100644 index 00000000000..e044a9b9e57 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java @@ -0,0 +1,185 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.jpa.dao.lastn.config.TestObservationIndexSearchConfig; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import org.hl7.fhir.r4.model.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.*; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestObservationIndexSearchConfig.class }) +public class PersistObservationIndexedSearchParamLastNTest { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; + + @Autowired + ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; + + @Before + public void before() { + + myResourceIndexedObservationLastNDao.deleteAll(); + myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); + + } + + @After + public void after() { + + myResourceIndexedObservationLastNDao.deleteAll(); + myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); + + } + + @Test + public void testIndexObservationSingle() { + + Observation myObservation = new Observation(); + String resourcePID = "123"; + myObservation.setId(resourcePID); + Reference subjectId = new Reference("4567"); + myObservation.setSubject(subjectId); + Date effectiveDtm = new Date(); + myObservation.setEffective(new DateTimeType(effectiveDtm)); + + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for for second category"); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText("Test Codeable Concept Field for third category"); + category3.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test-vitals-panel display")); + category3.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test-alt-vitals-panel display")); + category3.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test-2nd-alt-vitals-panel display")); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + myObservation.setCategory(categoryConcepts); + + // Create CodeableConcept for Code with three codings. + String observationCodeText = "Test Codeable Concept Field for Code"; + CodeableConcept codeableConceptField = new CodeableConcept().setText(observationCodeText); + codeableConceptField.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test-code display")); + codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); + codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); + myObservation.setCode(codeableConceptField); + + myObservationLastNIndexPersistSvc.indexObservation(myObservation); + + List persistedObservationEntities = myResourceIndexedObservationLastNDao.findAll(); + assertEquals(1, persistedObservationEntities.size()); + ObservationIndexedSearchParamLastNEntity persistedObservationEntity = persistedObservationEntities.get(0); + assertEquals("Patient/"+subjectId.getReference(), persistedObservationEntity.getSubject()); + assertEquals(resourcePID, persistedObservationEntity.getIdentifier()); + assertEquals(effectiveDtm, persistedObservationEntity.getEffectiveDtm()); + + String observationCodeNormalizedId = persistedObservationEntity.getCodeNormalizedId(); + + List persistedObservationCodes = myCodeableConceptIndexedSearchParamNormalizedDao.findAll(); + assertEquals(1, persistedObservationCodes.size()); + ObservationIndexedCodeCodeableConceptEntity persistedObservationCode = persistedObservationCodes.get(0); + assertEquals(observationCodeNormalizedId, persistedObservationCode.getCodeableConceptId()); + assertEquals(observationCodeText, persistedObservationCode.getCodeableConceptText()); + + } + + @Test + public void testIndexObservationMultiple() { + + // Create two CodeableConcept values each for a Code with three codings. + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + + // Create two CodeableConcept entities for category, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subjectId = String.valueOf(patientCount); + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + + Observation observation = new Observation(); + observation.setId(String.valueOf(entryCount + patientCount*10)); + Reference subject = new Reference(subjectId); + observation.setSubject(subject); + + if (entryCount%2 == 1) { + observation.setCategory(categoryConcepts1); + observation.setCode(codeableConceptField1); + } else { + observation.setCategory(categoryConcepts2); + observation.setCode(codeableConceptField2); + } + + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observation.setEffective(new DateTimeType(effectiveDtm)); + + myObservationLastNIndexPersistSvc.indexObservation(observation); + + } + + } + + assertEquals(100, myResourceIndexedObservationLastNDao.count()); + assertEquals(2, myCodeableConceptIndexedSearchParamNormalizedDao.count()); + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java new file mode 100644 index 00000000000..0aec82de5f1 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java @@ -0,0 +1,23 @@ +package ca.uhn.fhir.jpa.dao.lastn.config; + +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import java.io.IOException; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory", + basePackages = {"ca.uhn.fhir.jpa.dao"}) +@EnableTransactionManagement +public class TestIntegratedObservationIndexSearchConfig extends TestObservationIndexSearchConfig { + + @Bean() + public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java new file mode 100644 index 00000000000..ed8dc143e26 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java @@ -0,0 +1,88 @@ +package ca.uhn.fhir.jpa.dao.lastn.config; + +import ca.uhn.fhir.context.ConfigurationException; +import ca.uhn.fhir.jpa.config.TestR4Config; +import ca.uhn.fhir.jpa.config.r4.BaseR4Config; +import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory; +import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; +import org.hibernate.cfg.AvailableSettings; +import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus; +import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; +import pl.allegro.tech.embeddedelasticsearch.PopularProperties; + +import java.io.IOException; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory", + basePackages = {"ca.uhn.fhir.jpa.dao"}) +@EnableTransactionManagement +public class TestObservationIndexSearchConfig extends TestR4Config { + + final String elasticsearchHost = "127.0.0.1"; + final String elasticsearchUserId = ""; + final String elasticsearchPassword = ""; + + private static final String ELASTIC_VERSION = "6.5.4"; + + @Override + public Properties jpaProperties() { + + Properties extraProperties = new Properties(); + extraProperties.put("hibernate.dialect", org.hibernate.dialect.H2Dialect.class); + extraProperties.put("hibernate.format_sql", "false"); + extraProperties.put("hibernate.show_sql", "false"); + extraProperties.put(AvailableSettings.HBM2DDL_AUTO, "update"); + extraProperties.put("hibernate.jdbc.batch_size", "5000"); + extraProperties.put("hibernate.cache.use_query_cache", "false"); + extraProperties.put("hibernate.cache.use_second_level_cache", "false"); + extraProperties.put("hibernate.cache.use_structured_entries", "false"); + extraProperties.put("hibernate.cache.use_minimal_puts", "false"); + extraProperties.put("hibernate.search.model_mapping", LuceneSearchMappingFactory.class.getName()); + extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); + extraProperties.put("hibernate.search.default.worker.execution", "sync"); + + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); + new ElasticsearchHibernatePropertiesBuilder() + .setDebugRefreshAfterWrite(true) + .setDebugPrettyPrintJsonLog(true) + .setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE) + .setIndexManagementWaitTimeoutMillis(10000) + .setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW) + .setRestUrl("http://" + elasticsearchHost + ":" + elasticsearchPort) + .setUsername(elasticsearchUserId) + .setPassword(elasticsearchPassword) + .apply(extraProperties); + + extraProperties.setProperty("hibernate.search.default.elasticsearch.refresh_after_write", "true"); + return extraProperties; + } + + @Bean + public EmbeddedElastic embeddedElasticSearch() { + EmbeddedElastic embeddedElastic; + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } + + return embeddedElastic; + } + + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java new file mode 100644 index 00000000000..2e4c751093b --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java @@ -0,0 +1,195 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.SimpleStopWatch; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import com.fasterxml.jackson.databind.ObjectMapper; +/* +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.shadehapi.elasticsearch.index.query.QueryBuilders; +import org.shadehapi.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.shadehapi.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; +import org.shadehapi.elasticsearch.search.SearchHit; +import org.shadehapi.elasticsearch.search.aggregations.AggregationBuilders; +import org.shadehapi.elasticsearch.search.aggregations.Aggregations; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.shadehapi.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits; +import org.shadehapi.elasticsearch.search.aggregations.support.ValueType; +import org.shadehapi.elasticsearch.search.builder.SearchSourceBuilder; +import org.shadehapi.elasticsearch.search.sort.SortOrder; + */ +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +@Ignore +public class ElasticsearchPerformanceTests { + + private ElasticsearchSvcImpl elasticsearchSvc = new ElasticsearchSvcImpl("localhost", 9301, "elastic", "changeme"); + private List patientIds = new ArrayList<>(); + + @Before + public void before() throws IOException { +/* SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + RandomScoreFunctionBuilder randomScoreFunctionBuilder = new RandomScoreFunctionBuilder(); + Date now = new Date(); + randomScoreFunctionBuilder.seed(now.getTime()); + FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(randomScoreFunctionBuilder); + searchSourceBuilder.query(functionScoreQueryBuilder); + searchSourceBuilder.size(0); + + // Aggregation by order codes + TermsAggregationBuilder observationCodeValuesBuilder = new TermsAggregationBuilder("group_by_patient", ValueType.STRING).field("subject"); + observationCodeValuesBuilder.size(1000); + // Top Hits Aggregation + String[] topHitsInclude = {"subject"}; + observationCodeValuesBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + .sort("effectivedtm", SortOrder.DESC) + .fetchSource(topHitsInclude, null).size(1)); + + searchSourceBuilder.aggregation(observationCodeValuesBuilder); + searchRequest.source(searchSourceBuilder); + + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + patientIds = buildResults(response); +*/ + } + +// @Test + public void testObservationCodesQueryPerformance() throws IOException { +// SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(10000); + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(int i=0; i<1000; i++) { + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List codes = elasticsearchSvc.buildCodeResult(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); +// assertEquals(1000L, codes.size()); + } + System.out.println("Codes query Average elapsed time = " + totalElapsedTimes/1000 + " ms"); + System.out.println("Codes query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/1000 + " ms"); + } + + @Test + public void testObservationsQueryPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List observations = elasticsearchSvc.buildObservationTermsResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); +// assertEquals(25, observations.size()); + } + System.out.println("Observations query Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("Observations query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + +// @Test + public void testSingleObservationsQuery() throws IOException { + long totalElapsedTimes = 0L; + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "5b2091a5-a50e-447b-aff4-c34b69eb43d5"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List observations = elasticsearchSvc.buildObservationCompositeResults(response); +// assertEquals(25, observations.size()); + System.out.println("Average elapsed time = " + totalElapsedTimes/1000 + " ms"); + } + +// @Test + public void testLastNObservationsQueryTermsPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 5); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// elasticsearchSvc.buildObservationTermsResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); + } + System.out.println("LastN Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("LastN Average elapsed search and Build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + +// @Test + public void testLastNObservationsQueryCompPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 5); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// elasticsearchSvc.buildObservationCompositeResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); + } + System.out.println("LastN Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("LastN Average elapsed search and Build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + +/* private List buildResults(SearchResponse theSearchResponse) throws IOException { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedTerms aggregatedObservationCodes = responseAggregations.get("group_by_patient"); + aggregatedObservationCodes.getBuckets(); + List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); + ObjectMapper objectMapper = new ObjectMapper(); + List identifiers = new ArrayList<>(10000); + for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { + Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + SearchHit[] topHits = parsedTopHits.getHits().getHits(); + for (SearchHit topHit : topHits) { + String sources = topHit.getSourceAsString(); + ObservationJson code = objectMapper.readValue(sources,ObservationJson.class); + identifiers.add(code.getSubject().substring(8)); + } + } + return identifiers; + } +*/ + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java new file mode 100644 index 00000000000..ea9308d1896 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java @@ -0,0 +1,153 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.SimpleStopWatch; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits; +import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +@Ignore +public class ElasticsearchV5PerformanceTests { + + private ElasticsearchV5SvcImpl elasticsearchSvc = new ElasticsearchV5SvcImpl("localhost", 9301, "elastic", "changeme"); + private List patientIds = new ArrayList<>(); + + @Before + public void before() throws IOException { + SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + RandomScoreFunctionBuilder randomScoreFunctionBuilder = new RandomScoreFunctionBuilder(); + Date now = new Date(); + randomScoreFunctionBuilder.seed(now.getTime()); + FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(randomScoreFunctionBuilder); + searchSourceBuilder.query(functionScoreQueryBuilder); + searchSourceBuilder.size(0); + + // Aggregation by order codes + TermsAggregationBuilder observationCodeValuesBuilder = new TermsAggregationBuilder("group_by_patient", ValueType.STRING).field("subject"); + observationCodeValuesBuilder.size(1000); + // Top Hits Aggregation + String[] topHitsInclude = {"subject"}; + observationCodeValuesBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + .sort("effectivedtm", SortOrder.DESC) + .fetchSource(topHitsInclude, null).size(1)); + + searchSourceBuilder.aggregation(observationCodeValuesBuilder); + searchRequest.source(searchSourceBuilder); + +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); +// patientIds = buildResults(response); + + } + + @Test + public void testObservationCodesQueryPerformance() throws IOException { +// SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(10000); + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(int i=0; i<1000; i++) { + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List codes = elasticsearchSvc.buildCodeResult(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); +// assertEquals(1000L, codes.size()); + } + System.out.println("Codes query Average elapsed time = " + totalElapsedTimes/1000 + " ms"); + System.out.println("Codes query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/1000 + " ms"); + } + + @Test + public void testObservationsQueryPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List observations = elasticsearchSvc.buildObservationTermsResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); +// assertEquals(25, observations.size()); + } + System.out.println("Observations query Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("Observations query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + + @Test + public void testLastNObservationsQueryTermsPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 5); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// elasticsearchSvc.buildObservationTermsResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); + } + System.out.println("LastN Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("LastN Average elapsed search and Build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + + private List buildResults(SearchResponse theSearchResponse) throws IOException { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedTerms aggregatedObservationCodes = responseAggregations.get("group_by_patient"); + aggregatedObservationCodes.getBuckets(); + List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); + ObjectMapper objectMapper = new ObjectMapper(); + List identifiers = new ArrayList<>(10000); + for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { + Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + SearchHit[] topHits = parsedTopHits.getHits().getHits(); + for (SearchHit topHit : topHits) { + String sources = topHit.getSourceAsString(); + ObservationJson code = objectMapper.readValue(sources,ObservationJson.class); + identifiers.add(code.getSubject().substring(8)); + } + } + return identifiers; + } + + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java new file mode 100644 index 00000000000..b893a73c1a0 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java @@ -0,0 +1,323 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.param.TokenParamModifier; +import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchConfig; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.junit.*; +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.*; + +import static ca.uhn.fhir.jpa.search.lastn.IndexConstants.*; +import static org.junit.Assert.*; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestElasticsearchConfig.class } ) +public class LastNElasticsearchSvcMultipleObservationsTest { + + @Autowired + private ElasticsearchSvcImpl elasticsearchSvc; + + private static ObjectMapper ourMapperNonPrettyPrint; + + private Map>> createdPatientObservationMap = new HashMap<>(); + + + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + @Before + public void before() throws IOException { + createMultiplePatientsAndObservations(); + } + + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(CODE_INDEX); + } + + @Test + public void testLastNNoCriteriaQuery() throws IOException { + + // execute Observation ID search (Terms Aggregation) last 3 observations for each patient + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, null, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + validateQueryResponse(observationIdsOnly); + + // execute Observation ID search (Terms Aggregation) last 3 observations for each patient + searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, null, 3); + responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + + validateQueryResponse(observationIdsOnly); + + // Retrieve all Observation codes + SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + List codes = elasticsearchSvc.buildCodeResult(response); + assertEquals(2, codes.size()); + + } + + private void validateQueryResponse(List observationIdsOnly) { + assertEquals(60, observationIdsOnly.size()); + + // Observation documents should be grouped by subject, then by observation code, and then sorted by effective date/time + // within each observation code. Verify the grouping by creating a nested Map. + Map>> queriedPatientObservationMap = new HashMap<>(); + ObservationJson previousObservationJson = null; + for (ObservationJson observationJson : observationIdsOnly) { + assertNotNull(observationJson.getIdentifier()); + assertNotNull(observationJson.getSubject()); + assertNotNull(observationJson.getCode_concept_id()); + assertNotNull(observationJson.getEffectiveDtm()); + if (previousObservationJson == null) { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } else if (observationJson.getSubject().equals(previousObservationJson.getSubject())) { + if (observationJson.getCode_concept_id().equals(previousObservationJson.getCode_concept_id())) { + queriedPatientObservationMap.get(observationJson.getSubject()).get(observationJson.getCode_concept_id()). + add(observationJson.getEffectiveDtm()); + } else { + Map> codeObservationDateMap = queriedPatientObservationMap.get(observationJson.getSubject()); + // Ensure that code concept was not already retrieved out of order for this subject/patient. + assertFalse(codeObservationDateMap.containsKey(observationJson.getCode_concept_id())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + codeObservationDateMap.put(observationJson.getCode_concept_id(),observationDates); + } + } else { + // Ensure that subject/patient was not already retrieved out of order + assertFalse(queriedPatientObservationMap.containsKey(observationJson.getSubject())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } + previousObservationJson = observationJson; + } + + // Finally check that only the most recent effective date/time values were returned and in the correct order. + for(String subjectId : queriedPatientObservationMap.keySet()) { + Map> queriedObservationCodeMap = queriedPatientObservationMap.get(subjectId); + Map> createdObservationCodeMap = createdPatientObservationMap.get(subjectId); + for(String observationCode : queriedObservationCodeMap.keySet()) { + List queriedObservationDates = queriedObservationCodeMap.get(observationCode); + List createdObservationDates = createdObservationCodeMap.get(observationCode); + for (int dateIdx=0; dateIdx observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(10, observationIdsOnly.size()); + + } + + @Test + public void testLastNCodeCodeOnlyCategoryCodeOnly() throws IOException { + // Include subject and patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam ("test-heart-rate"); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("test-code-1"); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(5, observationIdsOnly.size()); + + } + + @Test + public void testLastNCodeSystemOnlyCategorySystemOnly() throws IOException { + // Include subject and patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", null); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(10, observationIdsOnly.size()); + } + + @Test + public void testLastNCodeCodeTextCategoryTextOnly() throws IOException { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("test-heart-rate display"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("test-code-1 display"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(5, observationIdsOnly.size()); + + } + + private void createMultiplePatientsAndObservations() throws IOException { + // Create CodeableConcepts for two Codes, each with three codings. + String codeableConceptId1 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1); + String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1); + + String codeableConceptId2 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2); + String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2); + + // Create CodeableConcepts for two categories, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subject = "Patient/"+patientCount; + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + + ObservationJson observationJson = new ObservationJson(); + String identifier = String.valueOf((entryCount + patientCount*10)); + observationJson.setIdentifier(identifier); + observationJson.setSubject(subject); + + if (entryCount%2 == 1) { + observationJson.setCategories(categoryConcepts1); + observationJson.setCode(codeableConceptField1); + observationJson.setCode_concept_id(codeableConceptId1); + assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId1, codeJson1Document, CODE_DOCUMENT_TYPE)); + } else { + observationJson.setCategories(categoryConcepts2); + observationJson.setCode(codeableConceptField2); + observationJson.setCode_concept_id(codeableConceptId2); + assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId2, codeJson2Document, CODE_DOCUMENT_TYPE)); + } + + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observationJson.setEffectiveDtm(effectiveDtm); + + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationJson); + assertTrue(elasticsearchSvc.performIndex(OBSERVATION_INDEX, identifier,observationDocument, OBSERVATION_DOCUMENT_TYPE)); + + if (createdPatientObservationMap.containsKey(subject)) { + Map> observationCodeMap = createdPatientObservationMap.get(subject); + if (observationCodeMap.containsKey(observationJson.getCode_concept_id())) { + List observationDates = observationCodeMap.get(observationJson.getCode_concept_id()); + // Want dates to be sorted in descending order + observationDates.add(0, effectiveDtm); + // Only keep the three most recent dates for later check. + if(observationDates.size() > 3) { + observationDates.remove(3); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + observationCodeMap.put(observationJson.getCode_concept_id(), observationDates); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(), observationDates); + createdPatientObservationMap.put(subject, codeObservationMap); + } + } + } + + try { + Thread.sleep(2000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java new file mode 100644 index 00000000000..167d88dba80 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java @@ -0,0 +1,357 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchConfig; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.junit.*; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestElasticsearchConfig.class } ) +public class LastNElasticsearchSvcSingleObservationTest { + + @Autowired + ElasticsearchSvcImpl elasticsearchSvc; + + static ObjectMapper ourMapperNonPrettyPrint; + + final String RESOURCEPID = "123"; + final String SUBJECTID = "4567"; + final String SUBJECTTYPEANDID = "Patient/4567"; + final Date EFFECTIVEDTM = new Date(); + final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category"; + final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; + final String CATEGORYSECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-category"; + final String CATEGORYTHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-category"; + final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; + final String FIRSTCATEGORYFIRSTCODINGDISPLAY = "test-heart-rate display"; + final String FIRSTCATEGORYSECONDCODINGCODE = "test-alt-heart-rate"; + final String FIRSTCATEGORYSECONDCODINGDISPLAY = "test-alt-heart-rate display"; + final String FIRSTCATEGORYTHIRDCODINGCODE = "test-2nd-alt-heart-rate"; + final String FIRSTCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-heart-rate display"; + final String SECONDCATEGORYTEXT = "Test Codeable Concept Field for for second category"; + final String SECONDCATEGORYFIRSTCODINGCODE = "test-vital-signs"; + final String SECONDCATEGORYFIRSTCODINGDISPLAY = "test-vital-signs display"; + final String SECONDCATEGORYSECONDCODINGCODE = "test-alt-vitals"; + final String SECONDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String SECONDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals"; + final String SECONDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals display"; + final String THIRDCATEGORYTEXT = "Test Codeable Concept Field for third category"; + final String THIRDCATEGORYFIRSTCODINGCODE = "test-vital-panel"; + final String THIRDCATEGORYFIRSTCODINGDISPLAY = "test-vitals-panel display"; + final String THIRDCATEGORYSECONDCODINGCODE = "test-alt-vitals-panel"; + final String THIRDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String THIRDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals-panel"; + final String THIRDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals-panel display"; + + final String OBSERVATIONSINGLECODEID = UUID.randomUUID().toString(); + final String OBSERVATIONCODETEXT = "Test Codeable Concept Field for Code"; + final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; + final String CODEFIRSTCODINGCODE = "test-code"; + final String CODEFIRSTCODINGDISPLAY = "test-code display"; + final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; + final String CODESECONDCODINGCODE = "test-alt-code"; + final String CODESECONDCODINGDISPLAY = "test-alt-code display"; + final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; + final String CODETHIRDCODINGCODE = "test-second-alt-code"; + final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; + + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + + } + + // @Before + public void before() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } + + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } + + @Test + public void testSingleObservationQuery() throws IOException { + + createSingleObservation(); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); + searchParameterMap.add("code", codeParam); + + // execute Observation ID search - Terms Aggregation + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(1, observationIdsOnly.size()); + ObservationJson observationIdOnly = observationIdsOnly.get(0); + assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + // execute Observation ID search - Composite Aggregation + searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 3); + responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + + assertEquals(1, observationIdsOnly.size()); + observationIdOnly = observationIdsOnly.get(0); + assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + // execute full Observation search - Terms Aggregation + SearchRequest searchRequestAllFields = elasticsearchSvc.buildObservationAllFieldsTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequestAllFields); + List observations = elasticsearchSvc.buildObservationTermsResults(response); + + validateFullObservationSearch(observations); + + // execute full Observation search - Composite Aggregation + searchRequestAllFields= elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); + response = elasticsearchSvc.executeSearchRequest(searchRequestAllFields); + observations = elasticsearchSvc.buildObservationCompositeResults(response); + + validateFullObservationSearch(observations); + } + + private void validateFullObservationSearch(List observations) throws IOException { + + assertEquals(1, observations.size()); + ObservationJson observation = observations.get(0); + assertEquals(RESOURCEPID, observation.getIdentifier()); + + assertEquals(SUBJECTTYPEANDID, observation.getSubject()); + assertEquals(RESOURCEPID, observation.getIdentifier()); + assertEquals(EFFECTIVEDTM, observation.getEffectiveDtm()); + assertEquals(OBSERVATIONSINGLECODEID, observation.getCode_concept_id()); + + List category_concept_text_values = observation.getCategory_concept_text(); + assertEquals(3,category_concept_text_values.size()); + assertEquals(FIRSTCATEGORYTEXT, category_concept_text_values.get(0)); + assertEquals(SECONDCATEGORYTEXT, category_concept_text_values.get(1)); + assertEquals(THIRDCATEGORYTEXT, category_concept_text_values.get(2)); + + List> category_codings_systems = observation.getCategory_coding_system(); + assertEquals(3,category_codings_systems.size()); + List category_coding_systems = category_codings_systems.get(0); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_coding_systems = category_codings_systems.get(1); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_coding_systems = category_codings_systems.get(2); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + + List> category_codings_codes = observation.getCategory_coding_code(); + assertEquals(3, category_codings_codes.size()); + List category_coding_codes = category_codings_codes.get(0); + assertEquals(3, category_coding_codes.size()); + assertEquals(FIRSTCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(1); + assertEquals(3, category_coding_codes.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(2); + assertEquals(3, category_coding_codes.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + + List> category_codings_displays = observation.getCategory_coding_display(); + assertEquals(3, category_codings_displays.size()); + List category_coding_displays = category_codings_displays.get(0); + assertEquals(FIRSTCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(1); + assertEquals(3, category_coding_displays.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(2); + assertEquals(3, category_coding_displays.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + + List> category_codings_code_system_hashes = observation.getCategory_coding_code_system_hash(); + assertEquals(3, category_codings_code_system_hashes.size()); + List category_coding_code_system_hashes = category_codings_code_system_hashes.get(0); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(1); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(2); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + + String code_concept_text_values = observation.getCode_concept_text(); + assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); + + List code_coding_systems = observation.getCode_coding_system(); + assertEquals(3,code_coding_systems.size()); + assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems.get(0)); + assertEquals(CODESECONDCODINGSYSTEM, code_coding_systems.get(1)); + assertEquals(CODETHIRDCODINGSYSTEM, code_coding_systems.get(2)); + + List code_coding_codes = observation.getCode_coding_code(); + assertEquals(3, code_coding_codes.size()); + assertEquals(CODEFIRSTCODINGCODE, code_coding_codes.get(0)); + assertEquals(CODESECONDCODINGCODE, code_coding_codes.get(1)); + assertEquals(CODETHIRDCODINGCODE, code_coding_codes.get(2)); + + List code_coding_display = observation.getCode_coding_display(); + assertEquals(3, code_coding_display.size()); + assertEquals(CODEFIRSTCODINGDISPLAY, code_coding_display.get(0)); + assertEquals(CODESECONDCODINGDISPLAY, code_coding_display.get(1)); + assertEquals(CODETHIRDCODINGDISPLAY, code_coding_display.get(2)); + + List code_coding_code_system_hash = observation.getCode_coding_code_system_hash(); + assertEquals(3, code_coding_code_system_hash.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), code_coding_code_system_hash.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), code_coding_code_system_hash.get(2)); + + // Retrieve all Observation codes + SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + List codes = elasticsearchSvc.buildCodeResult(response); + assertEquals(1, codes.size()); + CodeJson persistedObservationCode = codes.get(0); + + String persistedCodeConceptID = persistedObservationCode.getCodeableConceptId(); + assertEquals(OBSERVATIONSINGLECODEID, persistedCodeConceptID); + String persistedCodeConceptText = persistedObservationCode.getCodeableConceptText(); + assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText); + + List persistedCodeCodingSystems = persistedObservationCode.getCoding_system(); + assertEquals(3,persistedCodeCodingSystems.size()); + assertEquals(CODEFIRSTCODINGSYSTEM, persistedCodeCodingSystems.get(0)); + assertEquals(CODESECONDCODINGSYSTEM, persistedCodeCodingSystems.get(1)); + assertEquals(CODETHIRDCODINGSYSTEM, persistedCodeCodingSystems.get(2)); + + List persistedCodeCodingCodes = persistedObservationCode.getCoding_code(); + assertEquals(3, persistedCodeCodingCodes.size()); + assertEquals(CODEFIRSTCODINGCODE, persistedCodeCodingCodes.get(0)); + assertEquals(CODESECONDCODINGCODE, persistedCodeCodingCodes.get(1)); + assertEquals(CODETHIRDCODINGCODE, persistedCodeCodingCodes.get(2)); + + List persistedCodeCodingDisplays = persistedObservationCode.getCoding_display(); + assertEquals(3, persistedCodeCodingDisplays.size()); + assertEquals(CODEFIRSTCODINGDISPLAY, persistedCodeCodingDisplays.get(0)); + assertEquals(CODESECONDCODINGDISPLAY, persistedCodeCodingDisplays.get(1)); + assertEquals(CODETHIRDCODINGDISPLAY, persistedCodeCodingDisplays.get(2)); + + List persistedCodeCodingCodeSystemHashes = persistedObservationCode.getCoding_code_system_hash(); + assertEquals(3, persistedCodeCodingCodeSystemHashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(2)); + + + } + + private void createSingleObservation() throws IOException { + ObservationJson indexedObservation = new ObservationJson(); + indexedObservation.setIdentifier(RESOURCEPID); + indexedObservation.setSubject(SUBJECTTYPEANDID); + indexedObservation.setEffectiveDtm(EFFECTIVEDTM); + + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText(FIRSTCATEGORYTEXT); + category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY)); + category1.add(new Coding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY)); + category1.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText(SECONDCATEGORYTEXT); + category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY)); + category2.add(new Coding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY)); + category2.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText(THIRDCATEGORYTEXT); + category3.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY)); + category3.add(new Coding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY)); + category3.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + indexedObservation.setCategories(categoryConcepts); + + // Create CodeableConcept for Code with three codings. + indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID); + CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT); + codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY)); + codeableConceptField.addCoding(new Coding(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE, CODESECONDCODINGDISPLAY)); + codeableConceptField.addCoding(new Coding(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE, CODETHIRDCODINGDISPLAY)); + indexedObservation.setCode(codeableConceptField); + + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); + + CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); + String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); + + try { + Thread.sleep(1000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java new file mode 100644 index 00000000000..08f731e0f15 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java @@ -0,0 +1,312 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchV5Config; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.param.TokenParamModifier; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.*; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestElasticsearchV5Config.class } ) +public class LastNElasticsearchV5SvcMultipleObservationsTest { + + @Autowired + private ElasticsearchV5SvcImpl elasticsearchSvc; + + private static ObjectMapper ourMapperNonPrettyPrint; + + private Map>> createdPatientObservationMap = new HashMap<>(); + + + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + @Before + public void before() throws IOException { +// createMultiplePatientsAndObservations(); + } + + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } +/* + @Test + public void testLastNNoCriteriaQuery() throws IOException { + + // execute Observation ID search (Terms Aggregation) last 3 observations for each patient + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, null, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + validateQueryResponse(observationIdsOnly); + + } + + private void validateQueryResponse(List observationIdsOnly) { + assertEquals(60, observationIdsOnly.size()); + + // Observation documents should be grouped by subject, then by observation code, and then sorted by effective date/time + // within each observation code. Verify the grouping by creating a nested Map. + Map>> queriedPatientObservationMap = new HashMap<>(); + ObservationJson previousObservationJson = null; + for (ObservationJson observationJson : observationIdsOnly) { + assertNotNull(observationJson.getIdentifier()); + assertNotNull(observationJson.getSubject()); + assertNotNull(observationJson.getCode_concept_id()); + assertNotNull(observationJson.getEffectiveDtm()); + if (previousObservationJson == null) { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } else if (observationJson.getSubject().equals(previousObservationJson.getSubject())) { + if (observationJson.getCode_concept_id().equals(previousObservationJson.getCode_concept_id())) { + queriedPatientObservationMap.get(observationJson.getSubject()).get(observationJson.getCode_concept_id()). + add(observationJson.getEffectiveDtm()); + } else { + Map> codeObservationDateMap = queriedPatientObservationMap.get(observationJson.getSubject()); + // Ensure that code concept was not already retrieved out of order for this subject/patient. + assertFalse(codeObservationDateMap.containsKey(observationJson.getCode_concept_id())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + codeObservationDateMap.put(observationJson.getCode_concept_id(),observationDates); + } + } else { + // Ensure that subject/patient was not already retrieved out of order + assertFalse(queriedPatientObservationMap.containsKey(observationJson.getSubject())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } + previousObservationJson = observationJson; + } + + // Finally check that only the most recent effective date/time values were returned and in the correct order. + for(String subjectId : queriedPatientObservationMap.keySet()) { + Map> queriedObservationCodeMap = queriedPatientObservationMap.get(subjectId); + Map> createdObservationCodeMap = createdPatientObservationMap.get(subjectId); + for(String observationCode : queriedObservationCodeMap.keySet()) { + List queriedObservationDates = queriedObservationCodeMap.get(observationCode); + List createdObservationDates = createdObservationCodeMap.get(observationCode); + for (int dateIdx=0; dateIdx observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(10, observationIdsOnly.size()); + + } + + @Test + public void testLastNCodeCodeOnlyCategoryCodeOnly() throws IOException { + // Include subject and patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam ("test-heart-rate"); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("test-code-1"); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(5, observationIdsOnly.size()); + + } + + @Test + public void testLastNCodeSystemOnlyCategorySystemOnly() throws IOException { + // Include subject and patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", null); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(10, observationIdsOnly.size()); + } + + @Test + public void testLastNCodeCodeTextCategoryTextOnly() throws IOException { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("test-heart-rate display"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("test-code-1 display"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(5, observationIdsOnly.size()); + + } + + private void createMultiplePatientsAndObservations() throws IOException { + // Create CodeableConcepts for two Codes, each with three codings. + String codeableConceptId1 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1); + String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1); + + String codeableConceptId2 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2); + String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2); + + // Create CodeableConcepts for two categories, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subject = "Patient/"+patientCount; + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + + ObservationJson observationJson = new ObservationJson(); + String identifier = String.valueOf((entryCount + patientCount*10)); + observationJson.setIdentifier(identifier); + observationJson.setSubject(subject); + + if (entryCount%2 == 1) { + observationJson.setCategories(categoryConcepts1); + observationJson.setCode(codeableConceptField1); + observationJson.setCode_concept_id(codeableConceptId1); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, codeableConceptId1, codeJson1Document, IndexConstants.CODE_DOCUMENT_TYPE)); + } else { + observationJson.setCategories(categoryConcepts2); + observationJson.setCode(codeableConceptField2); + observationJson.setCode_concept_id(codeableConceptId2); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, codeableConceptId2, codeJson2Document, IndexConstants.CODE_DOCUMENT_TYPE)); + } + + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observationJson.setEffectiveDtm(effectiveDtm); + + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationJson); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, identifier,observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); + + if (createdPatientObservationMap.containsKey(subject)) { + Map> observationCodeMap = createdPatientObservationMap.get(subject); + if (observationCodeMap.containsKey(observationJson.getCode_concept_id())) { + List observationDates = observationCodeMap.get(observationJson.getCode_concept_id()); + // Want dates to be sorted in descending order + observationDates.add(0, effectiveDtm); + // Only keep the three most recent dates for later check. + if(observationDates.size() > 3) { + observationDates.remove(3); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + observationCodeMap.put(observationJson.getCode_concept_id(), observationDates); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(), observationDates); + createdPatientObservationMap.put(subject, codeObservationMap); + } + } + } + + try { + Thread.sleep(2000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } + + } +*/ +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java new file mode 100644 index 00000000000..48b42d31ec5 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java @@ -0,0 +1,346 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchV5Config; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestElasticsearchV5Config.class } ) +public class LastNElasticsearchV5SvcSingleObservationTest { + + @Autowired + ElasticsearchV5SvcImpl elasticsearchSvc; + + static ObjectMapper ourMapperNonPrettyPrint; + + final String RESOURCEPID = "123"; + final String SUBJECTID = "4567"; + final String SUBJECTTYPEANDID = "Patient/4567"; + final Date EFFECTIVEDTM = new Date(); + final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category"; + final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; + final String CATEGORYSECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-category"; + final String CATEGORYTHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-category"; + final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; + final String FIRSTCATEGORYFIRSTCODINGDISPLAY = "test-heart-rate display"; + final String FIRSTCATEGORYSECONDCODINGCODE = "test-alt-heart-rate"; + final String FIRSTCATEGORYSECONDCODINGDISPLAY = "test-alt-heart-rate display"; + final String FIRSTCATEGORYTHIRDCODINGCODE = "test-2nd-alt-heart-rate"; + final String FIRSTCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-heart-rate display"; + final String SECONDCATEGORYTEXT = "Test Codeable Concept Field for for second category"; + final String SECONDCATEGORYFIRSTCODINGCODE = "test-vital-signs"; + final String SECONDCATEGORYFIRSTCODINGDISPLAY = "test-vital-signs display"; + final String SECONDCATEGORYSECONDCODINGCODE = "test-alt-vitals"; + final String SECONDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String SECONDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals"; + final String SECONDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals display"; + final String THIRDCATEGORYTEXT = "Test Codeable Concept Field for third category"; + final String THIRDCATEGORYFIRSTCODINGCODE = "test-vital-panel"; + final String THIRDCATEGORYFIRSTCODINGDISPLAY = "test-vitals-panel display"; + final String THIRDCATEGORYSECONDCODINGCODE = "test-alt-vitals-panel"; + final String THIRDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String THIRDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals-panel"; + final String THIRDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals-panel display"; + + final String OBSERVATIONSINGLECODEID = UUID.randomUUID().toString(); + final String OBSERVATIONCODETEXT = "Test Codeable Concept Field for Code"; + final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; + final String CODEFIRSTCODINGCODE = "test-code"; + final String CODEFIRSTCODINGDISPLAY = "test-code display"; + final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; + final String CODESECONDCODINGCODE = "test-alt-code"; + final String CODESECONDCODINGDISPLAY = "test-alt-code display"; + final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; + final String CODETHIRDCODINGCODE = "test-second-alt-code"; + final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; + + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + + } + + // @Before + public void before() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } + + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } + @Test + public void testSingleObservationQuery() throws IOException { + + createSingleObservation(); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); + searchParameterMap.add("code", codeParam); + + // execute Observation ID search - Terms Aggregation +/* SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(1, observationIdsOnly.size()); + ObservationJson observationIdOnly = observationIdsOnly.get(0); + assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + // execute full Observation search - Terms Aggregation + SearchRequest searchRequestAllFields = elasticsearchSvc.buildObservationAllFieldsTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequestAllFields); + List observations = elasticsearchSvc.buildObservationTermsResults(response); + + validateFullObservationSearch(observations); +*/ + } + + private void validateFullObservationSearch(List observations) throws IOException { + + assertEquals(1, observations.size()); + ObservationJson observation = observations.get(0); + assertEquals(RESOURCEPID, observation.getIdentifier()); + + assertEquals(SUBJECTTYPEANDID, observation.getSubject()); + assertEquals(RESOURCEPID, observation.getIdentifier()); + assertEquals(EFFECTIVEDTM, observation.getEffectiveDtm()); + assertEquals(OBSERVATIONSINGLECODEID, observation.getCode_concept_id()); + + List category_concept_text_values = observation.getCategory_concept_text(); + assertEquals(3,category_concept_text_values.size()); + assertEquals(FIRSTCATEGORYTEXT, category_concept_text_values.get(0)); + assertEquals(SECONDCATEGORYTEXT, category_concept_text_values.get(1)); + assertEquals(THIRDCATEGORYTEXT, category_concept_text_values.get(2)); + + List> category_codings_systems = observation.getCategory_coding_system(); + assertEquals(3,category_codings_systems.size()); + List category_coding_systems = category_codings_systems.get(0); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_codings_systems.get(1); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_codings_systems.get(2); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + + List> category_codings_codes = observation.getCategory_coding_code(); + assertEquals(3, category_codings_codes.size()); + List category_coding_codes = category_codings_codes.get(0); + assertEquals(3, category_coding_codes.size()); + assertEquals(FIRSTCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(1); + assertEquals(3, category_coding_codes.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(2); + assertEquals(3, category_coding_codes.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + + List> category_codings_displays = observation.getCategory_coding_display(); + assertEquals(3, category_codings_displays.size()); + List category_coding_displays = category_codings_displays.get(0); + assertEquals(FIRSTCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(1); + assertEquals(3, category_coding_displays.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(2); + assertEquals(3, category_coding_displays.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + + List> category_codings_code_system_hashes = observation.getCategory_coding_code_system_hash(); + assertEquals(3, category_codings_code_system_hashes.size()); + List category_coding_code_system_hashes = category_codings_code_system_hashes.get(0); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(1); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(2); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + + String code_concept_text_values = observation.getCode_concept_text(); + assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); + + List code_coding_systems = observation.getCode_coding_system(); + assertEquals(3,code_coding_systems.size()); + assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems.get(0)); + assertEquals(CODESECONDCODINGSYSTEM, code_coding_systems.get(1)); + assertEquals(CODETHIRDCODINGSYSTEM, code_coding_systems.get(2)); + + List code_coding_codes = observation.getCode_coding_code(); + assertEquals(3, code_coding_codes.size()); + assertEquals(CODEFIRSTCODINGCODE, code_coding_codes.get(0)); + assertEquals(CODESECONDCODINGCODE, code_coding_codes.get(1)); + assertEquals(CODETHIRDCODINGCODE, code_coding_codes.get(2)); + + List code_coding_display = observation.getCode_coding_display(); + assertEquals(3, code_coding_display.size()); + assertEquals(CODEFIRSTCODINGDISPLAY, code_coding_display.get(0)); + assertEquals(CODESECONDCODINGDISPLAY, code_coding_display.get(1)); + assertEquals(CODETHIRDCODINGDISPLAY, code_coding_display.get(2)); + + List code_coding_code_system_hash = observation.getCode_coding_code_system_hash(); + assertEquals(3, code_coding_code_system_hash.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), code_coding_code_system_hash.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), code_coding_code_system_hash.get(2)); + + // Retrieve all Observation codes +/* SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + List codes = elasticsearchSvc.buildCodeResult(response); + assertEquals(1, codes.size()); + CodeJson persistedObservationCode = codes.get(0); + + String persistedCodeConceptID = persistedObservationCode.getCodeableConceptId(); + assertEquals(OBSERVATIONSINGLECODEID, persistedCodeConceptID); + String persistedCodeConceptText = persistedObservationCode.getCodeableConceptText(); + assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText); + + List persistedCodeCodingSystems = persistedObservationCode.getCoding_system(); + assertEquals(3,persistedCodeCodingSystems.size()); + assertEquals(CODEFIRSTCODINGSYSTEM, persistedCodeCodingSystems.get(0)); + assertEquals(CODESECONDCODINGSYSTEM, persistedCodeCodingSystems.get(1)); + assertEquals(CODETHIRDCODINGSYSTEM, persistedCodeCodingSystems.get(2)); + + List persistedCodeCodingCodes = persistedObservationCode.getCoding_code(); + assertEquals(3, persistedCodeCodingCodes.size()); + assertEquals(CODEFIRSTCODINGCODE, persistedCodeCodingCodes.get(0)); + assertEquals(CODESECONDCODINGCODE, persistedCodeCodingCodes.get(1)); + assertEquals(CODETHIRDCODINGCODE, persistedCodeCodingCodes.get(2)); + + List persistedCodeCodingDisplays = persistedObservationCode.getCoding_display(); + assertEquals(3, persistedCodeCodingDisplays.size()); + assertEquals(CODEFIRSTCODINGDISPLAY, persistedCodeCodingDisplays.get(0)); + assertEquals(CODESECONDCODINGDISPLAY, persistedCodeCodingDisplays.get(1)); + assertEquals(CODETHIRDCODINGDISPLAY, persistedCodeCodingDisplays.get(2)); + + List persistedCodeCodingCodeSystemHashes = persistedObservationCode.getCoding_code_system_hash(); + assertEquals(3, persistedCodeCodingCodeSystemHashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(2)); +*/ + + } + + private void createSingleObservation() throws IOException { + ObservationJson indexedObservation = new ObservationJson(); + indexedObservation.setIdentifier(RESOURCEPID); + indexedObservation.setSubject(SUBJECTTYPEANDID); + indexedObservation.setEffectiveDtm(EFFECTIVEDTM); + + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText(FIRSTCATEGORYTEXT); + category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY)); + category1.add(new Coding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY)); + category1.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText(SECONDCATEGORYTEXT); + category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY)); + category2.add(new Coding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY)); + category2.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText(THIRDCATEGORYTEXT); + category3.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY)); + category3.add(new Coding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY)); + category3.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + indexedObservation.setCategories(categoryConcepts); + + // Create CodeableConcept for Code with three codings. + indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID); + CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT); + codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY)); + codeableConceptField.addCoding(new Coding(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE, CODESECONDCODINGDISPLAY)); + codeableConceptField.addCoding(new Coding(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE, CODETHIRDCODINGDISPLAY)); + indexedObservation.setCode(codeableConceptField); + + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); + + CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); + String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); + + try { + Thread.sleep(1000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java new file mode 100644 index 00000000000..4b358584ca7 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java @@ -0,0 +1,53 @@ +package ca.uhn.fhir.jpa.search.lastn.config; + +import ca.uhn.fhir.context.ConfigurationException; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; +import pl.allegro.tech.embeddedelasticsearch.PopularProperties; + +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") +@EnableTransactionManagement +public class TestElasticsearchConfig { + + private final String elasticsearchHost = "127.0.0.1"; + private final String elasticsearchUserId = ""; + private final String elasticsearchPassword = ""; + + private static final String ELASTIC_VERSION = "6.5.4"; + + + @Bean() + public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + + @Bean + public EmbeddedElastic embeddedElasticSearch() { + EmbeddedElastic embeddedElastic = null; + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } + + return embeddedElastic; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java new file mode 100644 index 00000000000..a28e092bba9 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java @@ -0,0 +1,53 @@ +package ca.uhn.fhir.jpa.search.lastn.config; + +import ca.uhn.fhir.context.ConfigurationException; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchV5SvcImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; +import pl.allegro.tech.embeddedelasticsearch.PopularProperties; + +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") +@EnableTransactionManagement +public class TestElasticsearchV5Config { + + private final String elasticsearchHost = "127.0.0.1"; + private final String elasticsearchUserId = ""; + private final String elasticsearchPassword = ""; + + private static final String ELASTIC_VERSION = "5.6.16"; + + + @Bean() + public ElasticsearchV5SvcImpl myElasticsearchSvc() throws IOException { + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); + return new ElasticsearchV5SvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + + @Bean + public EmbeddedElastic embeddedElasticSearch() { + EmbeddedElastic embeddedElastic = null; + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } + + return embeddedElastic; + } + +} diff --git a/pom.xml b/pom.xml index bf88321d7d5..2b8c3bdb66b 100644 --- a/pom.xml +++ b/pom.xml @@ -1561,7 +1561,7 @@ org.basepom.maven duplicate-finder-maven-plugin - 1.3.0 + 1.4.0 de.jpdigital @@ -2466,6 +2466,7 @@ hapi-fhir-structures-r5 hapi-fhir-validation-resources-r5 hapi-fhir-igpacks + hapi-fhir-elasticsearch-6 hapi-fhir-jpaserver-model hapi-fhir-jpaserver-searchparam hapi-fhir-jpaserver-subscription From 83fde618461afa474a382f4a06023eb4b6f015a2 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Wed, 25 Mar 2020 18:02:57 -0400 Subject: [PATCH 02/31] Initial commit of changes to support $lastn operation. --- hapi-fhir-jpaserver-base/pom.xml | 11 +++++++++++ .../main/java/ca/uhn/fhir/jpa/config/BaseConfig.java | 2 +- .../java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index dd305ce4d61..b4a33a33486 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -612,6 +612,17 @@ 16.0.3 compile + + ca.uhn.hapi.fhir + hapi-fhir-elasticsearch-6 + 1.0-SNAPSHOT + shaded6 + + diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index 1d6007615f2..04cd79307fb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -280,7 +280,7 @@ public abstract class BaseConfig { public static void configureEntityManagerFactory(LocalContainerEntityManagerFactoryBean theFactory, FhirContext theCtx) { theFactory.setJpaDialect(hibernateJpaDialect(theCtx.getLocalizer())); - theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity"); + theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.dao.lastn.entity"); theFactory.setPersistenceProvider(new HibernatePersistenceProvider()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java index 3e550ffdc92..b26bb49cd2d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java @@ -7,6 +7,7 @@ import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl; import ca.uhn.fhir.jpa.dao.IFhirSystemDao; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.dao.TransactionProcessor; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.dao.r4.TransactionProcessorVersionAdapterR4; import ca.uhn.fhir.jpa.provider.GraphQLProvider; import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorR4; @@ -134,4 +135,11 @@ public class BaseR4Config extends BaseConfigDstu3Plus { return new TermReadSvcR4(); } + @Bean + public ObservationLastNIndexPersistSvc observationLastNIndexpersistSvc() { + return new ObservationLastNIndexPersistSvc(); + } + + + } From f819c91530b591d714c7da278cec5745a3d91fc6 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 31 Mar 2020 21:55:57 -0400 Subject: [PATCH 03/31] Initial implementation of lastn operation. --- .../main/java/ca/uhn/fhir/cli/BaseApp.java | 1 + .../RunJpaServerWithElasticsearchCommand.java | 250 ++++++++++++++++++ hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml | 4 + .../ca/uhn/fhir/jpa/demo/CommonConfig.java | 3 +- .../fhir/jpa/demo/CommonPostgreSQLConfig.java | 180 +++++++++++++ .../jpa/demo/ContextPostgreSQLHolder.java | 89 +++++++ .../demo/FhirServerElasticsearchConfigR4.java | 107 ++++++++ .../ca/uhn/fhir/jpa/config/BaseConfig.java | 1 + .../jpa/config/dstu3/BaseDstu3Config.java | 9 +- .../uhn/fhir/jpa/config/r4/BaseR4Config.java | 11 +- .../uhn/fhir/jpa/config/r5/BaseR5Config.java | 9 +- .../jpa/dao/IFhirResourceDaoObservation.java | 34 +++ .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 53 +++- ...exedCodeCodeableConceptSearchParamDao.java | 8 + .../FhirResourceDaoObservationDstu3.java | 113 ++++++++ .../ObservationLastNIndexPersistDstu3Svc.java | 87 ++++++ .../ObservationLastNIndexPersistR4Svc.java | 88 ++++++ .../ObservationLastNIndexPersistR5Svc.java | 87 ++++++ .../ObservationLastNIndexPersistSvc.java | 2 +- ...ationIndexedCodeCodeableConceptEntity.java | 16 +- .../ObservationIndexedCodeCodingEntity.java | 10 +- .../dao/r4/FhirResourceDaoObservationR4.java | 59 ++--- .../dao/r5/FhirResourceDaoObservationR5.java | 113 ++++++++ ...seJpaResourceProviderObservationDstu2.java | 148 +++++++++++ ...seJpaResourceProviderObservationDstu3.java | 148 +++++++++++ .../BaseJpaResourceProviderObservationR4.java | 149 +++++++++++ .../BaseJpaResourceProviderObservationR5.java | 148 +++++++++++ .../search/lastn/ElasticsearchSvcImpl.java | 116 +++++--- .../jpa/search/lastn/IElasticsearchSvc.java | 8 + .../lastn/config/ElasticsearchConfig.java | 26 -- .../search/lastn/json/ObservationJson.java | 42 ++- ...bservationIndexedSearchParamLastNTest.java | 91 ++++++- ...bservationIndexedSearchParamLastNTest.java | 13 +- ...ntegratedObservationIndexSearchConfig.java | 2 +- .../TestObservationIndexSearchConfig.java | 7 +- .../ElasticsearchV5PerformanceTests.java | 4 +- ...ElasticsearchSvcSingleObservationTest.java | 14 + ...icsearchV5SvcMultipleObservationsTest.java | 6 +- ...asticsearchV5SvcSingleObservationTest.java | 18 +- .../lastn/config/TestElasticsearchConfig.java | 6 + .../config/TestElasticsearchV5Config.java | 12 +- .../uhn/fhir/jpa/model/util/JpaConstants.java | 5 + .../jpa/searchparam/SearchParameterMap.java | 19 ++ .../resources/vm/jpa_resource_provider.vm | 2 +- .../src/main/resources/vm/jpa_spring_beans.vm | 2 + .../resources/vm/jpa_spring_beans_java.vm | 5 + 46 files changed, 2144 insertions(+), 181 deletions(-) create mode 100644 hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunJpaServerWithElasticsearchCommand.java create mode 100644 hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonPostgreSQLConfig.java create mode 100644 hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextPostgreSQLHolder.java create mode 100644 hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerElasticsearchConfigR4.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFhirResourceDaoObservation.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/BaseApp.java b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/BaseApp.java index c1b98297f41..bd1451d7730 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/BaseApp.java +++ b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/BaseApp.java @@ -173,6 +173,7 @@ public abstract class BaseApp { commands.add(new ExportConceptMapToCsvCommand()); commands.add(new ImportCsvToConceptMapCommand()); commands.add(new HapiFlywayMigrateDatabaseCommand()); + commands.add(new RunJpaServerWithElasticsearchCommand()); return commands; } diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunJpaServerWithElasticsearchCommand.java b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunJpaServerWithElasticsearchCommand.java new file mode 100644 index 00000000000..ca70d50845b --- /dev/null +++ b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunJpaServerWithElasticsearchCommand.java @@ -0,0 +1,250 @@ +package ca.uhn.fhir.cli; + +/*- + * #%L + * HAPI FHIR - Command Line Client - API + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.demo.*; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.webapp.WebAppContext; +import org.springframework.web.context.ContextLoader; +import org.springframework.web.context.ContextLoaderListener; +import org.springframework.web.context.support.AnnotationConfigWebApplicationContext; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import java.io.*; +import java.net.SocketException; + +public class RunJpaServerWithElasticsearchCommand extends BaseCommand { + + private static final String OPTION_DISABLE_REFERENTIAL_INTEGRITY = "disable-referential-integrity"; + private static final String OPTION_LOWMEM = "lowmem"; + private static final String OPTION_ALLOW_EXTERNAL_REFS = "allow-external-refs"; + private static final String OPTION_REUSE_SEARCH_RESULTS_MILLIS = "reuse-search-results-milliseconds"; + private static final String OPTION_EXTERNAL_ELASTICSEARCH = "external-elasticsearch"; + private static final int DEFAULT_PORT = 8080; + private static final String OPTION_P = "p"; + + // TODO: Don't use qualified names for loggers in HAPI CLI. + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(RunJpaServerWithElasticsearchCommand.class); + public static final String RUN_SERVER_COMMAND_ELASTICSEARCH = "run-server-elasticsearch"; + private int myPort; + + private Server myServer; + + @Override + public String getCommandName() { + return RUN_SERVER_COMMAND_ELASTICSEARCH; + } + + @Override + public Options getOptions() { + Options options = new Options(); + addFhirVersionOption(options); + options.addOption(OPTION_P, "port", true, "The port to listen on (default is " + DEFAULT_PORT + ")"); + options.addOption(null, OPTION_LOWMEM, false, "If this flag is set, the server will operate in low memory mode (some features disabled)"); + options.addOption(null, OPTION_ALLOW_EXTERNAL_REFS, false, "If this flag is set, the server will allow resources to be persisted contaning external resource references"); + options.addOption(null, OPTION_DISABLE_REFERENTIAL_INTEGRITY, false, "If this flag is set, the server will not enforce referential integrity"); + options.addOption(null, OPTION_EXTERNAL_ELASTICSEARCH, false, "If this flag is set, the server will attempt to use an external elasticsearch instance listening on port 9301"); + + addOptionalOption(options, "u", "url", "Url", "If this option is set, specifies the JDBC URL to use for the database connection"); + + Long defaultReuseSearchResults = DaoConfig.DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS; + String defaultReuseSearchResultsStr = defaultReuseSearchResults == null ? "off" : String.valueOf(defaultReuseSearchResults); + options.addOption(null, OPTION_REUSE_SEARCH_RESULTS_MILLIS, true, "The time in milliseconds within which the same results will be returned for multiple identical searches, or \"off\" (default is " + defaultReuseSearchResultsStr + ")"); + + return options; + } + + private int parseOptionInteger(CommandLine theCommandLine, String opt, int defaultPort) throws ParseException { + try { + return Integer.parseInt(theCommandLine.getOptionValue(opt, Integer.toString(defaultPort))); + } catch (NumberFormatException e) { + throw new ParseException("Invalid value '" + theCommandLine.getOptionValue(opt) + "' (must be numeric)"); + } + } + + @Override + public void run(CommandLine theCommandLine) throws ParseException { + parseFhirContext(theCommandLine); + + myPort = parseOptionInteger(theCommandLine, OPTION_P, DEFAULT_PORT); + + if (theCommandLine.hasOption(OPTION_LOWMEM)) { + ourLog.info("Running in low memory mode, some features disabled"); + System.setProperty(OPTION_LOWMEM, OPTION_LOWMEM); + } + + if (theCommandLine.hasOption(OPTION_ALLOW_EXTERNAL_REFS)) { + ourLog.info("Server is configured to allow external references"); + ContextPostgreSQLHolder.setAllowExternalRefs(true); + } + + if (theCommandLine.hasOption(OPTION_DISABLE_REFERENTIAL_INTEGRITY)) { + ourLog.info("Server is configured to not enforce referential integrity"); + ContextPostgreSQLHolder.setDisableReferentialIntegrity(true); + } + + if (theCommandLine.hasOption(OPTION_EXTERNAL_ELASTICSEARCH)) { + ourLog.info("Server is configured to use external elasticsearch"); + ContextPostgreSQLHolder.setExternalElasticsearch(true); + } + + ContextPostgreSQLHolder.setDatabaseUrl(theCommandLine.getOptionValue("u")); + + String reuseSearchResults = theCommandLine.getOptionValue(OPTION_REUSE_SEARCH_RESULTS_MILLIS); + if (reuseSearchResults != null) { + if (reuseSearchResults.equals("off")) { + ourLog.info("Server is configured to not reuse search results"); + ContextPostgreSQLHolder.setReuseCachedSearchResultsForMillis(null); + } else { + try { + long reuseSearchResultsMillis = Long.parseLong(reuseSearchResults); + if (reuseSearchResultsMillis < 0) { + throw new NumberFormatException("expected a positive integer"); + } + ourLog.info("Server is configured to reuse search results for " + String.valueOf(reuseSearchResultsMillis) + " milliseconds"); + ContextPostgreSQLHolder.setReuseCachedSearchResultsForMillis(reuseSearchResultsMillis); + } catch (NumberFormatException e) { + throw new ParseException("Invalid value '" + reuseSearchResults + "' (must be a positive integer)"); + } + } + } + + ContextPostgreSQLHolder.setCtx(getFhirContext()); + + + + ourLog.info("Preparing HAPI FHIR JPA server on port {}", myPort); + File tempWarFile; + try { + tempWarFile = File.createTempFile("hapi-fhir", ".war"); + tempWarFile.deleteOnExit(); + + InputStream inStream = RunJpaServerWithElasticsearchCommand.class.getResourceAsStream("/hapi-fhir-cli-jpaserver.war"); + OutputStream outStream = new BufferedOutputStream(new FileOutputStream(tempWarFile, false)); + IOUtils.copy(inStream, outStream); + } catch (IOException e) { + ourLog.error("Failed to create temporary file", e); + return; + } + + final ContextLoaderListener cll = new ContextLoaderListener(); + + ourLog.info("Starting HAPI FHIR JPA server in {} mode", ContextPostgreSQLHolder.getCtx().getVersion().getVersion()); + WebAppContext root = new WebAppContext(); + root.setAllowDuplicateFragmentNames(true); + root.setWar(tempWarFile.getAbsolutePath()); + root.setParentLoaderPriority(true); + root.setContextPath("/"); + root.addEventListener(new ServletContextListener() { + @Override + public void contextInitialized(ServletContextEvent theSce) { + theSce.getServletContext().setInitParameter(ContextLoader.CONTEXT_CLASS_PARAM, AnnotationConfigWebApplicationContext.class.getName()); + switch (ContextPostgreSQLHolder.getCtx().getVersion().getVersion()) { + case DSTU2: + theSce.getServletContext().setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, FhirServerConfig.class.getName()); + break; + case DSTU3: + theSce.getServletContext().setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, FhirServerConfigDstu3.class.getName()); + break; + case R4: + theSce.getServletContext().setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, FhirServerElasticsearchConfigR4.class.getName()); + break; + case DSTU2_1: + case DSTU2_HL7ORG: + break; + } + cll.contextInitialized(theSce); + } + + @Override + public void contextDestroyed(ServletContextEvent theSce) { + cll.contextDestroyed(theSce); + } + }); + + String path = ContextPostgreSQLHolder.getPath(); + root.addServlet("ca.uhn.fhir.jpa.demo.JpaServerDemo", path + "*"); + + myServer = new Server(myPort); + myServer.setHandler(root); + try { + myServer.start(); + } catch (SocketException e) { + throw new CommandFailureException("Server failed to start on port " + myPort + " because of the following error \"" + e.toString() + "\". Note that you can use the '-p' option to specify an alternate port."); + } catch (Exception e) { + ourLog.error("Server failed to start", e); + throw new CommandFailureException("Server failed to start", e); + } + + ourLog.info("Server started on port {}", myPort); + ourLog.info("Web Testing UI : http://localhost:{}/", myPort); + ourLog.info("Server Base URL: http://localhost:{}{}", myPort, path); + + // Never quit.. We'll let the user ctrl-C their way out. + loopForever(); + + } + + @SuppressWarnings("InfiniteLoopStatement") + private void loopForever() { + while (true) { + try { + Thread.sleep(DateUtils.MILLIS_PER_MINUTE); + } catch (InterruptedException theE) { + // ignore + } + } + } + + public static void main(String[] theArgs) { + + + Server server = new Server(22); + String path = "../hapi-fhir-cli-jpaserver"; + WebAppContext webAppContext = new WebAppContext(); + webAppContext.setContextPath("/"); + webAppContext.setDescriptor(path + "/src/main/webapp/WEB-INF/web.xml"); + webAppContext.setResourceBase(path + "/target/hapi-fhir-jpaserver-example"); + webAppContext.setParentLoaderPriority(true); + + server.setHandler(webAppContext); + try { + server.start(); + } catch (Exception e) { + e.printStackTrace(); + } + + ourLog.info("Started"); + } + + @Override + public String getCommandDescription() { + return "Start a FHIR server which can be used for testing"; + } + +} diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml index 3c58ce3131d..5c41b2faea1 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml @@ -136,6 +136,10 @@ + + org.postgresql + postgresql + diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java index 396b572ccb9..7a14ef468a1 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java @@ -35,7 +35,8 @@ import java.util.Properties; import static org.apache.commons.lang3.StringUtils.isNotBlank; @SuppressWarnings("Duplicates") -@Configuration +// TODO: Merge this with new CommonPostgreSQLConfig or find way to avoid conflicts with it. +//@Configuration public class CommonConfig { /** diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonPostgreSQLConfig.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonPostgreSQLConfig.java new file mode 100644 index 00000000000..2dc0bfb08b5 --- /dev/null +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonPostgreSQLConfig.java @@ -0,0 +1,180 @@ +package ca.uhn.fhir.jpa.demo; + +/*- + * #%L + * HAPI FHIR - Command Line Client - Server WAR + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.context.ConfigurationException; +import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.model.entity.ModelConfig; +import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory; +import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.lang3.time.DateUtils; +import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus; +import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; +import pl.allegro.tech.embeddedelasticsearch.PopularProperties; + +import javax.annotation.PreDestroy; +import javax.sql.DataSource; +import java.io.IOException; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +@SuppressWarnings("Duplicates") +@Configuration +public class CommonPostgreSQLConfig { + + static String elasticsearchHost = "localhost"; + static String elasticsearchUserId = ""; + static String elasticsearchPassword = ""; + static Integer elasticsearchPort; + + /** + * Configure FHIR properties around the the JPA server via this bean + */ + @Bean + public DaoConfig daoConfig() { + DaoConfig retVal = new DaoConfig(); + retVal.setSubscriptionEnabled(true); + retVal.setSubscriptionPollDelay(5000); + retVal.setSubscriptionPurgeInactiveAfterMillis(DateUtils.MILLIS_PER_HOUR); + retVal.setAllowMultipleDelete(true); + return retVal; + } + + @Bean + public ModelConfig modelConfig() { + return daoConfig().getModelConfig(); + } + + /** + * The following bean configures the database connection. The 'url' property value of "jdbc:postgresql://localhost:5432/hapi" indicates that the server should save resources in a + * PostgreSQL database named "hapi". + * + * A URL to a remote database could also be placed here, along with login credentials and other properties supported by BasicDataSource. + */ + @Bean(destroyMethod = "close") + public DataSource dataSource() { + String dbUrl = "jdbc:postgresql://localhost:5432/hapi"; + String dbUsername = "hapi"; + String dbPassword = "HapiFHIR"; + if (isNotBlank(ContextPostgreSQLHolder.getDatabaseUrl())) { + dbUrl = ContextPostgreSQLHolder.getDatabaseUrl(); + } + + BasicDataSource retVal = new BasicDataSource(); + retVal.setDriverClassName("org.postgresql.Driver"); + retVal.setUrl(dbUrl); + retVal.setUsername(dbUsername); + retVal.setPassword(dbPassword); + return retVal; + } + + @Bean + public Properties jpaProperties() { + + if(ContextPostgreSQLHolder.isExternalElasticsearch()) { + elasticsearchUserId = "elastic"; + elasticsearchPassword = "changeme"; + elasticsearchPort = 9301; + } else { + elasticsearchPort = embeddedElasticSearch().getHttpPort(); + } + + Properties extraProperties = new Properties(); + extraProperties.put("hibernate.dialect", org.hibernate.dialect.PostgreSQL94Dialect.class.getName()); + extraProperties.put("hibernate.format_sql", "false"); + extraProperties.put("hibernate.show_sql", "false"); + extraProperties.put("hibernate.hbm2ddl.auto", "update"); + extraProperties.put("hibernate.jdbc.batch_size", "20"); + extraProperties.put("hibernate.cache.use_query_cache", "false"); + extraProperties.put("hibernate.cache.use_second_level_cache", "false"); + extraProperties.put("hibernate.cache.use_structured_entries", "false"); + extraProperties.put("hibernate.cache.use_minimal_puts", "false"); + extraProperties.put("hibernate.search.model_mapping", LuceneSearchMappingFactory.class.getName()); + extraProperties.put("hibernate.search.default.directory_provider", "local-heap"); + extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); + extraProperties.put("hibernate.search.default.worker.execution", "sync"); + + if (System.getProperty("lowmem") != null) { + extraProperties.put("hibernate.search.autoregister_listeners", "false"); + } + + new ElasticsearchHibernatePropertiesBuilder() + .setDebugRefreshAfterWrite(true) + .setDebugPrettyPrintJsonLog(true) + .setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE) + .setIndexManagementWaitTimeoutMillis(10000) + .setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW) + .setRestUrl("http://" + elasticsearchHost + ":" + elasticsearchPort) + .setUsername(elasticsearchUserId) + .setPassword(elasticsearchPassword) + .apply(extraProperties); + +// extraProperties.setProperty("hibernate.search.default.elasticsearch.refresh_after_write", "true"); + return extraProperties; + } + + @Bean() + public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + if(ContextPostgreSQLHolder.isExternalElasticsearch()) { + elasticsearchUserId = "elastic"; + elasticsearchPassword = "changeme"; + elasticsearchPort = 9301; + } else { + elasticsearchPort = embeddedElasticSearch().getHttpPort(); + } + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + + @Bean + public EmbeddedElastic embeddedElasticSearch() { + String ELASTIC_VERSION = "6.5.4"; + + EmbeddedElastic embeddedElastic; + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } + + return embeddedElastic; + } + + @PreDestroy + public void stop() { + embeddedElasticSearch().stop(); + } + +} diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextPostgreSQLHolder.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextPostgreSQLHolder.java new file mode 100644 index 00000000000..f851057a083 --- /dev/null +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextPostgreSQLHolder.java @@ -0,0 +1,89 @@ +package ca.uhn.fhir.jpa.demo; + +/*- + * #%L + * HAPI FHIR - Command Line Client - Server WAR + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +public class ContextPostgreSQLHolder extends ContextHolder { + +// private static String myDbUsername; +// private static String myDbPassword; + private static boolean myExternalElasticsearch = false; +// private static String myElasticsearchHost; +// private static Integer myElasticsearchPort; +// private static String myElasticsearchUsername; +// private static String myElasticsearchPassword; + +/* public static void setDbUsername(String theDbUsername) { + myDbUsername = theDbUsername; + } + + public static String getDbUsername() { + return myDbUsername; + } + + public static void setDbPassword(String theDbPassword) { + myDbPassword = theDbPassword; + } + + public static String getDbPassword() { + return myDbPassword; + } +*/ + public static void setExternalElasticsearch(Boolean theExternalElasticsearch) { + myExternalElasticsearch = theExternalElasticsearch; + } + + public static Boolean isExternalElasticsearch() { + return myExternalElasticsearch; + } +/* + public static void setElasticsearchHost(String theElasticsearchHost) { + myElasticsearchHost = theElasticsearchHost; + } + + public static String getElasticsearchHost() { + return myElasticsearchHost; + } + + public static void setElasticsearchPort(Integer theElasticsearchPort) { + myElasticsearchPort = theElasticsearchPort; + } + + public static Integer getElasticsearchPort() { + return myElasticsearchPort; + } + + public static void setElasticsearchUsername(String theElasticsearchUsername) { + myElasticsearchUsername = theElasticsearchUsername; + } + + public static String getElasticsearchUsername() { + return myElasticsearchUsername; + } + + public static void setElasticsearchPassword(String theElasticsearchPassword) { + myElasticsearchPassword = theElasticsearchPassword; + } + + public static String getElasticsearchPassword() { + return myElasticsearchPassword; + } +*/ +} diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerElasticsearchConfigR4.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerElasticsearchConfigR4.java new file mode 100644 index 00000000000..59534483dc8 --- /dev/null +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerElasticsearchConfigR4.java @@ -0,0 +1,107 @@ +package ca.uhn.fhir.jpa.demo; + +/*- + * #%L + * HAPI FHIR - Command Line Client - Server WAR + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.config.BaseJavaConfigR4; +import ca.uhn.fhir.jpa.util.SubscriptionsRequireManualActivationInterceptorR4; +import ca.uhn.fhir.rest.server.interceptor.IServerInterceptor; +import ca.uhn.fhir.rest.server.interceptor.LoggingInterceptor; +import ca.uhn.fhir.rest.server.interceptor.ResponseHighlighterInterceptor; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.persistence.EntityManagerFactory; +import javax.sql.DataSource; +import java.util.Properties; + +/** + * This class isn't used by default by the example, but + * you can use it as a config if you want to support DSTU3 + * instead of DSTU2 in your server. + *

+ * See https://github.com/jamesagnew/hapi-fhir/issues/278 + */ +@Configuration +@EnableTransactionManagement() +@Import(CommonPostgreSQLConfig.class) +public class FhirServerElasticsearchConfigR4 extends BaseJavaConfigR4 { + + @Autowired + private DataSource myDataSource; + @Autowired() + @Qualifier("jpaProperties") + private Properties myJpaProperties; + + @Override + @Bean + public LocalContainerEntityManagerFactoryBean entityManagerFactory() { + LocalContainerEntityManagerFactoryBean retVal = super.entityManagerFactory(); + retVal.setPersistenceUnitName("HAPI_PU"); + retVal.setDataSource(myDataSource); + retVal.setJpaProperties(myJpaProperties); + return retVal; + } + + /** + * Do some fancy logging to create a nice access log that has details about each incoming request. + * @return + */ + public LoggingInterceptor loggingInterceptor() { + LoggingInterceptor retVal = new LoggingInterceptor(); + retVal.setLoggerName("fhirtest.access"); + retVal.setMessageFormat( + "Path[${servletPath}] Source[${requestHeader.x-forwarded-for}] Operation[${operationType} ${operationName} ${idOrResourceName}] UA[${requestHeader.user-agent}] Params[${requestParameters}] ResponseEncoding[${responseEncodingNoDefault}]"); + retVal.setLogExceptions(true); + retVal.setErrorMessageFormat("ERROR - ${requestVerb} ${requestUrl}"); + return retVal; + } + + /** + * This interceptor adds some pretty syntax highlighting in responses when a browser is detected + * @return + */ + @Bean(autowire = Autowire.BY_TYPE) + public ResponseHighlighterInterceptor responseHighlighterInterceptor() { + ResponseHighlighterInterceptor retVal = new ResponseHighlighterInterceptor(); + return retVal; + } + + @Bean(autowire = Autowire.BY_TYPE) + public IServerInterceptor subscriptionSecurityInterceptor() { + SubscriptionsRequireManualActivationInterceptorR4 retVal = new SubscriptionsRequireManualActivationInterceptorR4(); + return retVal; + } + + @Bean + public JpaTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { + JpaTransactionManager retVal = new JpaTransactionManager(); + retVal.setEntityManagerFactory(entityManagerFactory); + return retVal; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index c5d901f3c2d..5318c8cbaa2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -285,6 +285,7 @@ public abstract class BaseConfig { public static void configureEntityManagerFactory(LocalContainerEntityManagerFactoryBean theFactory, FhirContext theCtx) { theFactory.setJpaDialect(hibernateJpaDialect(theCtx.getLocalizer())); + //TODO: Consider moving the jpa.dao.lastn.entity.* classes into jpa.entity at some point. theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.dao.lastn.entity"); theFactory.setPersistenceProvider(new HibernatePersistenceProvider()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java index 9e9ad110a38..23b3c238bfd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java @@ -8,6 +8,7 @@ import ca.uhn.fhir.jpa.dao.IFhirSystemDao; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.dao.TransactionProcessor; import ca.uhn.fhir.jpa.dao.dstu3.TransactionProcessorVersionAdapterDstu3; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistDstu3Svc; import ca.uhn.fhir.jpa.provider.GraphQLProvider; import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorDstu3; import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl; @@ -17,10 +18,7 @@ import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc; import ca.uhn.fhir.jpa.term.api.ITermReadSvcDstu3; import ca.uhn.fhir.jpa.term.api.ITermVersionAdapterSvc; import ca.uhn.fhir.jpa.util.ResourceCountCache; -import ca.uhn.fhir.validation.IInstanceValidatorModule; import org.apache.commons.lang3.time.DateUtils; -import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator; -import org.hl7.fhir.r5.utils.IResourceValidator; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Lazy; @@ -131,4 +129,9 @@ public class BaseDstu3Config extends BaseConfigDstu3Plus { return new TermReadSvcDstu3(); } + @Bean + public ObservationLastNIndexPersistDstu3Svc observationLastNIndexpersistSvc() { + return new ObservationLastNIndexPersistDstu3Svc(); + } + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java index b26bb49cd2d..2709e74cc02 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java @@ -7,7 +7,7 @@ import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl; import ca.uhn.fhir.jpa.dao.IFhirSystemDao; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.dao.TransactionProcessor; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR4Svc; import ca.uhn.fhir.jpa.dao.r4.TransactionProcessorVersionAdapterR4; import ca.uhn.fhir.jpa.provider.GraphQLProvider; import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorR4; @@ -18,10 +18,7 @@ import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc; import ca.uhn.fhir.jpa.term.api.ITermReadSvcR4; import ca.uhn.fhir.jpa.term.api.ITermVersionAdapterSvc; import ca.uhn.fhir.jpa.util.ResourceCountCache; -import ca.uhn.fhir.validation.IInstanceValidatorModule; import org.apache.commons.lang3.time.DateUtils; -import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator; -import org.hl7.fhir.r5.utils.IResourceValidator; import org.springframework.beans.factory.annotation.Autowire; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -136,10 +133,8 @@ public class BaseR4Config extends BaseConfigDstu3Plus { } @Bean - public ObservationLastNIndexPersistSvc observationLastNIndexpersistSvc() { - return new ObservationLastNIndexPersistSvc(); + public ObservationLastNIndexPersistR4Svc observationLastNIndexpersistSvc() { + return new ObservationLastNIndexPersistR4Svc(); } - - } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java index eaffaced9d8..dedf8dc2377 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java @@ -7,6 +7,7 @@ import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl; import ca.uhn.fhir.jpa.dao.IFhirSystemDao; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.dao.TransactionProcessor; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR5Svc; import ca.uhn.fhir.jpa.dao.r5.TransactionProcessorVersionAdapterR5; import ca.uhn.fhir.jpa.provider.GraphQLProvider; import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorR5; @@ -17,11 +18,8 @@ import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc; import ca.uhn.fhir.jpa.term.api.ITermReadSvcR5; import ca.uhn.fhir.jpa.term.api.ITermVersionAdapterSvc; import ca.uhn.fhir.jpa.util.ResourceCountCache; -import ca.uhn.fhir.validation.IInstanceValidatorModule; import org.apache.commons.lang3.time.DateUtils; -import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator; import org.hl7.fhir.r5.model.Bundle; -import org.hl7.fhir.r5.utils.IResourceValidator; import org.springframework.beans.factory.annotation.Autowire; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -135,4 +133,9 @@ public class BaseR5Config extends BaseConfigDstu3Plus { return new TermReadSvcR5(); } + @Bean + public ObservationLastNIndexPersistR5Svc observationLastNIndexpersistSvc() { + return new ObservationLastNIndexPersistR5Svc(); + } + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFhirResourceDaoObservation.java new file mode 100644 index 00000000000..fd38cd2e807 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFhirResourceDaoObservation.java @@ -0,0 +1,34 @@ +package ca.uhn.fhir.jpa.dao; + +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import org.hl7.fhir.instance.model.api.IBaseResource; + +import javax.servlet.http.HttpServletResponse; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +public interface IFhirResourceDaoObservation extends IFhirResourceDao { + + IBundleProvider observationsLastN(SearchParameterMap paramMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse); + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 891296fba86..44c898dc44d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -45,6 +45,7 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.entity.ResourceTag; import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; +import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc; import ca.uhn.fhir.jpa.searchparam.JpaRuntimeSearchParam; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry; @@ -110,6 +111,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.defaultString; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -152,6 +154,8 @@ public class SearchBuilder implements ISearchBuilder { private IdHelperService myIdHelperService; @Autowired(required = false) private IFulltextSearchSvc myFulltextSearchSvc; + @Autowired(required = false) + private IElasticsearchSvc myIElasticsearchSvc; @Autowired private ISearchParamRegistry mySearchParamRegistry; @Autowired @@ -314,22 +318,51 @@ public class SearchBuilder implements ISearchBuilder { } /* - * Fulltext search + * Fulltext search and lastn */ - if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { - if (myFulltextSearchSvc == null) { - if (myParams.containsKey(Constants.PARAM_TEXT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); - } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { + List lastnPids = new ArrayList<>(); + List fullTextSearchPids = new ArrayList<>(); + + if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { + if (myFulltextSearchSvc == null) { + if (myParams.containsKey(Constants.PARAM_TEXT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + } + } + + if (myParams.getEverythingMode() != null) { + fullTextSearchPids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); + } else { + fullTextSearchPids = myFulltextSearchSvc.search(myResourceName, myParams); + } + } else { + if (myIElasticsearchSvc == null) { + if (myParams.isLastN()) { + throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); + } + } + + if (myParams.isLastN()) { + lastnPids = myIElasticsearchSvc.executeLastN(myParams, theRequest, myIdHelperService); } } + // List pids; - if (myParams.getEverythingMode() != null) { - pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); + if (fullTextSearchPids.isEmpty()) { + pids = lastnPids; + } else if (lastnPids.isEmpty()) { + pids = fullTextSearchPids; } else { - pids = myFulltextSearchSvc.search(myResourceName, myParams); + // Intersection of the fullTextSearchPids and lastnPids + Set pidIntersection = fullTextSearchPids.stream() + .distinct() + .filter(lastnPids::contains) + .collect(Collectors.toSet()); + pids = new ArrayList<>(pidIntersection); } if (pids.isEmpty()) { // Will never match diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java index ee3ecde5d84..2bde707c764 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java @@ -2,8 +2,16 @@ package ca.uhn.fhir.jpa.dao.data; import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @Repository public interface IObservationIndexedCodeCodeableConceptSearchParamDao extends JpaRepository { + @Query("" + + "SELECT t FROM ObservationIndexedCodeCodeableConceptEntity t " + + "WHERE t.myCodeableConceptId = :codeableConceptId" + + "") + ObservationIndexedCodeCodeableConceptEntity findByCodeableConceptId(@Param("codeableConceptId") String theCodeableConceptId); + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java new file mode 100644 index 00000000000..3cb0bb1f2f0 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java @@ -0,0 +1,113 @@ +package ca.uhn.fhir.jpa.dao.dstu3; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistDstu3Svc; +import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import ca.uhn.fhir.jpa.model.entity.ResourceLink; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; +import ca.uhn.fhir.model.api.IResource; +import ca.uhn.fhir.rest.api.CacheControlDirective; +import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.param.DateRangeParam; +import ca.uhn.fhir.rest.param.StringAndListParam; +import ca.uhn.fhir.rest.param.StringParam; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.hl7.fhir.dstu3.model.Observation; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; + +public class FhirResourceDaoObservationDstu3 extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { + + @Autowired + ObservationLastNIndexPersistDstu3Svc myObservationLastNIndexPersistDstu3Svc; + + private IBundleProvider doLastNOperation(IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequest) { + SearchParameterMap paramMap = new SearchParameterMap(); + if (theCount != null) { + paramMap.setCount(theCount.getValue()); + } + if (theContent != null) { + paramMap.add(Constants.PARAM_CONTENT, theContent); + } + if (theNarrative != null) { + paramMap.add(Constants.PARAM_TEXT, theNarrative); + } + paramMap.setIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive())); + paramMap.setEverythingMode(theId != null ? EverythingModeEnum.PATIENT_INSTANCE : EverythingModeEnum.PATIENT_TYPE); + paramMap.setSort(theSort); + paramMap.setLastUpdated(theLastUpdated); + if (theId != null) { + paramMap.add("_id", new StringParam(theId.getIdPart())); + } + + if (!isPagingProviderDatabaseBacked(theRequest)) { + paramMap.setLoadSynchronous(true); + } + + return mySearchCoordinatorSvc.registerSearch(this, paramMap, getResourceName(), new CacheControlDirective().parse(theRequest.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequest); + } + + @Override + public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse) { + return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); + } + + @Override + public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, + boolean theUpdateVersion, Date theUpdateTime, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { + ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theUpdateTime, theForceUpdate, theCreateNewHistoryEntry); + + if(thePerformIndexing) { + // Update indexes here for LastN operation. + Observation observation = (Observation)theResource; + Collection myResourceLinks = retVal.getResourceLinks(); + Long subjectID = null; + for (ResourceLink resourceLink : myResourceLinks) { + if(resourceLink.getSourcePath().equals("Observation.subject")) { + subjectID = resourceLink.getTargetResourcePid(); + } + } + if (subjectID != null) { + myObservationLastNIndexPersistDstu3Svc.indexObservation(observation, subjectID.toString()); + } else { + myObservationLastNIndexPersistDstu3Svc.indexObservation(observation); + } + } + + return retVal; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java new file mode 100644 index 00000000000..66ccd325db4 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java @@ -0,0 +1,87 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import org.hl7.fhir.dstu3.model.CodeableConcept; +import org.hl7.fhir.dstu3.model.Coding; +import org.hl7.fhir.dstu3.model.Observation; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Date; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class ObservationLastNIndexPersistDstu3Svc { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myObservationIndexedCodeableConceptSearchParamDao; + + @Autowired + IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; + + // TODO: Change theSubjectId to be a Long + public void indexObservation(Observation theObservation, String theSubjectId) { + ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + String resourcePID = theObservation.getIdElement().getIdPart(); + indexedObservation.setIdentifier(resourcePID); + indexedObservation.setSubject(theSubjectId); + Date effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); + indexedObservation.setEffectiveDtm(effectiveDtm); + + // Build CodeableConcept entities for Observation.Category + Set categoryConcepts = new HashSet<>(); + for(CodeableConcept categoryCodeableConcept : theObservation.getCategory()) { + // Build Coding entities for each category CodeableConcept + Set categoryCodingEntities = new HashSet<>(); + ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConceptEntity = new ObservationIndexedCategoryCodeableConceptEntity(categoryCodeableConcept.getText()); + for(Coding categoryCoding : categoryCodeableConcept.getCoding()){ + categoryCodingEntities.add(new ObservationIndexedCategoryCodingEntity(categoryCoding.getSystem(), categoryCoding.getCode(), categoryCoding.getDisplay())); + } + categoryCodeableConceptEntity.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); + categoryConcepts.add(categoryCodeableConceptEntity); + } + indexedObservation.setCategoryCodeableConcepts(categoryConcepts); + + // Build CodeableConcept entity for Observation.Code. + CodeableConcept codeCodeableConcept = theObservation.getCode(); + String observationCodeNormalizedId = null; + + // Determine if a Normalized ID was created previously for Observation Code + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + if (codeCoding.hasCode() && codeCoding.hasSystem()) { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); + } else { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); + } + } + // Generate a new a normalized ID if necessary + if (observationCodeNormalizedId == null) { + observationCodeNormalizedId = UUID.randomUUID().toString(); + } + + // Create/update normalized Observation Code index record + ObservationIndexedCodeCodeableConceptEntity codeableConceptField = new ObservationIndexedCodeCodeableConceptEntity(codeCodeableConcept.getText(), observationCodeNormalizedId); + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); + } + myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); + + indexedObservation.setObservationCode(codeableConceptField); + indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); + myResourceIndexedObservationLastNDao.save(indexedObservation); + + } + + // TODO: Remove this once Unit tests are updated. + public void indexObservation(Observation theObservation) { + String subjectId = "Patient/" + theObservation.getSubject().getReference(); + + indexObservation(theObservation, subjectId); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java new file mode 100644 index 00000000000..09f87f8e9b3 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java @@ -0,0 +1,88 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Observation; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Date; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class ObservationLastNIndexPersistR4Svc { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myObservationIndexedCodeableConceptSearchParamDao; + + @Autowired + IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; + + // TODO: Change theSubjectId to be a Long + public void indexObservation(Observation theObservation, String theSubjectId) { + ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + String resourcePID = theObservation.getIdElement().getIdPart(); + indexedObservation.setIdentifier(resourcePID); + indexedObservation.setSubject(theSubjectId); + Date effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); + indexedObservation.setEffectiveDtm(effectiveDtm); + + // Build CodeableConcept entities for Observation.Category + Set categoryConcepts = new HashSet<>(); + for(CodeableConcept categoryCodeableConcept : theObservation.getCategory()) { + // Build Coding entities for each category CodeableConcept + Set categoryCodingEntities = new HashSet<>(); + ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConceptEntity = new ObservationIndexedCategoryCodeableConceptEntity(categoryCodeableConcept.getText()); + for(Coding categoryCoding : categoryCodeableConcept.getCoding()){ + categoryCodingEntities.add(new ObservationIndexedCategoryCodingEntity(categoryCoding.getSystem(), categoryCoding.getCode(), categoryCoding.getDisplay())); + } + categoryCodeableConceptEntity.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); + categoryConcepts.add(categoryCodeableConceptEntity); + } + indexedObservation.setCategoryCodeableConcepts(categoryConcepts); + + // Build CodeableConcept entity for Observation.Code. + CodeableConcept codeCodeableConcept = theObservation.getCode(); + String observationCodeNormalizedId = null; + + // Determine if a Normalized ID was created previously for Observation Code + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + if (codeCoding.hasCode() && codeCoding.hasSystem()) { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); + } else { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); + } + } + // Generate a new a normalized ID if necessary + if (observationCodeNormalizedId == null) { + observationCodeNormalizedId = UUID.randomUUID().toString(); + } + + // Create/update normalized Observation Code index record + ObservationIndexedCodeCodeableConceptEntity codeableConceptField = new ObservationIndexedCodeCodeableConceptEntity(codeCodeableConcept.getText(), observationCodeNormalizedId); + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); + } + myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); + codeableConceptField = myObservationIndexedCodeableConceptSearchParamDao.findByCodeableConceptId(observationCodeNormalizedId); + + indexedObservation.setObservationCode(codeableConceptField); + indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); + myResourceIndexedObservationLastNDao.save(indexedObservation); + + } + + // TODO: Remove this once Unit tests are updated. + public void indexObservation(Observation theObservation) { + String subjectId = "Patient/" + theObservation.getSubject().getReference(); + + indexObservation(theObservation, subjectId); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java new file mode 100644 index 00000000000..e37d7a154b2 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java @@ -0,0 +1,87 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import org.hl7.fhir.r5.model.CodeableConcept; +import org.hl7.fhir.r5.model.Coding; +import org.hl7.fhir.r5.model.Observation; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Date; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class ObservationLastNIndexPersistR5Svc { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myObservationIndexedCodeableConceptSearchParamDao; + + @Autowired + IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; + + // TODO: Change theSubjectId to be a Long + public void indexObservation(Observation theObservation, String theSubjectId) { + ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + String resourcePID = theObservation.getIdElement().getIdPart(); + indexedObservation.setIdentifier(resourcePID); + indexedObservation.setSubject(theSubjectId); + Date effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); + indexedObservation.setEffectiveDtm(effectiveDtm); + + // Build CodeableConcept entities for Observation.Category + Set categoryConcepts = new HashSet<>(); + for(CodeableConcept categoryCodeableConcept : theObservation.getCategory()) { + // Build Coding entities for each category CodeableConcept + Set categoryCodingEntities = new HashSet<>(); + ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConceptEntity = new ObservationIndexedCategoryCodeableConceptEntity(categoryCodeableConcept.getText()); + for(Coding categoryCoding : categoryCodeableConcept.getCoding()){ + categoryCodingEntities.add(new ObservationIndexedCategoryCodingEntity(categoryCoding.getSystem(), categoryCoding.getCode(), categoryCoding.getDisplay())); + } + categoryCodeableConceptEntity.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); + categoryConcepts.add(categoryCodeableConceptEntity); + } + indexedObservation.setCategoryCodeableConcepts(categoryConcepts); + + // Build CodeableConcept entity for Observation.Code. + CodeableConcept codeCodeableConcept = theObservation.getCode(); + String observationCodeNormalizedId = null; + + // Determine if a Normalized ID was created previously for Observation Code + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + if (codeCoding.hasCode() && codeCoding.hasSystem()) { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); + } else { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); + } + } + // Generate a new a normalized ID if necessary + if (observationCodeNormalizedId == null) { + observationCodeNormalizedId = UUID.randomUUID().toString(); + } + + // Create/update normalized Observation Code index record + ObservationIndexedCodeCodeableConceptEntity codeableConceptField = new ObservationIndexedCodeCodeableConceptEntity(codeCodeableConcept.getText(), observationCodeNormalizedId); + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); + } + myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); + + indexedObservation.setObservationCode(codeableConceptField); + indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); + myResourceIndexedObservationLastNDao.save(indexedObservation); + + } + + // TODO: Remove this once Unit tests are updated. + public void indexObservation(Observation theObservation) { + String subjectId = "Patient/" + theObservation.getSubject().getReference(); + + indexObservation(theObservation, subjectId); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java index 69caae6ff39..3a4a2c4ab07 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Component; import java.util.*; -@Component +//@Component public class ObservationLastNIndexPersistSvc { @Autowired diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java index 6f9b559a476..cfe95e2f93d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java @@ -25,8 +25,11 @@ public class ObservationIndexedCodeCodeableConceptEntity { private String myCodeableConceptText; @IndexedEmbedded(depth=2, prefix = "coding") - @OneToMany(mappedBy = "myCodeableConceptId", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) - private Set myObservationIndexedCodeCodingEntitySet; +// @OneToMany(mappedBy = "myCodeableConceptId", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) + @JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_CONCEPT_CODE")) + @OneToOne(cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) +// private Set myObservationIndexedCodeCodingEntitySet; + private ObservationIndexedCodeCodingEntity myObservationIndexedCodeCodingEntity; public ObservationIndexedCodeCodeableConceptEntity() { @@ -38,10 +41,11 @@ public class ObservationIndexedCodeCodeableConceptEntity { } public void addCoding(ObservationIndexedCodeCodingEntity theObservationIndexedCodeCodingEntity) { - if (myObservationIndexedCodeCodingEntitySet == null) { - myObservationIndexedCodeCodingEntitySet = new HashSet<>(); - } - myObservationIndexedCodeCodingEntitySet.add(theObservationIndexedCodeCodingEntity); +// if (myObservationIndexedCodeCodingEntitySet == null) { +// myObservationIndexedCodeCodingEntitySet = new HashSet<>(); +// } +// myObservationIndexedCodeCodingEntitySet.add(theObservationIndexedCodeCodingEntity); + myObservationIndexedCodeCodingEntity = theObservationIndexedCodeCodingEntity; } public String getCodeableConceptId() { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java index 69183faf393..ecdbfd6e7e3 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java @@ -11,11 +11,13 @@ import javax.persistence.*; @Table(name = "HFJ_SPIDX_LASTN_CODING") public class ObservationIndexedCodeCodingEntity { - @Id - @SequenceGenerator(name = "SEQ_CODING_FIELD", sequenceName = "SEQ_CODING_FIELD") - @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CODING_FIELD") - private Long myId; + // TODO: Fix this to allow multiple codings for observation code +// @Id +// @SequenceGenerator(name = "SEQ_CODING_FIELD", sequenceName = "SEQ_CODING_FIELD") +// @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CODING_FIELD") +// private Long myId; + @Id @Column(name="CODEABLE_CONCEPT_ID") private String myCodeableConceptId; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index 91bc3679bf7..ebccb7a7696 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -21,7 +21,11 @@ package ca.uhn.fhir.jpa.dao.r4; */ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoPatient; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR4Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import ca.uhn.fhir.jpa.model.entity.ResourceLink; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; @@ -38,42 +42,26 @@ import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Patient; +import org.springframework.beans.factory.annotation.Autowired; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Collection; import java.util.Collections; import java.util.Date; -public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDao { +public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { - private IBundleProvider doLastNOperation(IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequest) { - SearchParameterMap paramMap = new SearchParameterMap(); - if (theCount != null) { - paramMap.setCount(theCount.getValue()); - } - if (theContent != null) { - paramMap.add(Constants.PARAM_CONTENT, theContent); - } - if (theNarrative != null) { - paramMap.add(Constants.PARAM_TEXT, theNarrative); - } - paramMap.setIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive())); - paramMap.setEverythingMode(theId != null ? EverythingModeEnum.PATIENT_INSTANCE : EverythingModeEnum.PATIENT_TYPE); - paramMap.setSort(theSort); - paramMap.setLastUpdated(theLastUpdated); - if (theId != null) { - paramMap.add("_id", new StringParam(theId.getIdPart())); - } - - if (!isPagingProviderDatabaseBacked(theRequest)) { - paramMap.setLoadSynchronous(true); - } - - return mySearchCoordinatorSvc.registerSearch(this, paramMap, getResourceName(), new CacheControlDirective().parse(theRequest.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequest); + @Autowired + ObservationLastNIndexPersistR4Svc myObservationLastNIndexPersistR4Svc; + + @Override + public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse) { + return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); } - public IBundleProvider observationsLastN(HttpServletRequest theServletRequest, IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequestDetails) { - return doLastNOperation(theId, theCount, theLastUpdated, theSort, theContent, theNarrative, theFilter, theRequestDetails); - } + @Override public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, @@ -83,10 +71,21 @@ public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDao myResourceLinks = retVal.getResourceLinks(); + Long subjectID = null; + for (ResourceLink resourceLink : myResourceLinks) { + if(resourceLink.getSourcePath().equals("Observation.subject")) { + subjectID = resourceLink.getTargetResourcePid(); + } + } + if (subjectID != null) { + myObservationLastNIndexPersistR4Svc.indexObservation(observation, subjectID.toString()); + } else { + myObservationLastNIndexPersistR4Svc.indexObservation(observation); + } } return retVal; } - } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java new file mode 100644 index 00000000000..665f425ea20 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java @@ -0,0 +1,113 @@ +package ca.uhn.fhir.jpa.dao.r5; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR5Svc; +import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import ca.uhn.fhir.jpa.model.entity.ResourceLink; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; +import ca.uhn.fhir.model.api.IResource; +import ca.uhn.fhir.rest.api.CacheControlDirective; +import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.param.DateRangeParam; +import ca.uhn.fhir.rest.param.StringAndListParam; +import ca.uhn.fhir.rest.param.StringParam; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.hl7.fhir.r5.model.Observation; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; + +public class FhirResourceDaoObservationR5 extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { + + @Autowired + ObservationLastNIndexPersistR5Svc myObservationLastNIndexPersistR5Svc; + + private IBundleProvider doLastNOperation(IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequest) { + SearchParameterMap paramMap = new SearchParameterMap(); + if (theCount != null) { + paramMap.setCount(theCount.getValue()); + } + if (theContent != null) { + paramMap.add(Constants.PARAM_CONTENT, theContent); + } + if (theNarrative != null) { + paramMap.add(Constants.PARAM_TEXT, theNarrative); + } + paramMap.setIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive())); + paramMap.setEverythingMode(theId != null ? EverythingModeEnum.PATIENT_INSTANCE : EverythingModeEnum.PATIENT_TYPE); + paramMap.setSort(theSort); + paramMap.setLastUpdated(theLastUpdated); + if (theId != null) { + paramMap.add("_id", new StringParam(theId.getIdPart())); + } + + if (!isPagingProviderDatabaseBacked(theRequest)) { + paramMap.setLoadSynchronous(true); + } + + return mySearchCoordinatorSvc.registerSearch(this, paramMap, getResourceName(), new CacheControlDirective().parse(theRequest.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequest); + } + + @Override + public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse) { + return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); + } + + @Override + public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, + boolean theUpdateVersion, Date theUpdateTime, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { + ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theUpdateTime, theForceUpdate, theCreateNewHistoryEntry); + + if(thePerformIndexing) { + // Update indexes here for LastN operation. + Observation observation = (Observation)theResource; + Collection myResourceLinks = retVal.getResourceLinks(); + Long subjectID = null; + for (ResourceLink resourceLink : myResourceLinks) { + if(resourceLink.getSourcePath().equals("Observation.subject")) { + subjectID = resourceLink.getTargetResourcePid(); + } + } + if (subjectID != null) { + myObservationLastNIndexPersistR5Svc.indexObservation(observation, subjectID.toString()); + } else { + myObservationLastNIndexPersistR5Svc.indexObservation(observation); + } + } + + return retVal; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java new file mode 100644 index 00000000000..ff9956b2c32 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java @@ -0,0 +1,148 @@ +package ca.uhn.fhir.jpa.provider; + +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.model.util.JpaConstants; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.api.Include; +import ca.uhn.fhir.model.api.annotation.Description; +import ca.uhn.fhir.model.dstu2.resource.Observation; +import ca.uhn.fhir.model.valueset.BundleTypeEnum; +import ca.uhn.fhir.rest.annotation.*; +import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.api.SummaryEnum; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.param.*; + +import java.util.Set; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +public class BaseJpaResourceProviderObservationDstu2 extends JpaResourceProviderDstu2 { + + /** + * Observation/$lastn + */ + @Operation(name = JpaConstants.OPERATION_LASTN, idempotent = true, bundleType = BundleTypeEnum.SEARCHSET) + public IBundleProvider observationLastN( + + javax.servlet.http.HttpServletRequest theServletRequest, + javax.servlet.http.HttpServletResponse theServletResponse, + + ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails, + + @Description(shortDefinition="Search the contents of the resource's data using a filter") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_FILTER) + StringAndListParam theFtFilter, + + @Description(shortDefinition="Search the contents of the resource's data using a fulltext search") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT) + StringAndListParam theFtContent, + + @Description(shortDefinition="Search the contents of the resource's narrative using a fulltext search") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_TEXT) + StringAndListParam theFtText, + + @Description(shortDefinition="The classification of the type of observation") + @OperationParam(name="category") + TokenAndListParam theCategory, + + @Description(shortDefinition="The code of the observation type") + @OperationParam(name="code") + TokenAndListParam theCode, + + @Description(shortDefinition="Obtained date/time. If the obtained element is a period, a date that falls in the period") + @OperationParam(name="date") + DateRangeParam theDate, + + @Description(shortDefinition="The maximum number of observations to return for each each observation code") + @OperationParam(name="max", max=1, min=0) + NumberParam theMax, + + @Description(shortDefinition="The subject that the observation is about (if patient)") + @OperationParam(name="patient") + ReferenceAndListParam thePatient, + + @Description(shortDefinition="The subject that the observation is about") + @OperationParam(name="subject" ) + ReferenceAndListParam theSubject, + + @IncludeParam(reverse=true) + Set theRevIncludes, + @Description(shortDefinition="Only return resources which were last updated as specified by the given range") + @OperationParam(name="_lastUpdated") + DateRangeParam theLastUpdated, + + @IncludeParam(allow= { + "Observation:based-on", + "Observation:derived-from", + "Observation:device", + "Observation:encounter", + "Observation:focus", + "Observation:has-member", + "Observation:part-of", + "Observation:patient", + "Observation:performer", + "Observation:specimen", + "Observation:subject", + "*" + }) + Set theIncludes, + + @Sort + SortSpec theSort, + + @ca.uhn.fhir.rest.annotation.Count + Integer theCount, + + SummaryEnum theSummaryMode, + + SearchTotalModeEnum theSearchTotalMode + + ) { + startRequest(theServletRequest); + try { + SearchParameterMap paramMap = new SearchParameterMap(); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_FILTER, theFtFilter); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, theFtContent); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_TEXT, theFtText); + paramMap.add("category", theCategory); + paramMap.add("code", theCode); + paramMap.add("date", theDate); + paramMap.add("max", theMax); + paramMap.add("patient", thePatient); + paramMap.add("subject", theSubject); + paramMap.setRevIncludes(theRevIncludes); + paramMap.setLastUpdated(theLastUpdated); + paramMap.setIncludes(theIncludes); + paramMap.setLastN(true); + paramMap.setSort(theSort); + paramMap.setCount(theCount); + paramMap.setSummaryMode(theSummaryMode); + paramMap.setSearchTotalMode(theSearchTotalMode); + + return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); + } finally { + endRequest(theServletRequest); + } + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java new file mode 100644 index 00000000000..eb852776af4 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java @@ -0,0 +1,148 @@ +package ca.uhn.fhir.jpa.provider.dstu3; + +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.model.util.JpaConstants; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.api.Include; +import ca.uhn.fhir.model.api.annotation.Description; +import ca.uhn.fhir.model.valueset.BundleTypeEnum; +import ca.uhn.fhir.rest.annotation.*; +import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.api.SummaryEnum; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.param.*; +import org.hl7.fhir.dstu3.model.Observation; + +import java.util.Set; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProviderDstu3 { + + /** + * Observation/$lastn + */ + @Operation(name = JpaConstants.OPERATION_LASTN, idempotent = true, bundleType = BundleTypeEnum.SEARCHSET) + public IBundleProvider observationLastN( + + javax.servlet.http.HttpServletRequest theServletRequest, + javax.servlet.http.HttpServletResponse theServletResponse, + + ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails, + + @Description(shortDefinition="Search the contents of the resource's data using a filter") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_FILTER) + StringAndListParam theFtFilter, + + @Description(shortDefinition="Search the contents of the resource's data using a fulltext search") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT) + StringAndListParam theFtContent, + + @Description(shortDefinition="Search the contents of the resource's narrative using a fulltext search") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_TEXT) + StringAndListParam theFtText, + + @Description(shortDefinition="The classification of the type of observation") + @OperationParam(name="category") + TokenAndListParam theCategory, + + @Description(shortDefinition="The code of the observation type") + @OperationParam(name="code") + TokenAndListParam theCode, + + @Description(shortDefinition="Obtained date/time. If the obtained element is a period, a date that falls in the period") + @OperationParam(name="date") + DateRangeParam theDate, + + @Description(shortDefinition="The maximum number of observations to return for each each observation code") + @OperationParam(name="max", max=1, min=0) + NumberParam theMax, + + @Description(shortDefinition="The subject that the observation is about (if patient)") + @OperationParam(name="patient") + ReferenceAndListParam thePatient, + + @Description(shortDefinition="The subject that the observation is about") + @OperationParam(name="subject" ) + ReferenceAndListParam theSubject, + + @IncludeParam(reverse=true) + Set theRevIncludes, + @Description(shortDefinition="Only return resources which were last updated as specified by the given range") + @OperationParam(name="_lastUpdated") + DateRangeParam theLastUpdated, + + @IncludeParam(allow= { + "Observation:based-on", + "Observation:derived-from", + "Observation:device", + "Observation:encounter", + "Observation:focus", + "Observation:has-member", + "Observation:part-of", + "Observation:patient", + "Observation:performer", + "Observation:specimen", + "Observation:subject", + "*" + }) + Set theIncludes, + + @Sort + SortSpec theSort, + + @ca.uhn.fhir.rest.annotation.Count + Integer theCount, + + SummaryEnum theSummaryMode, + + SearchTotalModeEnum theSearchTotalMode + + ) { + startRequest(theServletRequest); + try { + SearchParameterMap paramMap = new SearchParameterMap(); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_FILTER, theFtFilter); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, theFtContent); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_TEXT, theFtText); + paramMap.add("category", theCategory); + paramMap.add("code", theCode); + paramMap.add("date", theDate); + paramMap.add("max", theMax); + paramMap.add("patient", thePatient); + paramMap.add("subject", theSubject); + paramMap.setRevIncludes(theRevIncludes); + paramMap.setLastUpdated(theLastUpdated); + paramMap.setIncludes(theIncludes); + paramMap.setLastN(true); + paramMap.setSort(theSort); + paramMap.setCount(theCount); + paramMap.setSummaryMode(theSummaryMode); + paramMap.setSearchTotalMode(theSearchTotalMode); + + return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); + } finally { + endRequest(theServletRequest); + } + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java new file mode 100644 index 00000000000..d8fb025c1be --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java @@ -0,0 +1,149 @@ +package ca.uhn.fhir.jpa.provider.r4; + +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.model.util.JpaConstants; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.api.Include; +import ca.uhn.fhir.model.api.annotation.Description; +import ca.uhn.fhir.model.valueset.BundleTypeEnum; +import ca.uhn.fhir.rest.annotation.*; +import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.api.SummaryEnum; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.param.*; +import org.hl7.fhir.r4.model.*; + +import java.util.Set; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4 { + + /** + * Observation/$lastn + */ + @Operation(name = JpaConstants.OPERATION_LASTN, idempotent = true, bundleType = BundleTypeEnum.SEARCHSET) + public IBundleProvider observationLastN( + + javax.servlet.http.HttpServletRequest theServletRequest, + javax.servlet.http.HttpServletResponse theServletResponse, + + ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails, + + @Description(shortDefinition="Search the contents of the resource's data using a filter") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_FILTER) + StringAndListParam theFtFilter, + + @Description(shortDefinition="Search the contents of the resource's data using a fulltext search") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT) + StringAndListParam theFtContent, + + @Description(shortDefinition="Search the contents of the resource's narrative using a fulltext search") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_TEXT) + StringAndListParam theFtText, + + @Description(shortDefinition="The classification of the type of observation") + @OperationParam(name="category") + TokenAndListParam theCategory, + + @Description(shortDefinition="The code of the observation type") + @OperationParam(name="code") + TokenAndListParam theCode, + + @Description(shortDefinition="Obtained date/time. If the obtained element is a period, a date that falls in the period") + @OperationParam(name="date") + DateRangeParam theDate, + +// @Description(shortDefinition="The maximum number of observations to return for each each observation code") +// @OperationParam(name="max", max=1, min=0) +// NumberParam theMax, + + @Description(shortDefinition="The subject that the observation is about (if patient)") + @OperationParam(name="patient") + ReferenceAndListParam thePatient, + + @Description(shortDefinition="The subject that the observation is about") + @OperationParam(name="subject" ) + ReferenceAndListParam theSubject, + + @IncludeParam(reverse=true) + Set theRevIncludes, + @Description(shortDefinition="Only return resources which were last updated as specified by the given range") + @OperationParam(name="_lastUpdated") + DateRangeParam theLastUpdated, + + @IncludeParam(allow= { + "Observation:based-on", + "Observation:derived-from", + "Observation:device", + "Observation:encounter", + "Observation:focus", + "Observation:has-member", + "Observation:part-of", + "Observation:patient", + "Observation:performer", + "Observation:specimen", + "Observation:subject", + "*" + }) + Set theIncludes, + + @Sort + SortSpec theSort, + + @ca.uhn.fhir.rest.annotation.Count + Integer theCount, + + SummaryEnum theSummaryMode, + + SearchTotalModeEnum theSearchTotalMode + + ) { + startRequest(theServletRequest); + try { + SearchParameterMap paramMap = new SearchParameterMap(); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_FILTER, theFtFilter); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, theFtContent); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_TEXT, theFtText); + paramMap.add("category", theCategory); + paramMap.add("code", theCode); + paramMap.add("date", theDate); +// paramMap.add("max", theMax); + paramMap.add("patient", thePatient); + paramMap.add("subject", theSubject); + paramMap.setRevIncludes(theRevIncludes); + paramMap.setLastUpdated(theLastUpdated); + paramMap.setIncludes(theIncludes); + paramMap.setLastN(true); + paramMap.setSort(theSort); + paramMap.setCount(theCount); + paramMap.setSummaryMode(theSummaryMode); + paramMap.setSearchTotalMode(theSearchTotalMode); + + return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); + } finally { + endRequest(theServletRequest); + } + } + + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java new file mode 100644 index 00000000000..85a4dcd7c2a --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java @@ -0,0 +1,148 @@ +package ca.uhn.fhir.jpa.provider.r5; + +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.model.util.JpaConstants; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.api.Include; +import ca.uhn.fhir.model.api.annotation.Description; +import ca.uhn.fhir.model.valueset.BundleTypeEnum; +import ca.uhn.fhir.rest.annotation.*; +import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.api.SummaryEnum; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.param.*; +import org.hl7.fhir.r5.model.Observation; + +import java.util.Set; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5 { + + /** + * Observation/$lastn + */ + @Operation(name = JpaConstants.OPERATION_LASTN, idempotent = true, bundleType = BundleTypeEnum.SEARCHSET) + public IBundleProvider observationLastN( + + javax.servlet.http.HttpServletRequest theServletRequest, + javax.servlet.http.HttpServletResponse theServletResponse, + + ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails, + + @Description(shortDefinition="Search the contents of the resource's data using a filter") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_FILTER) + StringAndListParam theFtFilter, + + @Description(shortDefinition="Search the contents of the resource's data using a fulltext search") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT) + StringAndListParam theFtContent, + + @Description(shortDefinition="Search the contents of the resource's narrative using a fulltext search") + @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_TEXT) + StringAndListParam theFtText, + + @Description(shortDefinition="The classification of the type of observation") + @OperationParam(name="category") + TokenAndListParam theCategory, + + @Description(shortDefinition="The code of the observation type") + @OperationParam(name="code") + TokenAndListParam theCode, + + @Description(shortDefinition="Obtained date/time. If the obtained element is a period, a date that falls in the period") + @OperationParam(name="date") + DateRangeParam theDate, + + @Description(shortDefinition="The maximum number of observations to return for each each observation code") + @OperationParam(name="max", max=1, min=0) + NumberParam theMax, + + @Description(shortDefinition="The subject that the observation is about (if patient)") + @OperationParam(name="patient") + ReferenceAndListParam thePatient, + + @Description(shortDefinition="The subject that the observation is about") + @OperationParam(name="subject" ) + ReferenceAndListParam theSubject, + + @IncludeParam(reverse=true) + Set theRevIncludes, + @Description(shortDefinition="Only return resources which were last updated as specified by the given range") + @OperationParam(name="_lastUpdated") + DateRangeParam theLastUpdated, + + @IncludeParam(allow= { + "Observation:based-on", + "Observation:derived-from", + "Observation:device", + "Observation:encounter", + "Observation:focus", + "Observation:has-member", + "Observation:part-of", + "Observation:patient", + "Observation:performer", + "Observation:specimen", + "Observation:subject", + "*" + }) + Set theIncludes, + + @Sort + SortSpec theSort, + + @ca.uhn.fhir.rest.annotation.Count + Integer theCount, + + SummaryEnum theSummaryMode, + + SearchTotalModeEnum theSearchTotalMode + + ) { + startRequest(theServletRequest); + try { + SearchParameterMap paramMap = new SearchParameterMap(); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_FILTER, theFtFilter); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, theFtContent); + paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_TEXT, theFtText); + paramMap.add("category", theCategory); + paramMap.add("code", theCode); + paramMap.add("date", theDate); + paramMap.add("max", theMax); + paramMap.add("patient", thePatient); + paramMap.add("subject", theSubject); + paramMap.setRevIncludes(theRevIncludes); + paramMap.setLastUpdated(theLastUpdated); + paramMap.setIncludes(theIncludes); + paramMap.setLastN(true); + paramMap.setSort(theSort); + paramMap.setCount(theCount); + paramMap.setSummaryMode(theSummaryMode); + paramMap.setSearchTotalMode(theSearchTotalMode); + + return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); + } finally { + endRequest(theServletRequest); + } + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 62cc30fd6a1..8ac91289087 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -1,25 +1,28 @@ package ca.uhn.fhir.jpa.search.lastn; +import ca.uhn.fhir.jpa.dao.index.IdHelperService; +import ca.uhn.fhir.jpa.model.cross.ResourcePersistentId; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.IQueryParameterType; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.param.NumberParam; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import ca.uhn.fhir.rest.server.RestfulServerUtils; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -//import org.assertj.core.util.VisibleForTesting; import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.shadehapi.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.shadehapi.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.shadehapi.elasticsearch.action.DocWriteResponse; import org.shadehapi.elasticsearch.action.index.IndexRequest; import org.shadehapi.elasticsearch.action.index.IndexResponse; import org.shadehapi.elasticsearch.action.search.SearchRequest; import org.shadehapi.elasticsearch.action.search.SearchResponse; -import org.shadehapi.elasticsearch.action.support.master.AcknowledgedResponse; import org.shadehapi.elasticsearch.client.RequestOptions; import org.shadehapi.elasticsearch.client.RestHighLevelClient; import org.shadehapi.elasticsearch.common.xcontent.XContentType; @@ -39,7 +42,6 @@ import org.shadehapi.elasticsearch.search.aggregations.metrics.tophits.ParsedTop import org.shadehapi.elasticsearch.search.aggregations.support.ValueType; import org.shadehapi.elasticsearch.search.builder.SearchSourceBuilder; import org.shadehapi.elasticsearch.search.sort.SortOrder; -import org.springframework.stereotype.Component; import java.io.IOException; import java.util.ArrayList; @@ -47,7 +49,6 @@ import java.util.List; import static org.apache.commons.lang3.StringUtils.isBlank; -//@Component public class ElasticsearchSvcImpl implements IElasticsearchSvc { RestHighLevelClient myRestHighLevelClient; @@ -56,18 +57,17 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { public ElasticsearchSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { + myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername,thePassword); - myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername,thePassword); + try { + createObservationIndexIfMissing(); + createCodeIndexIfMissing(); + } catch (IOException theE) { + throw new RuntimeException("Failed to create document index", theE); + } + } - try { - createObservationIndexIfMissing(); - createCodeIndexIfMissing(); - } catch (IOException theE) { - throw new RuntimeException("Failed to create document index", theE); - } - } - - public void createObservationIndexIfMissing() throws IOException { + private void createObservationIndexIfMissing() throws IOException { if(indexExists(IndexConstants.OBSERVATION_INDEX)) { return; } @@ -129,7 +129,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } - public void createCodeIndexIfMissing() throws IOException { + private void createCodeIndexIfMissing() throws IOException { if(indexExists(IndexConstants.CODE_INDEX)) { return; } @@ -166,7 +166,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } - public boolean createIndex(String theIndexName, String theMapping) throws IOException { + private boolean createIndex(String theIndexName, String theMapping) throws IOException { CreateIndexRequest request = new CreateIndexRequest(theIndexName); request.source(theMapping, XContentType.JSON); CreateIndexResponse createIndexResponse = myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT); @@ -174,12 +174,11 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } - public boolean performIndex(String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) throws IOException { + boolean performIndex(String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) throws IOException { + IndexResponse indexResponse = myRestHighLevelClient.index(createIndexRequest(theIndexName, theDocumentId,theIndexDocument,theDocumentType), + RequestOptions.DEFAULT); - IndexResponse indexResponse = myRestHighLevelClient.index(createIndexRequest(theIndexName, theDocumentId,theIndexDocument,theDocumentType), - RequestOptions.DEFAULT); - - return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); + return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); } private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { @@ -191,12 +190,31 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return request; } - public boolean indexExists(String theIndexName) throws IOException { + private boolean indexExists(String theIndexName) throws IOException { GetIndexRequest request = new GetIndexRequest(); request.indices(theIndexName); return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); } + @Override + public List executeLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, IdHelperService theIdHelperService) { + Integer myMaxObservationsPerCode = 1; + String[] maxCountParams = theRequestDetails.getParameters().get("map"); + if (maxCountParams != null && maxCountParams.length > 0) { + myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]); + } + SearchRequest myLastNRequest = buildObservationCompositeSearchRequest(10000, theSearchParameterMap, myMaxObservationsPerCode); + SearchResponse lastnResponse = null; + try { + lastnResponse = executeSearchRequest(myLastNRequest); + List observationIds = buildObservationIdList(lastnResponse, theIdHelperService); + return observationIds; + } catch (IOException theE) { + throw new InvalidRequestException("Unable to execute LastN request", theE); + } + } + + SearchRequest buildObservationCodesSearchRequest(int theMaxResultSetSize) { SearchRequest searchRequest = new SearchRequest(IndexConstants.CODE_INDEX); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -236,15 +254,15 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } public SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { - return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); } public List buildObservationCompositeResults(SearchResponse theSearchResponse) throws IOException { - Aggregations responseAggregations = theSearchResponse.getAggregations(); - ParsedComposite aggregatedSubjects = responseAggregations.get("group_by_subject"); - List subjectBuckets = aggregatedSubjects.getBuckets(); - List codes = new ArrayList<>(); - for(ParsedComposite.ParsedBucket subjectBucket : subjectBuckets) { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedComposite aggregatedSubjects = responseAggregations.get("group_by_subject"); + List subjectBuckets = aggregatedSubjects.getBuckets(); + List codes = new ArrayList<>(); + for(ParsedComposite.ParsedBucket subjectBucket : subjectBuckets) { Aggregations observationCodeAggregations = subjectBucket.getAggregations(); ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); @@ -262,7 +280,30 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return codes; } - public List buildObservationTermsResults(SearchResponse theSearchResponse) throws IOException { + private List buildObservationIdList(SearchResponse theSearchResponse, IdHelperService theIdHelperService) throws IOException { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedComposite aggregatedSubjects = responseAggregations.get("group_by_subject"); + List subjectBuckets = aggregatedSubjects.getBuckets(); + List myObservationIds = new ArrayList<>(); + for(ParsedComposite.ParsedBucket subjectBucket : subjectBuckets) { + Aggregations observationCodeAggregations = subjectBucket.getAggregations();ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); + List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); + for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { + Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + SearchHit[] topHits = parsedTopHits.getHits().getHits(); + for (SearchHit topHit : topHits) { + String sources = topHit.getSourceAsString(); + ObservationJson code = objectMapper.readValue(sources, ObservationJson.class); + + myObservationIds.add(theIdHelperService.resolveResourcePersistentIds("Observation", code.getIdentifier())); + } + } + } + return myObservationIds; + } + + public List buildObservationTermsResults(SearchResponse theSearchResponse) throws IOException { Aggregations responseAggregations = theSearchResponse.getAggregations(); ParsedTerms aggregatedSubjects = responseAggregations.get("group_by_subject"); List subjectBuckets = aggregatedSubjects.getBuckets(); @@ -295,7 +336,6 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return codes; } -// @VisibleForTesting public SearchRequest buildObservationAllFieldsCompositeSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { return buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, null)); @@ -307,7 +347,6 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, topHitsInclude)); } -// @VisibleForTesting public SearchRequest buildObservationAllFieldsTermsSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { return buildObservationsSearchRequest(theSearchParameterMap, createTermsAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, null)); } @@ -518,18 +557,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } -// @VisibleForTesting - public boolean deleteIndex(String theIndexName) throws IOException { - DeleteIndexRequest request = new DeleteIndexRequest(theIndexName); - AcknowledgedResponse deleteIndexResponse = myRestHighLevelClient.indices().delete(request, RequestOptions.DEFAULT); - - return deleteIndexResponse.isAcknowledged(); - } - - - -// @VisibleForTesting - public void deleteAllDocuments(String theIndexName) throws IOException { + void deleteAllDocuments(String theIndexName) throws IOException { DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(theIndexName); deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java index 4c2be1cb2b5..bd6064481f1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -2,6 +2,13 @@ package ca.uhn.fhir.jpa.search.lastn; //import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.jpa.dao.index.IdHelperService; +import ca.uhn.fhir.jpa.model.cross.ResourcePersistentId; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.api.server.RequestDetails; + +import java.util.List; + public interface IElasticsearchSvc { @@ -9,4 +16,5 @@ public interface IElasticsearchSvc { // IBundleProvider uniqueCodes(javax.servlet.http.HttpServletRequest theServletRequest, ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails); + List executeLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, IdHelperService theIdHelperService); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java deleted file mode 100644 index 3ede8b1d67f..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java +++ /dev/null @@ -1,26 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.config; - -import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -import java.io.IOException; - -@Configuration -@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") -@EnableTransactionManagement -public class ElasticsearchConfig { - - private final String elasticsearchHost = "127.0.0.1"; - private final Integer elasticsearchPort = 9301; - private final String elasticsearchUserId = "elastic"; - private final String elasticsearchPassword = "changeme"; - - @Bean() - public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { - return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); - } - -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java index 959dba5c5e7..4f393e8d98f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java @@ -53,16 +53,24 @@ public class ObservationJson { private String myCode_concept_text; @JsonProperty(value = "codeconceptcodingcode", required = false) - private List myCode_coding_code = new ArrayList<>(); + // TODO: Temporary change until sort out how to deal with multiple observation code codings +// private List myCode_coding_code = new ArrayList<>(); + private String myCode_coding_code; @JsonProperty(value = "codeconceptcodingcode_system_hash", required = false) - private List myCode_coding_code_system_hash = new ArrayList<>(); + // TODO: Temporary change until sort out how to deal with multiple observation code codings +// private List myCode_coding_code_system_hash = new ArrayList<>(); + private String myCode_coding_code_system_hash; @JsonProperty(value = "codeconceptcodingdisplay", required = false) - private List myCode_coding_display = new ArrayList<>(); + // TODO: Temporary change until sort out how to deal with multiple observation code codings +// private List myCode_coding_display = new ArrayList<>(); + private String myCode_coding_display; @JsonProperty(value = "codeconceptcodingsystem", required = false) - private List myCode_coding_system = new ArrayList<>(); + // TODO: Temporary change until sort out how to deal with multiple observation code codings +// private List myCode_coding_system = new ArrayList<>(); + private String myCode_coding_system; @JsonProperty(value = "effectivedtm", required = true) private Date myEffectiveDtm; @@ -120,10 +128,16 @@ public class ObservationJson { public void setCode(CodeableConcept theCode) { myCode_concept_text = theCode.getText(); for(Coding theCodeCoding : theCode.getCoding()) { - myCode_coding_code_system_hash.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCodeCoding.getSystem(), theCodeCoding.getCode()))); + // TODO: Temporary change until address how to manage multiple Observation Code codings. +/* myCode_coding_code_system_hash.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCodeCoding.getSystem(), theCodeCoding.getCode()))); myCode_coding_code.add(theCodeCoding.getCode()); myCode_coding_display.add(theCodeCoding.getDisplay()); myCode_coding_system.add(theCodeCoding.getSystem()); + */ + myCode_coding_code_system_hash = String.valueOf(CodeSystemHash.hashCodeSystem(theCodeCoding.getSystem(), theCodeCoding.getCode())); + myCode_coding_code = theCodeCoding.getCode(); + myCode_coding_display = theCodeCoding.getDisplay(); + myCode_coding_system = theCodeCoding.getSystem(); } } @@ -132,6 +146,8 @@ public class ObservationJson { return myCode_concept_text; } + // TODO: Temporary changes until resolve problem of how to manage Observation Code with multiple codings +/* public List getCode_coding_code_system_hash() { return myCode_coding_code_system_hash; } @@ -147,6 +163,22 @@ public class ObservationJson { public List getCode_coding_system() { return myCode_coding_system; } +*/ + public String getCode_coding_code_system_hash() { + return myCode_coding_code_system_hash; + } + + public String getCode_coding_code() { + return myCode_coding_code; + } + + public String getCode_coding_display() { + return myCode_coding_display; + } + + public String getCode_coding_system() { + return myCode_coding_system; + } public void setCode_concept_id(String theCodeId) { myCode_concept_id = theCodeId; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java index 32a1324bea5..f3eabc1ba23 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java @@ -1,13 +1,19 @@ package ca.uhn.fhir.jpa.dao.lastn; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.dao.IFhirSystemDao; import ca.uhn.fhir.jpa.dao.lastn.config.TestIntegratedObservationIndexSearchConfig; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.parser.IParser; +import ca.uhn.fhir.parser.LenientErrorHandler; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import com.google.common.base.Charsets; +import org.apache.commons.io.IOUtils; import org.shadehapi.elasticsearch.action.search.SearchRequest; import org.shadehapi.elasticsearch.action.search.SearchResponse; import org.hl7.fhir.r4.model.*; @@ -16,11 +22,14 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import java.io.IOException; +import java.io.*; import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; @@ -35,11 +44,14 @@ public class IntegratedObservationIndexedSearchParamLastNTest { IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; @Autowired - ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; + ObservationLastNIndexPersistR4Svc myObservationLastNIndexPersistR4Svc; @Autowired private ElasticsearchSvcImpl elasticsearchSvc; + @Autowired + private IFhirSystemDao myDao; + final String RESOURCEPID = "123"; final String SUBJECTID = "4567"; final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; @@ -107,11 +119,11 @@ public class IntegratedObservationIndexedSearchParamLastNTest { String observationCodeText = "Test Codeable Concept Field for Code"; CodeableConcept codeableConceptField = new CodeableConcept().setText(observationCodeText); codeableConceptField.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test-code display")); - codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); - codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); +// codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); +// codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); myObservation.setCode(codeableConceptField); - myObservationLastNIndexPersistSvc.indexObservation(myObservation); + myObservationLastNIndexPersistR4Svc.indexObservation(myObservation); SearchParameterMap searchParameterMap = new SearchParameterMap(); ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); @@ -131,18 +143,18 @@ public class IntegratedObservationIndexedSearchParamLastNTest { assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); // execute Observation ID search - Composite Aggregation -// searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); -// responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); -// observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); + responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); -// assertEquals(1, observationIdsOnly.size()); -// observationIdOnly = observationIdsOnly.get(0); -// assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + assertEquals(1, observationIdsOnly.size()); + observationIdOnly = observationIdsOnly.get(0); + assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); } -// @Test + @Test public void testIndexObservationMultiple() { // Create two CodeableConcept values each for a Code with three codings. @@ -200,7 +212,7 @@ public class IntegratedObservationIndexedSearchParamLastNTest { Date effectiveDtm = observationDate.getTime(); observation.setEffective(new DateTimeType(effectiveDtm)); - myObservationLastNIndexPersistSvc.indexObservation(observation); + myObservationLastNIndexPersistR4Svc.indexObservation(observation); } @@ -211,4 +223,57 @@ public class IntegratedObservationIndexedSearchParamLastNTest { } + @Test + public void testSampleBundle() { + FhirContext myFhirCtx = FhirContext.forR4(); + + PathMatchingResourcePatternResolver provider = new PathMatchingResourcePatternResolver(); + final Resource[] bundleResources; + try { + bundleResources = provider.getResources("lastntestbundle.json"); + } catch (IOException e) { + throw new RuntimeException("Unexpected error during transmission: " + e.toString(), e); + } + + AtomicInteger index = new AtomicInteger(); + + Arrays.stream(bundleResources).forEach( + resource -> { + index.incrementAndGet(); + + InputStream resIs = null; + String nextBundleString; + try { + resIs = resource.getInputStream(); + nextBundleString = IOUtils.toString(resIs, Charsets.UTF_8); + } catch (IOException e) { + return; + } finally { + try { + if (resIs != null) { + resIs.close(); + } + } catch (final IOException ioe) { + // ignore + } + } + + /* + * SMART demo apps rely on the use of LOINC 3141-9 (Body Weight Measured) + * instead of LOINC 29463-7 (Body Weight) + */ + nextBundleString = nextBundleString.replace("\"29463-7\"", "\"3141-9\""); + + IParser parser = myFhirCtx.newJsonParser(); + parser.setParserErrorHandler(new LenientErrorHandler(false)); + Bundle bundle = parser.parseResource(Bundle.class, nextBundleString); + + myDao.transaction(null, bundle); + + + } + ); + + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java index e044a9b9e57..1ea31e450bb 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java @@ -29,7 +29,7 @@ public class PersistObservationIndexedSearchParamLastNTest { IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; @Autowired - ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; + ObservationLastNIndexPersistR4Svc myObservationLastNIndexPersistR4Svc; @Before public void before() { @@ -94,7 +94,7 @@ public class PersistObservationIndexedSearchParamLastNTest { codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); myObservation.setCode(codeableConceptField); - myObservationLastNIndexPersistSvc.indexObservation(myObservation); + myObservationLastNIndexPersistR4Svc.indexObservation(myObservation); List persistedObservationEntities = myResourceIndexedObservationLastNDao.findAll(); assertEquals(1, persistedObservationEntities.size()); @@ -171,7 +171,7 @@ public class PersistObservationIndexedSearchParamLastNTest { Date effectiveDtm = observationDate.getTime(); observation.setEffective(new DateTimeType(effectiveDtm)); - myObservationLastNIndexPersistSvc.indexObservation(observation); + myObservationLastNIndexPersistR4Svc.indexObservation(observation); } @@ -180,6 +180,11 @@ public class PersistObservationIndexedSearchParamLastNTest { assertEquals(100, myResourceIndexedObservationLastNDao.count()); assertEquals(2, myCodeableConceptIndexedSearchParamNormalizedDao.count()); - } + } + + @Test + public void testSampleObservationResource() { + + } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java index 0aec82de5f1..5f8be3aac8d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java @@ -10,7 +10,7 @@ import java.io.IOException; @Configuration @EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory", - basePackages = {"ca.uhn.fhir.jpa.dao"}) + basePackages = {"ca.uhn.fhir.jpa.dao.data"}) @EnableTransactionManagement public class TestIntegratedObservationIndexSearchConfig extends TestObservationIndexSearchConfig { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java index ed8dc143e26..64dab0a32b0 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java @@ -15,6 +15,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; import pl.allegro.tech.embeddedelasticsearch.PopularProperties; +import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Properties; import java.util.UUID; @@ -22,7 +23,7 @@ import java.util.concurrent.TimeUnit; @Configuration @EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory", - basePackages = {"ca.uhn.fhir.jpa.dao"}) + basePackages = {"ca.uhn.fhir.jpa.dao.data"}) @EnableTransactionManagement public class TestObservationIndexSearchConfig extends TestR4Config { @@ -84,5 +85,9 @@ public class TestObservationIndexSearchConfig extends TestR4Config { return embeddedElastic; } + @PreDestroy + public void stop() { + embeddedElasticSearch().stop(); + } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java index ea9308d1896..96a71aeacc8 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java @@ -35,7 +35,7 @@ import static org.junit.Assert.assertEquals; @Ignore public class ElasticsearchV5PerformanceTests { - +/* private ElasticsearchV5SvcImpl elasticsearchSvc = new ElasticsearchV5SvcImpl("localhost", 9301, "elastic", "changeme"); private List patientIds = new ArrayList<>(); @@ -148,6 +148,6 @@ public class ElasticsearchV5PerformanceTests { } return identifiers; } - +*/ } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java index 167d88dba80..70886b8d23c 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java @@ -234,6 +234,8 @@ public class LastNElasticsearchSvcSingleObservationTest { String code_concept_text_values = observation.getCode_concept_text(); assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. + /* List code_coding_systems = observation.getCode_coding_system(); assertEquals(3,code_coding_systems.size()); assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems.get(0)); @@ -257,6 +259,18 @@ public class LastNElasticsearchSvcSingleObservationTest { assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash.get(0)); assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), code_coding_code_system_hash.get(1)); assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), code_coding_code_system_hash.get(2)); +*/ + String code_coding_systems = observation.getCode_coding_system(); + assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems); + + String code_coding_codes = observation.getCode_coding_code(); + assertEquals(CODEFIRSTCODINGCODE, code_coding_codes); + + String code_coding_display = observation.getCode_coding_display(); + assertEquals(CODEFIRSTCODINGDISPLAY, code_coding_display); + + String code_coding_code_system_hash = observation.getCode_coding_code_system_hash(); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash); // Retrieve all Observation codes SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java index 08f731e0f15..a3ded8e1a86 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java @@ -34,7 +34,7 @@ import static org.junit.Assert.*; public class LastNElasticsearchV5SvcMultipleObservationsTest { @Autowired - private ElasticsearchV5SvcImpl elasticsearchSvc; +// private ElasticsearchV5SvcImpl elasticsearchSvc; private static ObjectMapper ourMapperNonPrettyPrint; @@ -56,8 +56,8 @@ public class LastNElasticsearchV5SvcMultipleObservationsTest { @After public void after() throws IOException { - elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); +// elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); +// elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); } /* @Test diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java index 48b42d31ec5..5c2479957aa 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java @@ -37,7 +37,7 @@ import static org.junit.Assert.assertTrue; public class LastNElasticsearchV5SvcSingleObservationTest { @Autowired - ElasticsearchV5SvcImpl elasticsearchSvc; +// ElasticsearchV5SvcImpl elasticsearchSvc; static ObjectMapper ourMapperNonPrettyPrint; @@ -93,14 +93,14 @@ public class LastNElasticsearchV5SvcSingleObservationTest { // @Before public void before() throws IOException { - elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); +// elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); +// elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); } @After public void after() throws IOException { - elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); +// elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); +// elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); } @Test public void testSingleObservationQuery() throws IOException { @@ -222,7 +222,7 @@ public class LastNElasticsearchV5SvcSingleObservationTest { String code_concept_text_values = observation.getCode_concept_text(); assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); - +/* List code_coding_systems = observation.getCode_coding_system(); assertEquals(3,code_coding_systems.size()); assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems.get(0)); @@ -246,7 +246,7 @@ public class LastNElasticsearchV5SvcSingleObservationTest { assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash.get(0)); assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), code_coding_code_system_hash.get(1)); assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), code_coding_code_system_hash.get(2)); - +*/ // Retrieve all Observation codes /* SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); @@ -329,11 +329,11 @@ public class LastNElasticsearchV5SvcSingleObservationTest { indexedObservation.setCode(codeableConceptField); String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); - assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); +// assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); - assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); +// assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); try { Thread.sleep(1000L); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java index 4b358584ca7..0ffb6a61118 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java @@ -9,6 +9,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; import pl.allegro.tech.embeddedelasticsearch.PopularProperties; +import javax.annotation.PreDestroy; import java.io.IOException; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -50,4 +51,9 @@ public class TestElasticsearchConfig { return embeddedElastic; } + @PreDestroy + public void stop() { + embeddedElasticSearch().stop(); + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java index a28e092bba9..973b19b00ba 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java @@ -1,7 +1,7 @@ package ca.uhn.fhir.jpa.search.lastn.config; import ca.uhn.fhir.context.ConfigurationException; -import ca.uhn.fhir.jpa.search.lastn.ElasticsearchV5SvcImpl; +//import ca.uhn.fhir.jpa.search.lastn.ElasticsearchV5SvcImpl; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; @@ -13,9 +13,9 @@ import java.io.IOException; import java.util.UUID; import java.util.concurrent.TimeUnit; -@Configuration -@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") -@EnableTransactionManagement +//@Configuration +//@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") +//@EnableTransactionManagement public class TestElasticsearchV5Config { private final String elasticsearchHost = "127.0.0.1"; @@ -24,7 +24,7 @@ public class TestElasticsearchV5Config { private static final String ELASTIC_VERSION = "5.6.16"; - +/* @Bean() public ElasticsearchV5SvcImpl myElasticsearchSvc() throws IOException { int elasticsearchPort = embeddedElasticSearch().getHttpPort(); @@ -49,5 +49,5 @@ public class TestElasticsearchV5Config { return embeddedElastic; } - +*/ } diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java index 66f2c5b17ed..1850e02dc0b 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java @@ -191,6 +191,11 @@ public class JpaConstants { */ public static final String OPERATION_EXPORT_POLL_STATUS = "$export-poll-status"; + /** + * Operation name for the "$lastn" operation + */ + public static final String OPERATION_LASTN = "$lastn"; + /** *

* This extension should be of type string and should be diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java index 1cdb6f4ca37..becf093ce6a 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java @@ -61,6 +61,7 @@ public class SearchParameterMap implements Serializable { private SummaryEnum mySummaryMode; private SearchTotalModeEnum mySearchTotalMode; private QuantityParam myNearDistanceParam; + private boolean myLastN; /** * Constructor @@ -303,6 +304,24 @@ public class SearchParameterMap implements Serializable { return this; } + /** + * If set, tells the server to use an Elasticsearch query to generate a list of + * Resource IDs for the LastN operation + */ + public boolean isLastN() { + return myLastN; + } + + /** + * If set, tells the server to use an Elasticsearch query to generate a list of + * Resource IDs for the LastN operation + */ + public SearchParameterMap setLastN(boolean theLastN) { + myLastN = theLastN; + return this; + } + + /** * This method creates a URL query string representation of the parameters in this * object, excluding the part before the parameters, e.g. diff --git a/hapi-tinder-plugin/src/main/resources/vm/jpa_resource_provider.vm b/hapi-tinder-plugin/src/main/resources/vm/jpa_resource_provider.vm index 77ffd948fbf..665f598fd98 100644 --- a/hapi-tinder-plugin/src/main/resources/vm/jpa_resource_provider.vm +++ b/hapi-tinder-plugin/src/main/resources/vm/jpa_resource_provider.vm @@ -24,7 +24,7 @@ import ca.uhn.fhir.rest.api.SearchTotalModeEnum; public class ${className}ResourceProvider extends ## We have specialized base classes for RPs that handle certain resource types. These ## RPs implement type specific operations -#if ( $version != 'dstu' && (${className} == 'Encounter' || ${className} == 'Patient' || ${className} == 'ValueSet' || ${className} == 'QuestionnaireAnswers' || ${className} == 'CodeSystem' || ($version != 'dstu2' && ${className} == 'ConceptMap') || ${className} == 'MessageHeader' || ${className} == 'Composition' || ${className} == 'StructureDefinition')) +#if ( $version != 'dstu' && (${className} == 'Encounter' || ${className} == 'Patient' || ${className} == 'ValueSet' || ${className} == 'QuestionnaireAnswers' || ${className} == 'CodeSystem' || ($version != 'dstu2' && ${className} == 'ConceptMap') || ${className} == 'MessageHeader' || ${className} == 'Composition' || ${className} == 'StructureDefinition' || ${className} == 'Observation' )) BaseJpaResourceProvider${className}${versionCapitalized} #else JpaResourceProvider${versionCapitalized}<${className}> diff --git a/hapi-tinder-plugin/src/main/resources/vm/jpa_spring_beans.vm b/hapi-tinder-plugin/src/main/resources/vm/jpa_spring_beans.vm index b67693d3fb9..1390f1dc1eb 100644 --- a/hapi-tinder-plugin/src/main/resources/vm/jpa_spring_beans.vm +++ b/hapi-tinder-plugin/src/main/resources/vm/jpa_spring_beans.vm @@ -50,6 +50,8 @@ class="ca.uhn.fhir.jpa.dao.FhirResourceDao${res.name}${versionCapitalized}"> #elseif ( ${versionCapitalized} != 'Dstu1' && ( ${res.name} == 'Bundle' || ${res.name} == 'Encounter' || ${res.name} == 'Composition' || ${res.name} == 'Everything' || ${res.name} == 'Patient' || ${res.name} == 'Subscription' || ${res.name} == 'ValueSet' || ${res.name} == 'QuestionnaireResponse' || ${res.name} == 'SearchParameter')) class="ca.uhn.fhir.jpa.dao.FhirResourceDao${res.name}${versionCapitalized}"> +#elseif ( ${versionCapitalized} != 'Dstu1' && ${versionCapitalized} != 'Dstu2' && ${res.name} == 'Observation') + class="ca.uhn.fhir.jpa.dao.FhirResourceDao${res.name}${versionCapitalized}"> #else class="ca.uhn.fhir.jpa.dao.FhirResourceDao${versionCapitalized}"> #end diff --git a/hapi-tinder-plugin/src/main/resources/vm/jpa_spring_beans_java.vm b/hapi-tinder-plugin/src/main/resources/vm/jpa_spring_beans_java.vm index 976ec354e00..d136e1b7248 100644 --- a/hapi-tinder-plugin/src/main/resources/vm/jpa_spring_beans_java.vm +++ b/hapi-tinder-plugin/src/main/resources/vm/jpa_spring_beans_java.vm @@ -68,6 +68,8 @@ public abstract class BaseJavaConfig${versionCapitalized} extends ca.uhn.fhir.jp IFhirResourceDaoConceptMap #elseif ( ${versionCapitalized} != 'Dstu1' && (${res.name} == 'Composition' || ${res.name} == 'Encounter' || ${res.name} == 'Everything' || ${res.name} == 'Patient' || ${res.name} == 'Subscription' || ${res.name} == 'SearchParameter' || ${res.name} == 'MessageHeader' || ${res.name} == 'StructureDefinition')) IFhirResourceDao${res.name}<${resourcePackage}.${res.declaringClassNameComplete}> +#elseif ( ${versionCapitalized} != 'Dstu1' && ${versionCapitalized} != 'Dstu2' && (${res.name} == 'Observation')) + IFhirResourceDao${res.name}<${resourcePackage}.${res.declaringClassNameComplete}> #else IFhirResourceDao<${resourcePackage}.${res.declaringClassNameComplete}> #end @@ -82,6 +84,9 @@ public abstract class BaseJavaConfig${versionCapitalized} extends ca.uhn.fhir.jp #elseif ( ${res.name} == 'Bundle' || ${res.name} == 'Encounter' || ${res.name} == 'Everything' || ${res.name} == 'Patient' || ${res.name} == 'Subscription' || ${res.name} == 'ValueSet' || ${res.name} == 'QuestionnaireResponse' || ${res.name} == 'SearchParameter' || ${res.name} == 'CodeSystem' || ${res.name} == 'MessageHeader' || ${res.name} == 'Composition' || ${res.name} == 'StructureDefinition') ca.uhn.fhir.jpa.dao${package_suffix}.FhirResourceDao${res.name}${versionCapitalized} retVal; retVal = new ca.uhn.fhir.jpa.dao${package_suffix}.FhirResourceDao${res.name}${versionCapitalized}(); +#elseif ( ${versionCapitalized} != 'Dstu1' && ${versionCapitalized} != 'Dstu2' && ${res.name} == 'Observation') + ca.uhn.fhir.jpa.dao${package_suffix}.FhirResourceDao${res.name}${versionCapitalized} retVal; + retVal = new ca.uhn.fhir.jpa.dao${package_suffix}.FhirResourceDao${res.name}${versionCapitalized}(); #else ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao<${resourcePackage}.${res.declaringClassNameComplete}> retVal; retVal = new ca.uhn.fhir.jpa.dao.JpaResourceDao<${resourcePackage}.${res.declaringClassNameComplete}>(); From 83e673e725bc70a9dbf069b1f461cde680b37657 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Fri, 3 Apr 2020 17:12:04 -0400 Subject: [PATCH 04/31] Consolidate changes to Command Line Tool and add/improve tests. --- .../main/java/ca/uhn/fhir/cli/BaseApp.java | 1 - .../RunJpaServerWithElasticsearchCommand.java | 250 - .../ca/uhn/fhir/cli/RunServerCommand.java | 35 +- hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml | 4 +- .../ca/uhn/fhir/jpa/demo/CommonConfig.java | 158 +- .../fhir/jpa/demo/CommonPostgreSQLConfig.java | 180 - .../ca/uhn/fhir/jpa/demo/ContextHolder.java | 37 + .../jpa/demo/ContextPostgreSQLHolder.java | 89 - .../demo/FhirServerElasticsearchConfigR4.java | 107 - .../ca/uhn/fhir/jpa/demo/JpaServerDemo.java | 3 +- .../dao/r4/FhirResourceDaoObservationR4.java | 4 + .../search/lastn/ElasticsearchSvcImpl.java | 6 +- ...bservationIndexedSearchParamLastNTest.java | 20 +- .../dao/r4/FhirResourceDaoR4LastNTest.java | 150 + ...sticsearchSvcMultipleObservationsTest.java | 28 +- ...ElasticsearchSvcSingleObservationTest.java | 37 +- .../src/test/resources/lastntestbundle.json | 4717 +++++++++++++++++ 17 files changed, 5157 insertions(+), 669 deletions(-) delete mode 100644 hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunJpaServerWithElasticsearchCommand.java delete mode 100644 hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonPostgreSQLConfig.java delete mode 100644 hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextPostgreSQLHolder.java delete mode 100644 hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerElasticsearchConfigR4.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java create mode 100644 hapi-fhir-jpaserver-base/src/test/resources/lastntestbundle.json diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/BaseApp.java b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/BaseApp.java index bd1451d7730..c1b98297f41 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/BaseApp.java +++ b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/BaseApp.java @@ -173,7 +173,6 @@ public abstract class BaseApp { commands.add(new ExportConceptMapToCsvCommand()); commands.add(new ImportCsvToConceptMapCommand()); commands.add(new HapiFlywayMigrateDatabaseCommand()); - commands.add(new RunJpaServerWithElasticsearchCommand()); return commands; } diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunJpaServerWithElasticsearchCommand.java b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunJpaServerWithElasticsearchCommand.java deleted file mode 100644 index ca70d50845b..00000000000 --- a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunJpaServerWithElasticsearchCommand.java +++ /dev/null @@ -1,250 +0,0 @@ -package ca.uhn.fhir.cli; - -/*- - * #%L - * HAPI FHIR - Command Line Client - API - * %% - * Copyright (C) 2014 - 2020 University Health Network - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.jpa.dao.DaoConfig; -import ca.uhn.fhir.jpa.demo.*; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.time.DateUtils; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.webapp.WebAppContext; -import org.springframework.web.context.ContextLoader; -import org.springframework.web.context.ContextLoaderListener; -import org.springframework.web.context.support.AnnotationConfigWebApplicationContext; - -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; -import java.io.*; -import java.net.SocketException; - -public class RunJpaServerWithElasticsearchCommand extends BaseCommand { - - private static final String OPTION_DISABLE_REFERENTIAL_INTEGRITY = "disable-referential-integrity"; - private static final String OPTION_LOWMEM = "lowmem"; - private static final String OPTION_ALLOW_EXTERNAL_REFS = "allow-external-refs"; - private static final String OPTION_REUSE_SEARCH_RESULTS_MILLIS = "reuse-search-results-milliseconds"; - private static final String OPTION_EXTERNAL_ELASTICSEARCH = "external-elasticsearch"; - private static final int DEFAULT_PORT = 8080; - private static final String OPTION_P = "p"; - - // TODO: Don't use qualified names for loggers in HAPI CLI. - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(RunJpaServerWithElasticsearchCommand.class); - public static final String RUN_SERVER_COMMAND_ELASTICSEARCH = "run-server-elasticsearch"; - private int myPort; - - private Server myServer; - - @Override - public String getCommandName() { - return RUN_SERVER_COMMAND_ELASTICSEARCH; - } - - @Override - public Options getOptions() { - Options options = new Options(); - addFhirVersionOption(options); - options.addOption(OPTION_P, "port", true, "The port to listen on (default is " + DEFAULT_PORT + ")"); - options.addOption(null, OPTION_LOWMEM, false, "If this flag is set, the server will operate in low memory mode (some features disabled)"); - options.addOption(null, OPTION_ALLOW_EXTERNAL_REFS, false, "If this flag is set, the server will allow resources to be persisted contaning external resource references"); - options.addOption(null, OPTION_DISABLE_REFERENTIAL_INTEGRITY, false, "If this flag is set, the server will not enforce referential integrity"); - options.addOption(null, OPTION_EXTERNAL_ELASTICSEARCH, false, "If this flag is set, the server will attempt to use an external elasticsearch instance listening on port 9301"); - - addOptionalOption(options, "u", "url", "Url", "If this option is set, specifies the JDBC URL to use for the database connection"); - - Long defaultReuseSearchResults = DaoConfig.DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS; - String defaultReuseSearchResultsStr = defaultReuseSearchResults == null ? "off" : String.valueOf(defaultReuseSearchResults); - options.addOption(null, OPTION_REUSE_SEARCH_RESULTS_MILLIS, true, "The time in milliseconds within which the same results will be returned for multiple identical searches, or \"off\" (default is " + defaultReuseSearchResultsStr + ")"); - - return options; - } - - private int parseOptionInteger(CommandLine theCommandLine, String opt, int defaultPort) throws ParseException { - try { - return Integer.parseInt(theCommandLine.getOptionValue(opt, Integer.toString(defaultPort))); - } catch (NumberFormatException e) { - throw new ParseException("Invalid value '" + theCommandLine.getOptionValue(opt) + "' (must be numeric)"); - } - } - - @Override - public void run(CommandLine theCommandLine) throws ParseException { - parseFhirContext(theCommandLine); - - myPort = parseOptionInteger(theCommandLine, OPTION_P, DEFAULT_PORT); - - if (theCommandLine.hasOption(OPTION_LOWMEM)) { - ourLog.info("Running in low memory mode, some features disabled"); - System.setProperty(OPTION_LOWMEM, OPTION_LOWMEM); - } - - if (theCommandLine.hasOption(OPTION_ALLOW_EXTERNAL_REFS)) { - ourLog.info("Server is configured to allow external references"); - ContextPostgreSQLHolder.setAllowExternalRefs(true); - } - - if (theCommandLine.hasOption(OPTION_DISABLE_REFERENTIAL_INTEGRITY)) { - ourLog.info("Server is configured to not enforce referential integrity"); - ContextPostgreSQLHolder.setDisableReferentialIntegrity(true); - } - - if (theCommandLine.hasOption(OPTION_EXTERNAL_ELASTICSEARCH)) { - ourLog.info("Server is configured to use external elasticsearch"); - ContextPostgreSQLHolder.setExternalElasticsearch(true); - } - - ContextPostgreSQLHolder.setDatabaseUrl(theCommandLine.getOptionValue("u")); - - String reuseSearchResults = theCommandLine.getOptionValue(OPTION_REUSE_SEARCH_RESULTS_MILLIS); - if (reuseSearchResults != null) { - if (reuseSearchResults.equals("off")) { - ourLog.info("Server is configured to not reuse search results"); - ContextPostgreSQLHolder.setReuseCachedSearchResultsForMillis(null); - } else { - try { - long reuseSearchResultsMillis = Long.parseLong(reuseSearchResults); - if (reuseSearchResultsMillis < 0) { - throw new NumberFormatException("expected a positive integer"); - } - ourLog.info("Server is configured to reuse search results for " + String.valueOf(reuseSearchResultsMillis) + " milliseconds"); - ContextPostgreSQLHolder.setReuseCachedSearchResultsForMillis(reuseSearchResultsMillis); - } catch (NumberFormatException e) { - throw new ParseException("Invalid value '" + reuseSearchResults + "' (must be a positive integer)"); - } - } - } - - ContextPostgreSQLHolder.setCtx(getFhirContext()); - - - - ourLog.info("Preparing HAPI FHIR JPA server on port {}", myPort); - File tempWarFile; - try { - tempWarFile = File.createTempFile("hapi-fhir", ".war"); - tempWarFile.deleteOnExit(); - - InputStream inStream = RunJpaServerWithElasticsearchCommand.class.getResourceAsStream("/hapi-fhir-cli-jpaserver.war"); - OutputStream outStream = new BufferedOutputStream(new FileOutputStream(tempWarFile, false)); - IOUtils.copy(inStream, outStream); - } catch (IOException e) { - ourLog.error("Failed to create temporary file", e); - return; - } - - final ContextLoaderListener cll = new ContextLoaderListener(); - - ourLog.info("Starting HAPI FHIR JPA server in {} mode", ContextPostgreSQLHolder.getCtx().getVersion().getVersion()); - WebAppContext root = new WebAppContext(); - root.setAllowDuplicateFragmentNames(true); - root.setWar(tempWarFile.getAbsolutePath()); - root.setParentLoaderPriority(true); - root.setContextPath("/"); - root.addEventListener(new ServletContextListener() { - @Override - public void contextInitialized(ServletContextEvent theSce) { - theSce.getServletContext().setInitParameter(ContextLoader.CONTEXT_CLASS_PARAM, AnnotationConfigWebApplicationContext.class.getName()); - switch (ContextPostgreSQLHolder.getCtx().getVersion().getVersion()) { - case DSTU2: - theSce.getServletContext().setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, FhirServerConfig.class.getName()); - break; - case DSTU3: - theSce.getServletContext().setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, FhirServerConfigDstu3.class.getName()); - break; - case R4: - theSce.getServletContext().setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, FhirServerElasticsearchConfigR4.class.getName()); - break; - case DSTU2_1: - case DSTU2_HL7ORG: - break; - } - cll.contextInitialized(theSce); - } - - @Override - public void contextDestroyed(ServletContextEvent theSce) { - cll.contextDestroyed(theSce); - } - }); - - String path = ContextPostgreSQLHolder.getPath(); - root.addServlet("ca.uhn.fhir.jpa.demo.JpaServerDemo", path + "*"); - - myServer = new Server(myPort); - myServer.setHandler(root); - try { - myServer.start(); - } catch (SocketException e) { - throw new CommandFailureException("Server failed to start on port " + myPort + " because of the following error \"" + e.toString() + "\". Note that you can use the '-p' option to specify an alternate port."); - } catch (Exception e) { - ourLog.error("Server failed to start", e); - throw new CommandFailureException("Server failed to start", e); - } - - ourLog.info("Server started on port {}", myPort); - ourLog.info("Web Testing UI : http://localhost:{}/", myPort); - ourLog.info("Server Base URL: http://localhost:{}{}", myPort, path); - - // Never quit.. We'll let the user ctrl-C their way out. - loopForever(); - - } - - @SuppressWarnings("InfiniteLoopStatement") - private void loopForever() { - while (true) { - try { - Thread.sleep(DateUtils.MILLIS_PER_MINUTE); - } catch (InterruptedException theE) { - // ignore - } - } - } - - public static void main(String[] theArgs) { - - - Server server = new Server(22); - String path = "../hapi-fhir-cli-jpaserver"; - WebAppContext webAppContext = new WebAppContext(); - webAppContext.setContextPath("/"); - webAppContext.setDescriptor(path + "/src/main/webapp/WEB-INF/web.xml"); - webAppContext.setResourceBase(path + "/target/hapi-fhir-jpaserver-example"); - webAppContext.setParentLoaderPriority(true); - - server.setHandler(webAppContext); - try { - server.start(); - } catch (Exception e) { - e.printStackTrace(); - } - - ourLog.info("Started"); - } - - @Override - public String getCommandDescription() { - return "Start a FHIR server which can be used for testing"; - } - -} diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java index 0ad0a398ca8..2dc562157b4 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java +++ b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java @@ -21,12 +21,8 @@ package ca.uhn.fhir.cli; */ import ca.uhn.fhir.jpa.dao.DaoConfig; -import ca.uhn.fhir.jpa.demo.ContextHolder; -import ca.uhn.fhir.jpa.demo.FhirServerConfig; -import ca.uhn.fhir.jpa.demo.FhirServerConfigDstu3; -import ca.uhn.fhir.jpa.demo.FhirServerConfigR4; +import ca.uhn.fhir.jpa.demo.*; import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.OptionGroup; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; @@ -48,8 +44,10 @@ public class RunServerCommand extends BaseCommand { private static final String OPTION_LOWMEM = "lowmem"; private static final String OPTION_ALLOW_EXTERNAL_REFS = "allow-external-refs"; private static final String OPTION_REUSE_SEARCH_RESULTS_MILLIS = "reuse-search-results-milliseconds"; + private static final String OPTION_EXTERNAL_ELASTICSEARCH = "external-elasticsearch"; private static final int DEFAULT_PORT = 8080; private static final String OPTION_P = "p"; + private static final String OPTION_POSTGRES = "postgresql"; // TODO: Don't use qualified names for loggers in HAPI CLI. private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(RunServerCommand.class); @@ -71,8 +69,12 @@ public class RunServerCommand extends BaseCommand { options.addOption(null, OPTION_LOWMEM, false, "If this flag is set, the server will operate in low memory mode (some features disabled)"); options.addOption(null, OPTION_ALLOW_EXTERNAL_REFS, false, "If this flag is set, the server will allow resources to be persisted contaning external resource references"); options.addOption(null, OPTION_DISABLE_REFERENTIAL_INTEGRITY, false, "If this flag is set, the server will not enforce referential integrity"); + options.addOption(null, OPTION_EXTERNAL_ELASTICSEARCH, false, "If this flag is set, the server will attempt to use a local elasticsearch server listening on port 9301"); + options.addOption(null, OPTION_POSTGRES, false, "If this flag is set, the server will attempt to use a local postgresql DB instance listening on port 5432"); addOptionalOption(options, "u", "url", "Url", "If this option is set, specifies the JDBC URL to use for the database connection"); + addOptionalOption(options, "d", "default-size", "PageSize", "If this option is set, specifies the default page size for number of query results"); + addOptionalOption(options, "m", "max-size", "MaxSize", "If this option is set, specifies the maximum result set size for queries"); Long defaultReuseSearchResults = DaoConfig.DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS; String defaultReuseSearchResultsStr = defaultReuseSearchResults == null ? "off" : String.valueOf(defaultReuseSearchResults); @@ -109,7 +111,28 @@ public class RunServerCommand extends BaseCommand { ContextHolder.setDisableReferentialIntegrity(true); } - ContextHolder.setDatabaseUrl(theCommandLine.getOptionValue("u")); + if (theCommandLine.hasOption(OPTION_EXTERNAL_ELASTICSEARCH)) { + ourLog.info("Server is configured to use external elasticsearch"); + ContextHolder.setExternalElasticsearch(true); + } + + ContextHolder.setDatabaseUrl(theCommandLine.getOptionValue("u")); + + if (theCommandLine.hasOption(OPTION_POSTGRES)) { + ourLog.info("Server is configured to use PostgreSQL database"); + ContextHolder.setPostgreSql(true); + } + + String defaultPageSize = theCommandLine.getOptionValue("d"); + String maxPageSize = theCommandLine.getOptionValue("m"); + if (defaultPageSize != null) { + ContextHolder.setDefaultPageSize(Integer.valueOf(defaultPageSize)); + if (maxPageSize != null) { + ContextHolder.setMaxPageSize(Integer.valueOf(maxPageSize)); + } else { + ContextHolder.setMaxPageSize(Integer.valueOf(defaultPageSize)); + } + } String reuseSearchResults = theCommandLine.getOptionValue(OPTION_REUSE_SEARCH_RESULTS_MILLIS); if (reuseSearchResults != null) { diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml index 5c41b2faea1..2509802ef16 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml @@ -183,10 +183,10 @@ src/main/webapp/WEB-INF/web.xml true - + WEB-INF/lib/Saxon-HE-*, WEB-INF/lib/hapi-* - + diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java index 7a14ef468a1..82b9a9b98dd 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java @@ -20,23 +20,34 @@ package ca.uhn.fhir.jpa.demo; * #L% */ +import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.model.entity.ModelConfig; import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory; +import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.lang3.time.DateUtils; import org.hibernate.dialect.H2Dialect; +import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus; +import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; +import pl.allegro.tech.embeddedelasticsearch.PopularProperties; +import javax.annotation.PreDestroy; import javax.sql.DataSource; +import java.io.IOException; import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isNotBlank; @SuppressWarnings("Duplicates") // TODO: Merge this with new CommonPostgreSQLConfig or find way to avoid conflicts with it. -//@Configuration +@Configuration public class CommonConfig { /** @@ -65,6 +76,42 @@ public class CommonConfig { */ @Bean(destroyMethod = "close") public DataSource dataSource() { + if (ContextHolder.isPostGreSql()) { + return getPostgreSqlDataSource(); + } else { + return getH2DataSource(); + } + } + + /** + * The following method creates a PostgreSQL database connection. The 'url' property value of "jdbc:postgresql://localhost:5432/hapi" indicates that the server should save resources in a + * PostgreSQL database named "hapi". + * + * A URL to a remote database could also be placed here, along with login credentials and other properties supported by BasicDataSource. + */ + private DataSource getPostgreSqlDataSource() { + String dbUrl = "jdbc:postgresql://localhost:5432/hapi"; + String dbUsername = "hapi"; + String dbPassword = "HapiFHIR"; + if (isNotBlank(ContextHolder.getDatabaseUrl())) { + dbUrl = ContextHolder.getDatabaseUrl(); + } + + BasicDataSource retVal = new BasicDataSource(); + retVal.setDriverClassName("org.postgresql.Driver"); + retVal.setUrl(dbUrl); + retVal.setUsername(dbUsername); + retVal.setPassword(dbPassword); + return retVal; + } + + /** + * The following method creates an H2 database connection. The 'url' property value of "jdbc:h2:file:target./jpaserver_h2_files" indicates that the server should save resources in a + * directory called "jpaserver_h2_files". + * + * A URL to a remote database could also be placed here, along with login credentials and other properties supported by BasicDataSource. + */ + private DataSource getH2DataSource() { String url = "jdbc:h2:file:./target/jpaserver_h2_files"; if (isNotBlank(ContextHolder.getDatabaseUrl())) { url = ContextHolder.getDatabaseUrl(); @@ -80,6 +127,14 @@ public class CommonConfig { @Bean public Properties jpaProperties() { + if (ContextHolder.isPostGreSql()) { + return getPostGreSqlJpaProperties(); + } else { + return getH2JpaProperties(); + } + } + + private Properties getH2JpaProperties() { Properties extraProperties = new Properties(); extraProperties.put("hibernate.dialect", H2Dialect.class.getName()); extraProperties.put("hibernate.format_sql", "true"); @@ -99,8 +154,105 @@ public class CommonConfig { if (System.getProperty("lowmem") != null) { extraProperties.put("hibernate.search.autoregister_listeners", "false"); } - - return extraProperties; + + return configureElasticearch(extraProperties); + } + + private Properties getPostGreSqlJpaProperties() { + + Properties extraProperties = new Properties(); + extraProperties.put("hibernate.dialect", org.hibernate.dialect.PostgreSQL94Dialect.class.getName()); + extraProperties.put("hibernate.format_sql", "false"); + extraProperties.put("hibernate.show_sql", "false"); + extraProperties.put("hibernate.hbm2ddl.auto", "update"); + extraProperties.put("hibernate.jdbc.batch_size", "20"); + extraProperties.put("hibernate.cache.use_query_cache", "false"); + extraProperties.put("hibernate.cache.use_second_level_cache", "false"); + extraProperties.put("hibernate.cache.use_structured_entries", "false"); + extraProperties.put("hibernate.cache.use_minimal_puts", "false"); + extraProperties.put("hibernate.search.model_mapping", LuceneSearchMappingFactory.class.getName()); + extraProperties.put("hibernate.search.default.directory_provider", "local-heap"); + extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); + extraProperties.put("hibernate.search.default.worker.execution", "sync"); + + if (System.getProperty("lowmem") != null) { + extraProperties.put("hibernate.search.autoregister_listeners", "false"); + } + + return configureElasticearch(extraProperties); + } + + private Properties configureElasticearch(Properties theExtraProperties) { + + String elasticsearchHost = "localhost"; + String elasticsearchUserId = ""; + String elasticsearchPassword = ""; + Integer elasticsearchPort; + + if(ContextHolder.isExternalElasticsearch()) { + elasticsearchUserId = "elastic"; + elasticsearchPassword = "changeme"; + elasticsearchPort = 9301; + } else { + elasticsearchPort = embeddedElasticSearch().getHttpPort(); + } + + new ElasticsearchHibernatePropertiesBuilder() + .setDebugRefreshAfterWrite(true) + .setDebugPrettyPrintJsonLog(true) + .setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE) + .setIndexManagementWaitTimeoutMillis(10000) + .setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW) + .setRestUrl("http://" + elasticsearchHost + ":" + elasticsearchPort) + .setUsername(elasticsearchUserId) + .setPassword(elasticsearchPassword) + .apply(theExtraProperties); + + return theExtraProperties; + + } + + @Bean() + public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + String elasticsearchHost = "localhost"; + String elasticsearchUserId = ""; + String elasticsearchPassword = ""; + Integer elasticsearchPort; + + if(ContextHolder.isExternalElasticsearch()) { + elasticsearchUserId = "elastic"; + elasticsearchPassword = "changeme"; + elasticsearchPort = 9301; + } else { + elasticsearchPort = embeddedElasticSearch().getHttpPort(); + } + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + + @Bean + public EmbeddedElastic embeddedElasticSearch() { + String ELASTIC_VERSION = "6.5.4"; + + EmbeddedElastic embeddedElastic; + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } + + return embeddedElastic; + } + + @PreDestroy + public void stop() { + embeddedElasticSearch().stop(); } } diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonPostgreSQLConfig.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonPostgreSQLConfig.java deleted file mode 100644 index 2dc0bfb08b5..00000000000 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonPostgreSQLConfig.java +++ /dev/null @@ -1,180 +0,0 @@ -package ca.uhn.fhir.jpa.demo; - -/*- - * #%L - * HAPI FHIR - Command Line Client - Server WAR - * %% - * Copyright (C) 2014 - 2020 University Health Network - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.context.ConfigurationException; -import ca.uhn.fhir.jpa.dao.DaoConfig; -import ca.uhn.fhir.jpa.model.entity.ModelConfig; -import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory; -import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; -import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; -import org.apache.commons.dbcp2.BasicDataSource; -import org.apache.commons.lang3.time.DateUtils; -import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus; -import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; -import pl.allegro.tech.embeddedelasticsearch.PopularProperties; - -import javax.annotation.PreDestroy; -import javax.sql.DataSource; -import java.io.IOException; -import java.util.Properties; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import static org.apache.commons.lang3.StringUtils.isNotBlank; - -@SuppressWarnings("Duplicates") -@Configuration -public class CommonPostgreSQLConfig { - - static String elasticsearchHost = "localhost"; - static String elasticsearchUserId = ""; - static String elasticsearchPassword = ""; - static Integer elasticsearchPort; - - /** - * Configure FHIR properties around the the JPA server via this bean - */ - @Bean - public DaoConfig daoConfig() { - DaoConfig retVal = new DaoConfig(); - retVal.setSubscriptionEnabled(true); - retVal.setSubscriptionPollDelay(5000); - retVal.setSubscriptionPurgeInactiveAfterMillis(DateUtils.MILLIS_PER_HOUR); - retVal.setAllowMultipleDelete(true); - return retVal; - } - - @Bean - public ModelConfig modelConfig() { - return daoConfig().getModelConfig(); - } - - /** - * The following bean configures the database connection. The 'url' property value of "jdbc:postgresql://localhost:5432/hapi" indicates that the server should save resources in a - * PostgreSQL database named "hapi". - * - * A URL to a remote database could also be placed here, along with login credentials and other properties supported by BasicDataSource. - */ - @Bean(destroyMethod = "close") - public DataSource dataSource() { - String dbUrl = "jdbc:postgresql://localhost:5432/hapi"; - String dbUsername = "hapi"; - String dbPassword = "HapiFHIR"; - if (isNotBlank(ContextPostgreSQLHolder.getDatabaseUrl())) { - dbUrl = ContextPostgreSQLHolder.getDatabaseUrl(); - } - - BasicDataSource retVal = new BasicDataSource(); - retVal.setDriverClassName("org.postgresql.Driver"); - retVal.setUrl(dbUrl); - retVal.setUsername(dbUsername); - retVal.setPassword(dbPassword); - return retVal; - } - - @Bean - public Properties jpaProperties() { - - if(ContextPostgreSQLHolder.isExternalElasticsearch()) { - elasticsearchUserId = "elastic"; - elasticsearchPassword = "changeme"; - elasticsearchPort = 9301; - } else { - elasticsearchPort = embeddedElasticSearch().getHttpPort(); - } - - Properties extraProperties = new Properties(); - extraProperties.put("hibernate.dialect", org.hibernate.dialect.PostgreSQL94Dialect.class.getName()); - extraProperties.put("hibernate.format_sql", "false"); - extraProperties.put("hibernate.show_sql", "false"); - extraProperties.put("hibernate.hbm2ddl.auto", "update"); - extraProperties.put("hibernate.jdbc.batch_size", "20"); - extraProperties.put("hibernate.cache.use_query_cache", "false"); - extraProperties.put("hibernate.cache.use_second_level_cache", "false"); - extraProperties.put("hibernate.cache.use_structured_entries", "false"); - extraProperties.put("hibernate.cache.use_minimal_puts", "false"); - extraProperties.put("hibernate.search.model_mapping", LuceneSearchMappingFactory.class.getName()); - extraProperties.put("hibernate.search.default.directory_provider", "local-heap"); - extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); - extraProperties.put("hibernate.search.default.worker.execution", "sync"); - - if (System.getProperty("lowmem") != null) { - extraProperties.put("hibernate.search.autoregister_listeners", "false"); - } - - new ElasticsearchHibernatePropertiesBuilder() - .setDebugRefreshAfterWrite(true) - .setDebugPrettyPrintJsonLog(true) - .setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE) - .setIndexManagementWaitTimeoutMillis(10000) - .setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW) - .setRestUrl("http://" + elasticsearchHost + ":" + elasticsearchPort) - .setUsername(elasticsearchUserId) - .setPassword(elasticsearchPassword) - .apply(extraProperties); - -// extraProperties.setProperty("hibernate.search.default.elasticsearch.refresh_after_write", "true"); - return extraProperties; - } - - @Bean() - public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { - if(ContextPostgreSQLHolder.isExternalElasticsearch()) { - elasticsearchUserId = "elastic"; - elasticsearchPassword = "changeme"; - elasticsearchPort = 9301; - } else { - elasticsearchPort = embeddedElasticSearch().getHttpPort(); - } - return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); - } - - @Bean - public EmbeddedElastic embeddedElasticSearch() { - String ELASTIC_VERSION = "6.5.4"; - - EmbeddedElastic embeddedElastic; - try { - embeddedElastic = EmbeddedElastic.builder() - .withElasticVersion(ELASTIC_VERSION) - .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) - .withSetting(PopularProperties.HTTP_PORT, 0) - .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) - .withStartTimeout(60, TimeUnit.SECONDS) - .build() - .start(); - } catch (IOException | InterruptedException e) { - throw new ConfigurationException(e); - } - - return embeddedElastic; - } - - @PreDestroy - public void stop() { - embeddedElasticSearch().stop(); - } - -} diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java index cc6d90d990e..cf0aba3e92c 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java @@ -33,6 +33,10 @@ public class ContextHolder { private static String ourPath; private static Long ourReuseSearchResultsMillis; private static String ourDatabaseUrl; + private static boolean myExternalElasticsearch = false; + private static boolean myPostGreSql = false; + private static Integer myDefaultPageSize = 10; + private static Integer myMaxPageSize = 50; static { ourReuseSearchResultsMillis = DaoConfig.DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS; @@ -100,4 +104,37 @@ public class ContextHolder { public static void setDatabaseUrl(String theDatabaseUrl) { ourDatabaseUrl = theDatabaseUrl; } + + public static void setExternalElasticsearch(Boolean theExternalElasticsearch) { + myExternalElasticsearch = theExternalElasticsearch; + } + + public static Boolean isExternalElasticsearch() { + return myExternalElasticsearch; + } + + public static void setPostgreSql(boolean thePostGreSql) { + myPostGreSql = thePostGreSql; + } + + public static boolean isPostGreSql() { + return myPostGreSql; + } + + public static void setDefaultPageSize(Integer theDefaultPageSize) { + myDefaultPageSize = theDefaultPageSize; + } + + public static Integer getDefaultPageSize() { + return myDefaultPageSize; + } + + public static void setMaxPageSize(Integer theMaxPageSize) { + myMaxPageSize = theMaxPageSize; + } + + public static Integer getMaxPageSize() { + return myMaxPageSize; + } + } diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextPostgreSQLHolder.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextPostgreSQLHolder.java deleted file mode 100644 index f851057a083..00000000000 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextPostgreSQLHolder.java +++ /dev/null @@ -1,89 +0,0 @@ -package ca.uhn.fhir.jpa.demo; - -/*- - * #%L - * HAPI FHIR - Command Line Client - Server WAR - * %% - * Copyright (C) 2014 - 2020 University Health Network - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -public class ContextPostgreSQLHolder extends ContextHolder { - -// private static String myDbUsername; -// private static String myDbPassword; - private static boolean myExternalElasticsearch = false; -// private static String myElasticsearchHost; -// private static Integer myElasticsearchPort; -// private static String myElasticsearchUsername; -// private static String myElasticsearchPassword; - -/* public static void setDbUsername(String theDbUsername) { - myDbUsername = theDbUsername; - } - - public static String getDbUsername() { - return myDbUsername; - } - - public static void setDbPassword(String theDbPassword) { - myDbPassword = theDbPassword; - } - - public static String getDbPassword() { - return myDbPassword; - } -*/ - public static void setExternalElasticsearch(Boolean theExternalElasticsearch) { - myExternalElasticsearch = theExternalElasticsearch; - } - - public static Boolean isExternalElasticsearch() { - return myExternalElasticsearch; - } -/* - public static void setElasticsearchHost(String theElasticsearchHost) { - myElasticsearchHost = theElasticsearchHost; - } - - public static String getElasticsearchHost() { - return myElasticsearchHost; - } - - public static void setElasticsearchPort(Integer theElasticsearchPort) { - myElasticsearchPort = theElasticsearchPort; - } - - public static Integer getElasticsearchPort() { - return myElasticsearchPort; - } - - public static void setElasticsearchUsername(String theElasticsearchUsername) { - myElasticsearchUsername = theElasticsearchUsername; - } - - public static String getElasticsearchUsername() { - return myElasticsearchUsername; - } - - public static void setElasticsearchPassword(String theElasticsearchPassword) { - myElasticsearchPassword = theElasticsearchPassword; - } - - public static String getElasticsearchPassword() { - return myElasticsearchPassword; - } -*/ -} diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerElasticsearchConfigR4.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerElasticsearchConfigR4.java deleted file mode 100644 index 59534483dc8..00000000000 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerElasticsearchConfigR4.java +++ /dev/null @@ -1,107 +0,0 @@ -package ca.uhn.fhir.jpa.demo; - -/*- - * #%L - * HAPI FHIR - Command Line Client - Server WAR - * %% - * Copyright (C) 2014 - 2020 University Health Network - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.jpa.config.BaseJavaConfigR4; -import ca.uhn.fhir.jpa.util.SubscriptionsRequireManualActivationInterceptorR4; -import ca.uhn.fhir.rest.server.interceptor.IServerInterceptor; -import ca.uhn.fhir.rest.server.interceptor.LoggingInterceptor; -import ca.uhn.fhir.rest.server.interceptor.ResponseHighlighterInterceptor; -import org.springframework.beans.factory.annotation.Autowire; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.orm.jpa.JpaTransactionManager; -import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; -import java.util.Properties; - -/** - * This class isn't used by default by the example, but - * you can use it as a config if you want to support DSTU3 - * instead of DSTU2 in your server. - *

- * See https://github.com/jamesagnew/hapi-fhir/issues/278 - */ -@Configuration -@EnableTransactionManagement() -@Import(CommonPostgreSQLConfig.class) -public class FhirServerElasticsearchConfigR4 extends BaseJavaConfigR4 { - - @Autowired - private DataSource myDataSource; - @Autowired() - @Qualifier("jpaProperties") - private Properties myJpaProperties; - - @Override - @Bean - public LocalContainerEntityManagerFactoryBean entityManagerFactory() { - LocalContainerEntityManagerFactoryBean retVal = super.entityManagerFactory(); - retVal.setPersistenceUnitName("HAPI_PU"); - retVal.setDataSource(myDataSource); - retVal.setJpaProperties(myJpaProperties); - return retVal; - } - - /** - * Do some fancy logging to create a nice access log that has details about each incoming request. - * @return - */ - public LoggingInterceptor loggingInterceptor() { - LoggingInterceptor retVal = new LoggingInterceptor(); - retVal.setLoggerName("fhirtest.access"); - retVal.setMessageFormat( - "Path[${servletPath}] Source[${requestHeader.x-forwarded-for}] Operation[${operationType} ${operationName} ${idOrResourceName}] UA[${requestHeader.user-agent}] Params[${requestParameters}] ResponseEncoding[${responseEncodingNoDefault}]"); - retVal.setLogExceptions(true); - retVal.setErrorMessageFormat("ERROR - ${requestVerb} ${requestUrl}"); - return retVal; - } - - /** - * This interceptor adds some pretty syntax highlighting in responses when a browser is detected - * @return - */ - @Bean(autowire = Autowire.BY_TYPE) - public ResponseHighlighterInterceptor responseHighlighterInterceptor() { - ResponseHighlighterInterceptor retVal = new ResponseHighlighterInterceptor(); - return retVal; - } - - @Bean(autowire = Autowire.BY_TYPE) - public IServerInterceptor subscriptionSecurityInterceptor() { - SubscriptionsRequireManualActivationInterceptorR4 retVal = new SubscriptionsRequireManualActivationInterceptorR4(); - return retVal; - } - - @Bean - public JpaTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { - JpaTransactionManager retVal = new JpaTransactionManager(); - retVal.setEntityManagerFactory(entityManagerFactory); - return retVal; - } - -} diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java index 6f63fdd5a0e..86d2bdadc6f 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java @@ -160,7 +160,8 @@ public class JpaServerDemo extends RestfulServer { /* * This is a simple paging strategy that keeps the last 10 searches in memory */ - setPagingProvider(new FifoMemoryPagingProvider(10)); + // TODO: Make this configurable via the ContextHolder + setPagingProvider(new FifoMemoryPagingProvider(10).setDefaultPageSize(ContextHolder.getDefaultPageSize()).setMaximumPageSize(ContextHolder.getMaxPageSize())); // Register a CORS filter CorsInterceptor corsInterceptor = new CorsInterceptor(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index ebccb7a7696..18aa3098b2f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -58,6 +58,10 @@ public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDao executeLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, IdHelperService theIdHelperService) { Integer myMaxObservationsPerCode = 1; - String[] maxCountParams = theRequestDetails.getParameters().get("map"); + String[] maxCountParams = theRequestDetails.getParameters().get("max"); if (maxCountParams != null && maxCountParams.length > 0) { myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]); } @@ -237,6 +237,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { compositeAggSubjectSources.add(subjectValuesBuilder); CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder("group_by_subject", compositeAggSubjectSources); compositeAggregationSubjectBuilder.subAggregation(observationCodeAggregationBuilder); + compositeAggregationSubjectBuilder.size(theMaximumResultSetSize); return compositeAggregationSubjectBuilder; } @@ -286,7 +287,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { List subjectBuckets = aggregatedSubjects.getBuckets(); List myObservationIds = new ArrayList<>(); for(ParsedComposite.ParsedBucket subjectBucket : subjectBuckets) { - Aggregations observationCodeAggregations = subjectBucket.getAggregations();ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); + Aggregations observationCodeAggregations = subjectBucket.getAggregations(); + ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java index f3eabc1ba23..9413e5ae26d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java @@ -116,6 +116,7 @@ public class IntegratedObservationIndexedSearchParamLastNTest { myObservation.setCategory(categoryConcepts); // Create CodeableConcept for Code with three codings. + // TODO: Temporarily limit this to two codings until we sort out how to manage multiple codings String observationCodeText = "Test Codeable Concept Field for Code"; CodeableConcept codeableConceptField = new CodeableConcept().setText(observationCodeText); codeableConceptField.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test-code display")); @@ -224,7 +225,7 @@ public class IntegratedObservationIndexedSearchParamLastNTest { } @Test - public void testSampleBundle() { + public void testSampleBundleInTransaction() throws IOException { FhirContext myFhirCtx = FhirContext.forR4(); PathMatchingResourcePatternResolver provider = new PathMatchingResourcePatternResolver(); @@ -274,6 +275,23 @@ public class IntegratedObservationIndexedSearchParamLastNTest { } ); + SearchParameterMap searchParameterMap = new SearchParameterMap(); + + // execute Observation ID search - Composite Aggregation + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 1); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + + assertEquals(20, observationIdsOnly.size()); + ObservationJson observationIdOnly = observationIdsOnly.get(0); + + searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); + responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + + assertEquals(38, observationIdsOnly.size()); + observationIdOnly = observationIdsOnly.get(0); + } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java new file mode 100644 index 00000000000..86f54337192 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java @@ -0,0 +1,150 @@ +package ca.uhn.fhir.jpa.dao.r4; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.interceptor.executor.InterceptorService; +import ca.uhn.fhir.jpa.dao.*; +import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl.Suggestion; +import ca.uhn.fhir.jpa.dao.lastn.config.TestIntegratedObservationIndexSearchConfig; +import ca.uhn.fhir.jpa.model.util.JpaConstants; +import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.param.*; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.util.BundleUtil; +import ca.uhn.fhir.util.TestUtil; +import org.hl7.fhir.instance.model.api.IBaseBundle; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.r4.model.Observation.ObservationStatus; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.servlet.http.HttpServletRequest; +import java.util.*; + +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestIntegratedObservationIndexSearchConfig.class }) +public class FhirResourceDaoR4LastNTest extends BaseJpaTest { + + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4LastNTest.class); + + @Autowired + @Qualifier("myPatientDaoR4") + protected IFhirResourceDaoPatient myPatientDao; + + @Autowired + @Qualifier("myObservationDaoR4") + protected IFhirResourceDaoObservation myObservationDao; + + @Autowired + protected DaoConfig myDaoConfig; + + @Autowired + protected FhirContext myFhirCtx; + + @Autowired + protected PlatformTransactionManager myPlatformTransactionManager; + + @Override + protected FhirContext getContext() { + return myFhirCtx; + } + + @Override + protected PlatformTransactionManager getTxManager() { + return myPlatformTransactionManager; + } + + @Before + public void beforeDisableResultReuse() { + myDaoConfig.setReuseCachedSearchResultsForMillis(null); + } + + private ServletRequestDetails mockSrd() { + return mySrd; + } + + @Test + public void testLastN() { + Patient pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Arthur"); + IIdType ptId = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + + Map requestParameters = new HashMap<>(); + String[] maxParam = new String[1]; + maxParam[0] = "1"; + requestParameters.put("max", maxParam); + when(mySrd.getParameters()).thenReturn(requestParameters); + + Map observationIds = new HashMap<>(); + for(int observationIdx = 0 ; observationIdx < 20 ; observationIdx++) { + Calendar observationDate = new GregorianCalendar(); + String idxSuffix = String.valueOf(observationIdx); + + Observation obs = new Observation(); + obs.getText().setDivAsString("

OBSTEXT0_" + idxSuffix + "
"); + obs.getSubject().setReferenceElement(ptId); + obs.getCode().addCoding().setCode("CODE_" + idxSuffix).setSystem("http://mycode.com"); + obs.setValue(new StringType("obsvalue0_" + idxSuffix)); + observationDate.add(Calendar.HOUR, -2); + Date effectiveDtm = observationDate.getTime(); + obs.setEffective(new DateTimeType(effectiveDtm)); + observationIds.put(myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless(), obs.getCode()); + + obs = new Observation(); + obs.getText().setDivAsString("
OBSTEXT1_" + idxSuffix + "
"); + obs.getSubject().setReferenceElement(ptId); + obs.getCode().addCoding().setCode("CODE_" + idxSuffix).setSystem("http://mycode.com"); + obs.setValue(new StringType("obsvalue1_" + idxSuffix)); + observationDate.add(Calendar.HOUR, -1); + effectiveDtm = observationDate.getTime(); + obs.setEffective(new DateTimeType(effectiveDtm)); + observationIds.put(myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless(), obs.getCode()); + + obs = new Observation(); + obs.getText().setDivAsString("
OBSTEXT2_" + idxSuffix + "
"); + obs.getSubject().setReferenceElement(ptId); + obs.getCode().addCoding().setCode("CODE_" + idxSuffix).setSystem("http://mycode.com"); + obs.setValue(new StringType("obsvalue2_" + idxSuffix)); + observationDate.add(Calendar.HOUR, -0); + effectiveDtm = observationDate.getTime(); + obs.setEffective(new DateTimeType(effectiveDtm)); + observationIds.put(myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless(), obs.getCode()); + } + + HttpServletRequest request; + List actual; + SearchParameterMap params = new SearchParameterMap(); + params.setLastN(true); + actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + + assertEquals(20, actual.size()); + + } + + @AfterClass + public static void afterClassClearContext() { + TestUtil.clearAllStaticFieldsForUnitTest(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java index b893a73c1a0..58d69592b8d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java @@ -68,7 +68,7 @@ public class LastNElasticsearchSvcMultipleObservationsTest { validateQueryResponse(observationIdsOnly); - // execute Observation ID search (Terms Aggregation) last 3 observations for each patient + // execute Observation ID search (Composite Aggregation) last 3 observations for each patient searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, null, 3); responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); @@ -153,9 +153,9 @@ public class LastNElasticsearchSvcMultipleObservationsTest { TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add("code", codeParam); - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); assertEquals(10, observationIdsOnly.size()); @@ -172,9 +172,9 @@ public class LastNElasticsearchSvcMultipleObservationsTest { TokenParam codeParam = new TokenParam("test-code-1"); searchParameterMap.add("code", codeParam); - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); assertEquals(5, observationIdsOnly.size()); @@ -191,9 +191,9 @@ public class LastNElasticsearchSvcMultipleObservationsTest { TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); searchParameterMap.add("code", codeParam); - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); assertEquals(10, observationIdsOnly.size()); } @@ -210,9 +210,9 @@ public class LastNElasticsearchSvcMultipleObservationsTest { codeParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add("code", codeParam); - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); assertEquals(5, observationIdsOnly.size()); @@ -223,16 +223,18 @@ public class LastNElasticsearchSvcMultipleObservationsTest { String codeableConceptId1 = UUID.randomUUID().toString(); CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); - codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); - codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + // TODO: uncomment the following once there is a solution to supporting multiple codings for Observation Code +// codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); +// codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1); String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1); String codeableConceptId2 = UUID.randomUUID().toString(); CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); - codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); - codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + // TODO: uncomment the following once there is a solution to supporting multiple codings for Observation Code +// codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); +// codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2); String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java index 70886b8d23c..59011bc48f8 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java @@ -285,28 +285,36 @@ public class LastNElasticsearchSvcSingleObservationTest { assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText); List persistedCodeCodingSystems = persistedObservationCode.getCoding_system(); - assertEquals(3,persistedCodeCodingSystems.size()); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. +// assertEquals(3,persistedCodeCodingSystems.size()); + assertEquals(1,persistedCodeCodingSystems.size()); assertEquals(CODEFIRSTCODINGSYSTEM, persistedCodeCodingSystems.get(0)); - assertEquals(CODESECONDCODINGSYSTEM, persistedCodeCodingSystems.get(1)); - assertEquals(CODETHIRDCODINGSYSTEM, persistedCodeCodingSystems.get(2)); +// assertEquals(CODESECONDCODINGSYSTEM, persistedCodeCodingSystems.get(1)); +// assertEquals(CODETHIRDCODINGSYSTEM, persistedCodeCodingSystems.get(2)); List persistedCodeCodingCodes = persistedObservationCode.getCoding_code(); - assertEquals(3, persistedCodeCodingCodes.size()); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. +// assertEquals(3, persistedCodeCodingCodes.size()); + assertEquals(1, persistedCodeCodingCodes.size()); assertEquals(CODEFIRSTCODINGCODE, persistedCodeCodingCodes.get(0)); - assertEquals(CODESECONDCODINGCODE, persistedCodeCodingCodes.get(1)); - assertEquals(CODETHIRDCODINGCODE, persistedCodeCodingCodes.get(2)); +// assertEquals(CODESECONDCODINGCODE, persistedCodeCodingCodes.get(1)); +// assertEquals(CODETHIRDCODINGCODE, persistedCodeCodingCodes.get(2)); List persistedCodeCodingDisplays = persistedObservationCode.getCoding_display(); - assertEquals(3, persistedCodeCodingDisplays.size()); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. +// assertEquals(3, persistedCodeCodingDisplays.size()); + assertEquals(1, persistedCodeCodingDisplays.size()); assertEquals(CODEFIRSTCODINGDISPLAY, persistedCodeCodingDisplays.get(0)); - assertEquals(CODESECONDCODINGDISPLAY, persistedCodeCodingDisplays.get(1)); - assertEquals(CODETHIRDCODINGDISPLAY, persistedCodeCodingDisplays.get(2)); +// assertEquals(CODESECONDCODINGDISPLAY, persistedCodeCodingDisplays.get(1)); +// assertEquals(CODETHIRDCODINGDISPLAY, persistedCodeCodingDisplays.get(2)); List persistedCodeCodingCodeSystemHashes = persistedObservationCode.getCoding_code_system_hash(); - assertEquals(3, persistedCodeCodingCodeSystemHashes.size()); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. +// assertEquals(3, persistedCodeCodingCodeSystemHashes.size()); + assertEquals(1, persistedCodeCodingCodeSystemHashes.size()); assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(2)); +// assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(1)); +// assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(2)); } @@ -349,8 +357,9 @@ public class LastNElasticsearchSvcSingleObservationTest { indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID); CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT); codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY)); - codeableConceptField.addCoding(new Coding(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE, CODESECONDCODINGDISPLAY)); - codeableConceptField.addCoding(new Coding(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE, CODETHIRDCODINGDISPLAY)); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. +// codeableConceptField.addCoding(new Coding(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE, CODESECONDCODINGDISPLAY)); +// codeableConceptField.addCoding(new Coding(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE, CODETHIRDCODINGDISPLAY)); indexedObservation.setCode(codeableConceptField); String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); diff --git a/hapi-fhir-jpaserver-base/src/test/resources/lastntestbundle.json b/hapi-fhir-jpaserver-base/src/test/resources/lastntestbundle.json new file mode 100644 index 00000000000..fee68b5d8f0 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/resources/lastntestbundle.json @@ -0,0 +1,4717 @@ +{ + "resourceType": "Bundle", + "type": "transaction", + "entry": [ + { + "fullUrl": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69", + "resource": { + "resourceType": "Patient", + "id": "8ea0ad2c-db87-4198-91ac-57608b314b69", + "text": { + "status": "generated", + "div": "
Generated by Synthea.Version identifier: v2.5.0-210-g1bfa8275\n . Person seed: -3790280920277372548 Population seed: 1581444392913
" + }, + "extension": [ + { + "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-race", + "extension": [ + { + "url": "ombCategory", + "valueCoding": { + "system": "urn:oid:2.16.840.1.113883.6.238", + "code": "2106-3", + "display": "White" + } + }, + { + "url": "text", + "valueString": "White" + } + ] + }, + { + "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity", + "extension": [ + { + "url": "ombCategory", + "valueCoding": { + "system": "urn:oid:2.16.840.1.113883.6.238", + "code": "2186-5", + "display": "Not Hispanic or Latino" + } + }, + { + "url": "text", + "valueString": "Not Hispanic or Latino" + } + ] + }, + { + "url": "http://hl7.org/fhir/StructureDefinition/patient-mothersMaidenName", + "valueString": "Dorine689 Stokes453" + }, + { + "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-birthsex", + "valueCode": "M" + }, + { + "url": "http://hl7.org/fhir/StructureDefinition/patient-birthPlace", + "valueAddress": { + "city": "Brockton", + "state": "Massachusetts", + "country": "US" + } + }, + { + "url": "http://synthetichealth.github.io/synthea/disability-adjusted-life-years", + "valueDecimal": 0.0 + }, + { + "url": "http://synthetichealth.github.io/synthea/quality-adjusted-life-years", + "valueDecimal": 0.0 + } + ], + "identifier": [ + { + "system": "https://github.com/synthetichealth/synthea", + "value": "8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + { + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "MR", + "display": "Medical Record Number" + } + ], + "text": "Medical Record Number" + }, + "system": "http://hospital.smarthealthit.org", + "value": "8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + { + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "SS", + "display": "Social Security Number" + } + ], + "text": "Social Security Number" + }, + "system": "http://hl7.org/fhir/sid/us-ssn", + "value": "999-46-8998" + } + ], + "name": [ + { + "use": "official", + "family": "Ritchie586", + "given": [ + "David908" + ] + } + ], + "telecom": [ + { + "system": "phone", + "value": "555-368-9896", + "use": "home" + } + ], + "gender": "male", + "birthDate": "2019-09-28", + "address": [ + { + "extension": [ + { + "url": "http://hl7.org/fhir/StructureDefinition/geolocation", + "extension": [ + { + "url": "latitude", + "valueDecimal": 41.92789793124249 + }, + { + "url": "longitude", + "valueDecimal": -70.65867748418283 + } + ] + } + ], + "line": [ + "655 Williamson Vista" + ], + "city": "Plymouth", + "state": "Massachusetts", + "country": "US" + } + ], + "maritalStatus": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v3-MaritalStatus", + "code": "S", + "display": "Never Married" + } + ], + "text": "Never Married" + }, + "multipleBirthBoolean": false, + "communication": [ + { + "language": { + "coding": [ + { + "system": "urn:ietf:bcp:47", + "code": "en-US", + "display": "English" + } + ], + "text": "English" + } + } + ] + }, + "request": { + "method": "POST", + "url": "Patient" + } + }, + { + "fullUrl": "urn:uuid:6157c34e-8d56-3512-8471-8f6314f7f248", + "resource": { + "resourceType": "Organization", + "id": "6157c34e-8d56-3512-8471-8f6314f7f248", + "identifier": [ + { + "system": "https://github.com/synthetichealth/synthea", + "value": "6157c34e-8d56-3512-8471-8f6314f7f248" + } + ], + "active": true, + "type": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/organization-type", + "code": "prov", + "display": "Healthcare Provider" + } + ], + "text": "Healthcare Provider" + } + ], + "name": "BAY STATE PHYSICAL THERAPY OF RANDOLPH PC", + "telecom": [ + { + "system": "phone", + "value": "508-830-0093" + } + ], + "address": [ + { + "line": [ + "16 ALDRIN RD" + ], + "city": "PLYMOUTH", + "state": "MA", + "postalCode": "02360-4804", + "country": "US" + } + ] + }, + "request": { + "method": "POST", + "url": "Organization" + } + }, + { + "fullUrl": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3", + "resource": { + "resourceType": "Practitioner", + "id": "b98d4677-b4e7-3541-a38e-7d0e573fb4a3", + "identifier": [ + { + "system": "http://hl7.org/fhir/sid/us-npi", + "value": "84860" + } + ], + "active": true, + "name": [ + { + "family": "Witting912", + "given": [ + "Ada662" + ], + "prefix": [ + "Dr." + ] + } + ], + "telecom": [ + { + "system": "email", + "value": "Ada662.Witting912@example.com", + "use": "work" + } + ], + "address": [ + { + "line": [ + "16 ALDRIN RD" + ], + "city": "PLYMOUTH", + "state": "MA", + "postalCode": "02360-4804", + "country": "US" + } + ], + "gender": "female" + }, + "request": { + "method": "POST", + "url": "Practitioner" + } + }, + { + "fullUrl": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7", + "resource": { + "resourceType": "Encounter", + "id": "ac848b54-772f-44be-a5b1-b5757330e6d7", + "status": "finished", + "class": { + "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode", + "code": "AMB" + }, + "type": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "410620009", + "display": "Well child visit (procedure)" + } + ], + "text": "Well child visit (procedure)" + } + ], + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69", + "display": "David908 Ritchie586" + }, + "participant": [ + { + "individual": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3", + "display": "Dr. Ada662 Witting912" + } + } + ], + "period": { + "start": "2019-09-28T15:53:49-04:00", + "end": "2019-09-28T16:08:49-04:00" + }, + "serviceProvider": { + "reference": "urn:uuid:6157c34e-8d56-3512-8471-8f6314f7f248", + "display": "BAY STATE PHYSICAL THERAPY OF RANDOLPH PC" + } + }, + "request": { + "method": "POST", + "url": "Encounter" + } + }, + { + "fullUrl": "urn:uuid:d5138f03-5ebc-4f1a-8855-bb613e5f9958", + "resource": { + "resourceType": "Observation", + "id": "d5138f03-5ebc-4f1a-8855-bb613e5f9958", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8302-2", + "display": "Body Height" + } + ], + "text": "Body Height" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 56.8, + "unit": "cm", + "system": "http://unitsofmeasure.org", + "code": "cm" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:8233f9ec-c3ad-454e-aeb9-40fcab9605c7", + "resource": { + "resourceType": "Observation", + "id": "8233f9ec-c3ad-454e-aeb9-40fcab9605c7", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "72514-3", + "display": "Pain severity - 0-10 verbal numeric rating [Score] - Reported" + } + ], + "text": "Pain severity - 0-10 verbal numeric rating [Score] - Reported" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 3, + "unit": "{score}", + "system": "http://unitsofmeasure.org", + "code": "{score}" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:58ef7a27-d929-41b1-86b6-bc6b652f78d7", + "resource": { + "resourceType": "Observation", + "id": "58ef7a27-d929-41b1-86b6-bc6b652f78d7", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "29463-7", + "display": "Body Weight" + } + ], + "text": "Body Weight" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 4.7, + "unit": "kg", + "system": "http://unitsofmeasure.org", + "code": "kg" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:a18c8cb4-53e5-4524-9abc-add762d0518c", + "resource": { + "resourceType": "Observation", + "id": "a18c8cb4-53e5-4524-9abc-add762d0518c", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "77606-2", + "display": "Weight-for-length Per age and sex" + } + ], + "text": "Weight-for-length Per age and sex" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 9.994, + "unit": "%", + "system": "http://unitsofmeasure.org", + "code": "%" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:227781b6-78ab-4aed-92f6-6ab77f0b7ada", + "resource": { + "resourceType": "Observation", + "id": "227781b6-78ab-4aed-92f6-6ab77f0b7ada", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "9843-4", + "display": "Head Occipital-frontal circumference" + } + ], + "text": "Head Occipital-frontal circumference" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 38.38, + "unit": "cm", + "system": "http://unitsofmeasure.org", + "code": "cm" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:2e9be57d-ed5e-4559-acd5-ab7572ac47de", + "resource": { + "resourceType": "Observation", + "id": "2e9be57d-ed5e-4559-acd5-ab7572ac47de", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "85354-9", + "display": "Blood Pressure" + } + ], + "text": "Blood Pressure" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "component": [ + { + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8462-4", + "display": "Diastolic Blood Pressure" + } + ], + "text": "Diastolic Blood Pressure" + }, + "valueQuantity": { + "value": 77, + "unit": "mm[Hg]", + "system": "http://unitsofmeasure.org", + "code": "mm[Hg]" + } + }, + { + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8480-6", + "display": "Systolic Blood Pressure" + } + ], + "text": "Systolic Blood Pressure" + }, + "valueQuantity": { + "value": 116, + "unit": "mm[Hg]", + "system": "http://unitsofmeasure.org", + "code": "mm[Hg]" + } + } + ] + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:92259409-3d33-4750-b0f0-f417d973ccf1", + "resource": { + "resourceType": "Observation", + "id": "92259409-3d33-4750-b0f0-f417d973ccf1", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8867-4", + "display": "Heart rate" + } + ], + "text": "Heart rate" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 74, + "unit": "/min", + "system": "http://unitsofmeasure.org", + "code": "/min" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:c5dcd90f-a37a-4382-a1d1-89480c2bcf49", + "resource": { + "resourceType": "Observation", + "id": "c5dcd90f-a37a-4382-a1d1-89480c2bcf49", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "9279-1", + "display": "Respiratory rate" + } + ], + "text": "Respiratory rate" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 15, + "unit": "/min", + "system": "http://unitsofmeasure.org", + "code": "/min" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:6f2f9a0b-93e7-4912-9f11-4060608fce5e", + "resource": { + "resourceType": "Observation", + "id": "6f2f9a0b-93e7-4912-9f11-4060608fce5e", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "6690-2", + "display": "Leukocytes [#/volume] in Blood by Automated count" + } + ], + "text": "Leukocytes [#/volume] in Blood by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 8.927, + "unit": "10*3/uL", + "system": "http://unitsofmeasure.org", + "code": "10*3/uL" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:5d9ed91e-60a2-4b0b-b622-26fda13421bd", + "resource": { + "resourceType": "Observation", + "id": "5d9ed91e-60a2-4b0b-b622-26fda13421bd", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "789-8", + "display": "Erythrocytes [#/volume] in Blood by Automated count" + } + ], + "text": "Erythrocytes [#/volume] in Blood by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 5.1186, + "unit": "10*6/uL", + "system": "http://unitsofmeasure.org", + "code": "10*6/uL" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:d17d378b-4c8e-4e3a-b4a9-259fc7889adc", + "resource": { + "resourceType": "Observation", + "id": "d17d378b-4c8e-4e3a-b4a9-259fc7889adc", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "718-7", + "display": "Hemoglobin [Mass/volume] in Blood" + } + ], + "text": "Hemoglobin [Mass/volume] in Blood" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 12.724, + "unit": "g/dL", + "system": "http://unitsofmeasure.org", + "code": "g/dL" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:49b5340f-5ee3-4959-bdbf-e0b06aa06d69", + "resource": { + "resourceType": "Observation", + "id": "49b5340f-5ee3-4959-bdbf-e0b06aa06d69", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "4544-3", + "display": "Hematocrit [Volume Fraction] of Blood by Automated count" + } + ], + "text": "Hematocrit [Volume Fraction] of Blood by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 44.072, + "unit": "%", + "system": "http://unitsofmeasure.org", + "code": "%" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:593c6ba3-3dfb-44aa-8323-1c72691f3c94", + "resource": { + "resourceType": "Observation", + "id": "593c6ba3-3dfb-44aa-8323-1c72691f3c94", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "787-2", + "display": "MCV [Entitic volume] by Automated count" + } + ], + "text": "MCV [Entitic volume] by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 93.504, + "unit": "fL", + "system": "http://unitsofmeasure.org", + "code": "fL" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:ab26fd5e-e72a-4d10-be37-637d1dcd8e63", + "resource": { + "resourceType": "Observation", + "id": "ab26fd5e-e72a-4d10-be37-637d1dcd8e63", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "785-6", + "display": "MCH [Entitic mass] by Automated count" + } + ], + "text": "MCH [Entitic mass] by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 32.026, + "unit": "pg", + "system": "http://unitsofmeasure.org", + "code": "pg" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:0c764cf6-2d7f-49ca-b7c2-64241b65c17f", + "resource": { + "resourceType": "Observation", + "id": "0c764cf6-2d7f-49ca-b7c2-64241b65c17f", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "786-4", + "display": "MCHC [Mass/volume] by Automated count" + } + ], + "text": "MCHC [Mass/volume] by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 35.13, + "unit": "g/dL", + "system": "http://unitsofmeasure.org", + "code": "g/dL" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:3a76d110-66df-4f66-9373-bfe90a557aa7", + "resource": { + "resourceType": "Observation", + "id": "3a76d110-66df-4f66-9373-bfe90a557aa7", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "21000-5", + "display": "Erythrocyte distribution width [Entitic volume] by Automated count" + } + ], + "text": "Erythrocyte distribution width [Entitic volume] by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 41.305, + "unit": "fL", + "system": "http://unitsofmeasure.org", + "code": "fL" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:c94a46e2-fd8d-4331-b500-ac9219f10949", + "resource": { + "resourceType": "Observation", + "id": "c94a46e2-fd8d-4331-b500-ac9219f10949", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "777-3", + "display": "Platelets [#/volume] in Blood by Automated count" + } + ], + "text": "Platelets [#/volume] in Blood by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 175.6, + "unit": "10*3/uL", + "system": "http://unitsofmeasure.org", + "code": "10*3/uL" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:41858c2b-7373-4085-b6a8-b1b9829a108a", + "resource": { + "resourceType": "Observation", + "id": "41858c2b-7373-4085-b6a8-b1b9829a108a", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "32207-3", + "display": "Platelet distribution width [Entitic volume] in Blood by Automated count" + } + ], + "text": "Platelet distribution width [Entitic volume] in Blood by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 254.23, + "unit": "fL", + "system": "http://unitsofmeasure.org", + "code": "fL" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:79db4c37-445e-4e0e-98ce-c98232654bd0", + "resource": { + "resourceType": "Observation", + "id": "79db4c37-445e-4e0e-98ce-c98232654bd0", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "32623-1", + "display": "Platelet mean volume [Entitic volume] in Blood by Automated count" + } + ], + "text": "Platelet mean volume [Entitic volume] in Blood by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueQuantity": { + "value": 10.091, + "unit": "fL", + "system": "http://unitsofmeasure.org", + "code": "fL" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:452801cf-c274-43a0-8097-0a5903609a99", + "resource": { + "resourceType": "Observation", + "id": "452801cf-c274-43a0-8097-0a5903609a99", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "survey", + "display": "survey" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "72166-2", + "display": "Tobacco smoking status NHIS" + } + ], + "text": "Tobacco smoking status NHIS" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "266919005", + "display": "Never smoker" + } + ], + "text": "Never smoker" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:e90d1150-8c4f-4138-ae88-7e32177dbeb3", + "resource": { + "resourceType": "Immunization", + "id": "e90d1150-8c4f-4138-ae88-7e32177dbeb3", + "status": "completed", + "vaccineCode": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "08", + "display": "Hep B, adolescent or pediatric" + } + ], + "text": "Hep B, adolescent or pediatric" + }, + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "occurrenceDateTime": "2019-09-28T15:53:49-04:00", + "primarySource": true + }, + "request": { + "method": "POST", + "url": "Immunization" + } + }, + { + "fullUrl": "urn:uuid:c11b52d3-0834-4ddd-8c25-7d8d30a93eb6", + "resource": { + "resourceType": "DiagnosticReport", + "id": "c11b52d3-0834-4ddd-8c25-7d8d30a93eb6", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v2-0074", + "code": "LAB", + "display": "Laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "58410-2", + "display": "Complete blood count (hemogram) panel - Blood by Automated count" + } + ], + "text": "Complete blood count (hemogram) panel - Blood by Automated count" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + }, + "effectiveDateTime": "2019-09-28T15:53:49-04:00", + "issued": "2019-09-28T15:53:49.871-04:00", + "result": [ + { + "reference": "urn:uuid:6f2f9a0b-93e7-4912-9f11-4060608fce5e", + "display": "Leukocytes [#/volume] in Blood by Automated count" + }, + { + "reference": "urn:uuid:5d9ed91e-60a2-4b0b-b622-26fda13421bd", + "display": "Erythrocytes [#/volume] in Blood by Automated count" + }, + { + "reference": "urn:uuid:d17d378b-4c8e-4e3a-b4a9-259fc7889adc", + "display": "Hemoglobin [Mass/volume] in Blood" + }, + { + "reference": "urn:uuid:49b5340f-5ee3-4959-bdbf-e0b06aa06d69", + "display": "Hematocrit [Volume Fraction] of Blood by Automated count" + }, + { + "reference": "urn:uuid:593c6ba3-3dfb-44aa-8323-1c72691f3c94", + "display": "MCV [Entitic volume] by Automated count" + }, + { + "reference": "urn:uuid:ab26fd5e-e72a-4d10-be37-637d1dcd8e63", + "display": "MCH [Entitic mass] by Automated count" + }, + { + "reference": "urn:uuid:0c764cf6-2d7f-49ca-b7c2-64241b65c17f", + "display": "MCHC [Mass/volume] by Automated count" + }, + { + "reference": "urn:uuid:3a76d110-66df-4f66-9373-bfe90a557aa7", + "display": "Erythrocyte distribution width [Entitic volume] by Automated count" + }, + { + "reference": "urn:uuid:c94a46e2-fd8d-4331-b500-ac9219f10949", + "display": "Platelets [#/volume] in Blood by Automated count" + }, + { + "reference": "urn:uuid:41858c2b-7373-4085-b6a8-b1b9829a108a", + "display": "Platelet distribution width [Entitic volume] in Blood by Automated count" + }, + { + "reference": "urn:uuid:79db4c37-445e-4e0e-98ce-c98232654bd0", + "display": "Platelet mean volume [Entitic volume] in Blood by Automated count" + } + ] + }, + "request": { + "method": "POST", + "url": "DiagnosticReport" + } + }, + { + "fullUrl": "urn:uuid:f19d5fec-cee6-4b1e-a9b1-4c318038d9f8", + "resource": { + "resourceType": "Claim", + "id": "f19d5fec-cee6-4b1e-a9b1-4c318038d9f8", + "status": "active", + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claim-type", + "code": "institutional" + } + ] + }, + "use": "claim", + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69", + "display": "David908 Ritchie586" + }, + "billablePeriod": { + "start": "2019-09-28T15:53:49-04:00", + "end": "2019-09-28T16:08:49-04:00" + }, + "created": "2019-09-28T16:08:49-04:00", + "provider": { + "reference": "urn:uuid:6157c34e-8d56-3512-8471-8f6314f7f248", + "display": "BAY STATE PHYSICAL THERAPY OF RANDOLPH PC" + }, + "priority": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/processpriority", + "code": "normal" + } + ] + }, + "supportingInfo": [ + { + "sequence": 1, + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claiminformationcategory", + "code": "info" + } + ] + }, + "valueReference": { + "reference": "urn:uuid:e90d1150-8c4f-4138-ae88-7e32177dbeb3" + } + } + ], + "insurance": [ + { + "sequence": 1, + "focal": true, + "coverage": { + "display": "Anthem" + } + } + ], + "item": [ + { + "sequence": 1, + "productOrService": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "410620009", + "display": "Well child visit (procedure)" + } + ], + "text": "Well child visit (procedure)" + }, + "encounter": [ + { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + } + ] + }, + { + "sequence": 2, + "informationSequence": [ + 1 + ], + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "08", + "display": "Hep B, adolescent or pediatric" + } + ], + "text": "Hep B, adolescent or pediatric" + }, + "net": { + "value": 140.52, + "currency": "USD" + } + } + ], + "total": { + "value": 129.16, + "currency": "USD" + } + }, + "request": { + "method": "POST", + "url": "Claim" + } + }, + { + "fullUrl": "urn:uuid:d738c588-3b6d-4651-af10-849c45d25247", + "resource": { + "resourceType": "ExplanationOfBenefit", + "id": "d738c588-3b6d-4651-af10-849c45d25247", + "contained": [ + { + "resourceType": "ServiceRequest", + "id": "referral", + "status": "completed", + "intent": "order", + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "requester": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + }, + "performer": [ + { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + } + ] + }, + { + "resourceType": "Coverage", + "id": "coverage", + "status": "active", + "type": { + "text": "Anthem" + }, + "beneficiary": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "payor": [ + { + "display": "Anthem" + } + ] + } + ], + "identifier": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/clm_id", + "value": "f19d5fec-cee6-4b1e-a9b1-4c318038d9f8" + }, + { + "system": "https://bluebutton.cms.gov/resources/identifier/claim-group", + "value": "99999999999" + } + ], + "status": "active", + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claim-type", + "code": "institutional" + } + ] + }, + "use": "claim", + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "billablePeriod": { + "start": "2019-09-28T16:08:49-04:00", + "end": "2020-09-28T16:08:49-04:00" + }, + "created": "2019-09-28T16:08:49-04:00", + "insurer": { + "display": "Anthem" + }, + "provider": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + }, + "referral": { + "reference": "#referral" + }, + "claim": { + "reference": "urn:uuid:f19d5fec-cee6-4b1e-a9b1-4c318038d9f8" + }, + "outcome": "complete", + "careTeam": [ + { + "sequence": 1, + "provider": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + }, + "role": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claimcareteamrole", + "code": "primary", + "display": "Primary Care Practitioner" + } + ] + } + } + ], + "insurance": [ + { + "focal": true, + "coverage": { + "reference": "#coverage", + "display": "Anthem" + } + } + ], + "item": [ + { + "sequence": 1, + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "410620009", + "display": "Well child visit (procedure)" + } + ], + "text": "Well child visit (procedure)" + }, + "servicedPeriod": { + "start": "2019-09-28T15:53:49-04:00", + "end": "2019-09-28T16:08:49-04:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "encounter": [ + { + "reference": "urn:uuid:ac848b54-772f-44be-a5b1-b5757330e6d7" + } + ] + }, + { + "sequence": 2, + "informationSequence": [ + 1 + ], + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "08", + "display": "Hep B, adolescent or pediatric" + } + ], + "text": "Hep B, adolescent or pediatric" + }, + "servicedPeriod": { + "start": "2019-09-28T15:53:49-04:00", + "end": "2019-09-28T16:08:49-04:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "net": { + "value": 140.52, + "currency": "USD" + }, + "adjudication": [ + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt", + "display": "Line Beneficiary Coinsurance Amount" + } + ] + }, + "amount": { + "value": 28.104000000000003, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt", + "display": "Line Provider Payment Amount" + } + ] + }, + "amount": { + "value": 112.41600000000001, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt", + "display": "Line Submitted Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt", + "display": "Line Allowed Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt", + "display": "Line Beneficiary Part B Deductible Amount" + } + ] + }, + "amount": { + "value": 0, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd", + "display": "Line Processing Indicator Code" + } + ] + } + } + ] + } + ], + "total": [ + { + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "submitted", + "display": "Submitted Amount" + } + ], + "text": "Submitted Amount" + }, + "amount": { + "value": 129.16, + "currency": "USD" + } + } + ], + "payment": { + "amount": { + "value": 112.41600000000001, + "currency": "USD" + } + } + }, + "request": { + "method": "POST", + "url": "ExplanationOfBenefit" + } + }, + { + "fullUrl": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769", + "resource": { + "resourceType": "Encounter", + "id": "814acd74-5d13-4acc-94dd-d21270db8769", + "status": "finished", + "class": { + "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode", + "code": "AMB" + }, + "type": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "410620009", + "display": "Well child visit (procedure)" + } + ], + "text": "Well child visit (procedure)" + } + ], + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69", + "display": "David908 Ritchie586" + }, + "participant": [ + { + "individual": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3", + "display": "Dr. Ada662 Witting912" + } + } + ], + "period": { + "start": "2019-11-02T15:53:49-04:00", + "end": "2019-11-02T16:23:49-04:00" + }, + "serviceProvider": { + "reference": "urn:uuid:6157c34e-8d56-3512-8471-8f6314f7f248", + "display": "BAY STATE PHYSICAL THERAPY OF RANDOLPH PC" + } + }, + "request": { + "method": "POST", + "url": "Encounter" + } + }, + { + "fullUrl": "urn:uuid:db9368a7-6e18-46d2-9ee5-5efd09fdbab4", + "resource": { + "resourceType": "Observation", + "id": "db9368a7-6e18-46d2-9ee5-5efd09fdbab4", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8302-2", + "display": "Body Height" + } + ], + "text": "Body Height" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "effectiveDateTime": "2019-11-02T15:53:49-04:00", + "issued": "2019-11-02T15:53:49.871-04:00", + "valueQuantity": { + "value": 60.7, + "unit": "cm", + "system": "http://unitsofmeasure.org", + "code": "cm" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:636ef1b9-62b1-45a2-850d-8961f2fa45be", + "resource": { + "resourceType": "Observation", + "id": "636ef1b9-62b1-45a2-850d-8961f2fa45be", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "72514-3", + "display": "Pain severity - 0-10 verbal numeric rating [Score] - Reported" + } + ], + "text": "Pain severity - 0-10 verbal numeric rating [Score] - Reported" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "effectiveDateTime": "2019-11-02T15:53:49-04:00", + "issued": "2019-11-02T15:53:49.871-04:00", + "valueQuantity": { + "value": 2, + "unit": "{score}", + "system": "http://unitsofmeasure.org", + "code": "{score}" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:d3b861fb-23f3-41e3-b7c4-bab9860434d6", + "resource": { + "resourceType": "Observation", + "id": "d3b861fb-23f3-41e3-b7c4-bab9860434d6", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "29463-7", + "display": "Body Weight" + } + ], + "text": "Body Weight" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "effectiveDateTime": "2019-11-02T15:53:49-04:00", + "issued": "2019-11-02T15:53:49.871-04:00", + "valueQuantity": { + "value": 5.7, + "unit": "kg", + "system": "http://unitsofmeasure.org", + "code": "kg" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:97f94626-13e3-4dca-abe7-b0628d4bdd3b", + "resource": { + "resourceType": "Observation", + "id": "97f94626-13e3-4dca-abe7-b0628d4bdd3b", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "77606-2", + "display": "Weight-for-length Per age and sex" + } + ], + "text": "Weight-for-length Per age and sex" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "effectiveDateTime": "2019-11-02T15:53:49-04:00", + "issued": "2019-11-02T15:53:49.871-04:00", + "valueQuantity": { + "value": 11.175, + "unit": "%", + "system": "http://unitsofmeasure.org", + "code": "%" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:e266363b-4570-4f97-90b5-ed1cd36a1457", + "resource": { + "resourceType": "Observation", + "id": "e266363b-4570-4f97-90b5-ed1cd36a1457", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "9843-4", + "display": "Head Occipital-frontal circumference" + } + ], + "text": "Head Occipital-frontal circumference" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "effectiveDateTime": "2019-11-02T15:53:49-04:00", + "issued": "2019-11-02T15:53:49.871-04:00", + "valueQuantity": { + "value": 41.5, + "unit": "cm", + "system": "http://unitsofmeasure.org", + "code": "cm" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:4cf0063c-d651-4f28-bf84-4c70bef13a23", + "resource": { + "resourceType": "Observation", + "id": "4cf0063c-d651-4f28-bf84-4c70bef13a23", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "85354-9", + "display": "Blood Pressure" + } + ], + "text": "Blood Pressure" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "effectiveDateTime": "2019-11-02T15:53:49-04:00", + "issued": "2019-11-02T15:53:49.871-04:00", + "component": [ + { + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8462-4", + "display": "Diastolic Blood Pressure" + } + ], + "text": "Diastolic Blood Pressure" + }, + "valueQuantity": { + "value": 78, + "unit": "mm[Hg]", + "system": "http://unitsofmeasure.org", + "code": "mm[Hg]" + } + }, + { + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8480-6", + "display": "Systolic Blood Pressure" + } + ], + "text": "Systolic Blood Pressure" + }, + "valueQuantity": { + "value": 125, + "unit": "mm[Hg]", + "system": "http://unitsofmeasure.org", + "code": "mm[Hg]" + } + } + ] + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:814babff-8e0f-4ec4-b7cb-84f724f25442", + "resource": { + "resourceType": "Observation", + "id": "814babff-8e0f-4ec4-b7cb-84f724f25442", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8867-4", + "display": "Heart rate" + } + ], + "text": "Heart rate" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "effectiveDateTime": "2019-11-02T15:53:49-04:00", + "issued": "2019-11-02T15:53:49.871-04:00", + "valueQuantity": { + "value": 68, + "unit": "/min", + "system": "http://unitsofmeasure.org", + "code": "/min" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:b44aaf1c-6aec-4d3f-8afc-73445af69726", + "resource": { + "resourceType": "Observation", + "id": "b44aaf1c-6aec-4d3f-8afc-73445af69726", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "9279-1", + "display": "Respiratory rate" + } + ], + "text": "Respiratory rate" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "effectiveDateTime": "2019-11-02T15:53:49-04:00", + "issued": "2019-11-02T15:53:49.871-04:00", + "valueQuantity": { + "value": 15, + "unit": "/min", + "system": "http://unitsofmeasure.org", + "code": "/min" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:6a23a333-b13a-49f5-97fa-bf5d22d819bd", + "resource": { + "resourceType": "Observation", + "id": "6a23a333-b13a-49f5-97fa-bf5d22d819bd", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "survey", + "display": "survey" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "72166-2", + "display": "Tobacco smoking status NHIS" + } + ], + "text": "Tobacco smoking status NHIS" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "effectiveDateTime": "2019-11-02T15:53:49-04:00", + "issued": "2019-11-02T15:53:49.871-04:00", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "266919005", + "display": "Never smoker" + } + ], + "text": "Never smoker" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:65e4510f-749c-4862-8699-e6442ce5b0d3", + "resource": { + "resourceType": "Procedure", + "id": "65e4510f-749c-4862-8699-e6442ce5b0d3", + "status": "completed", + "code": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "430193006", + "display": "Medication Reconciliation (procedure)" + } + ], + "text": "Medication Reconciliation (procedure)" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "performedPeriod": { + "start": "2019-11-02T15:53:49-04:00", + "end": "2019-11-02T16:08:49-04:00" + } + }, + "request": { + "method": "POST", + "url": "Procedure" + } + }, + { + "fullUrl": "urn:uuid:d795547e-b179-4f25-802b-e5081c07d23c", + "resource": { + "resourceType": "Immunization", + "id": "d795547e-b179-4f25-802b-e5081c07d23c", + "status": "completed", + "vaccineCode": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "08", + "display": "Hep B, adolescent or pediatric" + } + ], + "text": "Hep B, adolescent or pediatric" + }, + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + }, + "occurrenceDateTime": "2019-11-02T15:53:49-04:00", + "primarySource": true + }, + "request": { + "method": "POST", + "url": "Immunization" + } + }, + { + "fullUrl": "urn:uuid:3e2e0555-74cf-44c3-9839-49a4a0eea61c", + "resource": { + "resourceType": "Claim", + "id": "3e2e0555-74cf-44c3-9839-49a4a0eea61c", + "status": "active", + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claim-type", + "code": "institutional" + } + ] + }, + "use": "claim", + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69", + "display": "David908 Ritchie586" + }, + "billablePeriod": { + "start": "2019-11-02T15:53:49-04:00", + "end": "2019-11-02T16:23:49-04:00" + }, + "created": "2019-11-02T16:23:49-04:00", + "provider": { + "reference": "urn:uuid:6157c34e-8d56-3512-8471-8f6314f7f248", + "display": "BAY STATE PHYSICAL THERAPY OF RANDOLPH PC" + }, + "priority": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/processpriority", + "code": "normal" + } + ] + }, + "supportingInfo": [ + { + "sequence": 1, + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claiminformationcategory", + "code": "info" + } + ] + }, + "valueReference": { + "reference": "urn:uuid:d795547e-b179-4f25-802b-e5081c07d23c" + } + } + ], + "procedure": [ + { + "sequence": 1, + "procedureReference": { + "reference": "urn:uuid:65e4510f-749c-4862-8699-e6442ce5b0d3" + } + } + ], + "insurance": [ + { + "sequence": 1, + "focal": true, + "coverage": { + "display": "Anthem" + } + } + ], + "item": [ + { + "sequence": 1, + "productOrService": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "410620009", + "display": "Well child visit (procedure)" + } + ], + "text": "Well child visit (procedure)" + }, + "encounter": [ + { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + } + ] + }, + { + "sequence": 2, + "informationSequence": [ + 1 + ], + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "08", + "display": "Hep B, adolescent or pediatric" + } + ], + "text": "Hep B, adolescent or pediatric" + }, + "net": { + "value": 140.52, + "currency": "USD" + } + }, + { + "sequence": 3, + "procedureSequence": [ + 1 + ], + "productOrService": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "430193006", + "display": "Medication Reconciliation (procedure)" + } + ], + "text": "Medication Reconciliation (procedure)" + }, + "net": { + "value": 545.27, + "currency": "USD" + } + } + ], + "total": { + "value": 129.16, + "currency": "USD" + } + }, + "request": { + "method": "POST", + "url": "Claim" + } + }, + { + "fullUrl": "urn:uuid:1cba9ebf-a18f-4a30-b2af-5497d02be506", + "resource": { + "resourceType": "ExplanationOfBenefit", + "id": "1cba9ebf-a18f-4a30-b2af-5497d02be506", + "contained": [ + { + "resourceType": "ServiceRequest", + "id": "referral", + "status": "completed", + "intent": "order", + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "requester": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + }, + "performer": [ + { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + } + ] + }, + { + "resourceType": "Coverage", + "id": "coverage", + "status": "active", + "type": { + "text": "Anthem" + }, + "beneficiary": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "payor": [ + { + "display": "Anthem" + } + ] + } + ], + "identifier": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/clm_id", + "value": "3e2e0555-74cf-44c3-9839-49a4a0eea61c" + }, + { + "system": "https://bluebutton.cms.gov/resources/identifier/claim-group", + "value": "99999999999" + } + ], + "status": "active", + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claim-type", + "code": "institutional" + } + ] + }, + "use": "claim", + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "billablePeriod": { + "start": "2019-11-02T16:23:49-04:00", + "end": "2020-11-02T16:23:49-05:00" + }, + "created": "2019-11-02T16:23:49-04:00", + "insurer": { + "display": "Anthem" + }, + "provider": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + }, + "referral": { + "reference": "#referral" + }, + "claim": { + "reference": "urn:uuid:3e2e0555-74cf-44c3-9839-49a4a0eea61c" + }, + "outcome": "complete", + "careTeam": [ + { + "sequence": 1, + "provider": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + }, + "role": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claimcareteamrole", + "code": "primary", + "display": "Primary Care Practitioner" + } + ] + } + } + ], + "insurance": [ + { + "focal": true, + "coverage": { + "reference": "#coverage", + "display": "Anthem" + } + } + ], + "item": [ + { + "sequence": 1, + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "410620009", + "display": "Well child visit (procedure)" + } + ], + "text": "Well child visit (procedure)" + }, + "servicedPeriod": { + "start": "2019-11-02T15:53:49-04:00", + "end": "2019-11-02T16:23:49-04:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "encounter": [ + { + "reference": "urn:uuid:814acd74-5d13-4acc-94dd-d21270db8769" + } + ] + }, + { + "sequence": 2, + "informationSequence": [ + 1 + ], + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "08", + "display": "Hep B, adolescent or pediatric" + } + ], + "text": "Hep B, adolescent or pediatric" + }, + "servicedPeriod": { + "start": "2019-11-02T15:53:49-04:00", + "end": "2019-11-02T16:23:49-04:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "net": { + "value": 140.52, + "currency": "USD" + }, + "adjudication": [ + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt", + "display": "Line Beneficiary Coinsurance Amount" + } + ] + }, + "amount": { + "value": 28.104000000000003, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt", + "display": "Line Provider Payment Amount" + } + ] + }, + "amount": { + "value": 112.41600000000001, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt", + "display": "Line Submitted Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt", + "display": "Line Allowed Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt", + "display": "Line Beneficiary Part B Deductible Amount" + } + ] + }, + "amount": { + "value": 0, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd", + "display": "Line Processing Indicator Code" + } + ] + } + } + ] + }, + { + "sequence": 3, + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "430193006", + "display": "Medication Reconciliation (procedure)" + } + ], + "text": "Medication Reconciliation (procedure)" + }, + "servicedPeriod": { + "start": "2019-11-02T15:53:49-04:00", + "end": "2019-11-02T16:23:49-04:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "net": { + "value": 545.27, + "currency": "USD" + }, + "adjudication": [ + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt", + "display": "Line Beneficiary Coinsurance Amount" + } + ] + }, + "amount": { + "value": 109.054, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt", + "display": "Line Provider Payment Amount" + } + ] + }, + "amount": { + "value": 436.216, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt", + "display": "Line Submitted Charge Amount" + } + ] + }, + "amount": { + "value": 545.27, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt", + "display": "Line Allowed Charge Amount" + } + ] + }, + "amount": { + "value": 545.27, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt", + "display": "Line Beneficiary Part B Deductible Amount" + } + ] + }, + "amount": { + "value": 0, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd", + "display": "Line Processing Indicator Code" + } + ] + } + } + ] + } + ], + "total": [ + { + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "submitted", + "display": "Submitted Amount" + } + ], + "text": "Submitted Amount" + }, + "amount": { + "value": 129.16, + "currency": "USD" + } + } + ], + "payment": { + "amount": { + "value": 548.6320000000001, + "currency": "USD" + } + } + }, + "request": { + "method": "POST", + "url": "ExplanationOfBenefit" + } + }, + { + "fullUrl": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893", + "resource": { + "resourceType": "Encounter", + "id": "beb5bd2f-e787-4852-a541-0e6f88950893", + "status": "finished", + "class": { + "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode", + "code": "AMB" + }, + "type": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "410620009", + "display": "Well child visit (procedure)" + } + ], + "text": "Well child visit (procedure)" + } + ], + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69", + "display": "David908 Ritchie586" + }, + "participant": [ + { + "individual": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3", + "display": "Dr. Ada662 Witting912" + } + } + ], + "period": { + "start": "2020-01-04T14:53:49-05:00", + "end": "2020-01-04T15:08:49-05:00" + }, + "serviceProvider": { + "reference": "urn:uuid:6157c34e-8d56-3512-8471-8f6314f7f248", + "display": "BAY STATE PHYSICAL THERAPY OF RANDOLPH PC" + } + }, + "request": { + "method": "POST", + "url": "Encounter" + } + }, + { + "fullUrl": "urn:uuid:63c1a058-e25a-4fc3-9ea6-ad9c9a369874", + "resource": { + "resourceType": "Observation", + "id": "63c1a058-e25a-4fc3-9ea6-ad9c9a369874", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8302-2", + "display": "Body Height" + } + ], + "text": "Body Height" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "effectiveDateTime": "2020-01-04T14:53:49-05:00", + "issued": "2020-01-04T14:53:49.871-05:00", + "valueQuantity": { + "value": 66.3, + "unit": "cm", + "system": "http://unitsofmeasure.org", + "code": "cm" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:e68c91b8-cb78-4d84-9f25-3aa5d9f9d6af", + "resource": { + "resourceType": "Observation", + "id": "e68c91b8-cb78-4d84-9f25-3aa5d9f9d6af", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "72514-3", + "display": "Pain severity - 0-10 verbal numeric rating [Score] - Reported" + } + ], + "text": "Pain severity - 0-10 verbal numeric rating [Score] - Reported" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "effectiveDateTime": "2020-01-04T14:53:49-05:00", + "issued": "2020-01-04T14:53:49.871-05:00", + "valueQuantity": { + "value": 0, + "unit": "{score}", + "system": "http://unitsofmeasure.org", + "code": "{score}" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:55fcc028-e7b1-425b-b8b7-3b61628e0c41", + "resource": { + "resourceType": "Observation", + "id": "55fcc028-e7b1-425b-b8b7-3b61628e0c41", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "29463-7", + "display": "Body Weight" + } + ], + "text": "Body Weight" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "effectiveDateTime": "2020-01-04T14:53:49-05:00", + "issued": "2020-01-04T14:53:49.871-05:00", + "valueQuantity": { + "value": 7.4, + "unit": "kg", + "system": "http://unitsofmeasure.org", + "code": "kg" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:309e745a-58a4-4a75-a266-54892ad0396e", + "resource": { + "resourceType": "Observation", + "id": "309e745a-58a4-4a75-a266-54892ad0396e", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "77606-2", + "display": "Weight-for-length Per age and sex" + } + ], + "text": "Weight-for-length Per age and sex" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "effectiveDateTime": "2020-01-04T14:53:49-05:00", + "issued": "2020-01-04T14:53:49.871-05:00", + "valueQuantity": { + "value": 39.641, + "unit": "%", + "system": "http://unitsofmeasure.org", + "code": "%" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:835d9f97-4742-48ad-8760-07561fca0325", + "resource": { + "resourceType": "Observation", + "id": "835d9f97-4742-48ad-8760-07561fca0325", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "9843-4", + "display": "Head Occipital-frontal circumference" + } + ], + "text": "Head Occipital-frontal circumference" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "effectiveDateTime": "2020-01-04T14:53:49-05:00", + "issued": "2020-01-04T14:53:49.871-05:00", + "valueQuantity": { + "value": 43.9, + "unit": "cm", + "system": "http://unitsofmeasure.org", + "code": "cm" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:d00376c9-a549-42ad-aac0-c49188f769ea", + "resource": { + "resourceType": "Observation", + "id": "d00376c9-a549-42ad-aac0-c49188f769ea", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "85354-9", + "display": "Blood Pressure" + } + ], + "text": "Blood Pressure" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "effectiveDateTime": "2020-01-04T14:53:49-05:00", + "issued": "2020-01-04T14:53:49.871-05:00", + "component": [ + { + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8462-4", + "display": "Diastolic Blood Pressure" + } + ], + "text": "Diastolic Blood Pressure" + }, + "valueQuantity": { + "value": 81, + "unit": "mm[Hg]", + "system": "http://unitsofmeasure.org", + "code": "mm[Hg]" + } + }, + { + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8480-6", + "display": "Systolic Blood Pressure" + } + ], + "text": "Systolic Blood Pressure" + }, + "valueQuantity": { + "value": 112, + "unit": "mm[Hg]", + "system": "http://unitsofmeasure.org", + "code": "mm[Hg]" + } + } + ] + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:ee087a61-363d-4df1-9921-e009ff4a4aef", + "resource": { + "resourceType": "Observation", + "id": "ee087a61-363d-4df1-9921-e009ff4a4aef", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "8867-4", + "display": "Heart rate" + } + ], + "text": "Heart rate" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "effectiveDateTime": "2020-01-04T14:53:49-05:00", + "issued": "2020-01-04T14:53:49.871-05:00", + "valueQuantity": { + "value": 82, + "unit": "/min", + "system": "http://unitsofmeasure.org", + "code": "/min" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:a876e6d8-d652-4aea-a321-2f645a1084f4", + "resource": { + "resourceType": "Observation", + "id": "a876e6d8-d652-4aea-a321-2f645a1084f4", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display": "vital-signs" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "9279-1", + "display": "Respiratory rate" + } + ], + "text": "Respiratory rate" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "effectiveDateTime": "2020-01-04T14:53:49-05:00", + "issued": "2020-01-04T14:53:49.871-05:00", + "valueQuantity": { + "value": 14, + "unit": "/min", + "system": "http://unitsofmeasure.org", + "code": "/min" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:a80539a1-8df9-4576-8cea-9e201358453b", + "resource": { + "resourceType": "Observation", + "id": "a80539a1-8df9-4576-8cea-9e201358453b", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "survey", + "display": "survey" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "72166-2", + "display": "Tobacco smoking status NHIS" + } + ], + "text": "Tobacco smoking status NHIS" + }, + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "effectiveDateTime": "2020-01-04T14:53:49-05:00", + "issued": "2020-01-04T14:53:49.871-05:00", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "266919005", + "display": "Never smoker" + } + ], + "text": "Never smoker" + } + }, + "request": { + "method": "POST", + "url": "Observation" + } + }, + { + "fullUrl": "urn:uuid:9563a43c-20de-4877-9584-daf8876e1926", + "resource": { + "resourceType": "Immunization", + "id": "9563a43c-20de-4877-9584-daf8876e1926", + "status": "completed", + "vaccineCode": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "49", + "display": "Hib (PRP-OMP)" + } + ], + "text": "Hib (PRP-OMP)" + }, + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "occurrenceDateTime": "2020-01-04T14:53:49-05:00", + "primarySource": true + }, + "request": { + "method": "POST", + "url": "Immunization" + } + }, + { + "fullUrl": "urn:uuid:c8f088bf-ab61-46a5-9e23-7677f72d5560", + "resource": { + "resourceType": "Immunization", + "id": "c8f088bf-ab61-46a5-9e23-7677f72d5560", + "status": "completed", + "vaccineCode": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "119", + "display": "rotavirus, monovalent" + } + ], + "text": "rotavirus, monovalent" + }, + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "occurrenceDateTime": "2020-01-04T14:53:49-05:00", + "primarySource": true + }, + "request": { + "method": "POST", + "url": "Immunization" + } + }, + { + "fullUrl": "urn:uuid:61c6628d-2cd7-4104-8077-2dd9317e03ee", + "resource": { + "resourceType": "Immunization", + "id": "61c6628d-2cd7-4104-8077-2dd9317e03ee", + "status": "completed", + "vaccineCode": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "10", + "display": "IPV" + } + ], + "text": "IPV" + }, + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "occurrenceDateTime": "2020-01-04T14:53:49-05:00", + "primarySource": true + }, + "request": { + "method": "POST", + "url": "Immunization" + } + }, + { + "fullUrl": "urn:uuid:4b8c55c7-082c-4033-994b-e671e8532ab9", + "resource": { + "resourceType": "Immunization", + "id": "4b8c55c7-082c-4033-994b-e671e8532ab9", + "status": "completed", + "vaccineCode": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "20", + "display": "DTaP" + } + ], + "text": "DTaP" + }, + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "occurrenceDateTime": "2020-01-04T14:53:49-05:00", + "primarySource": true + }, + "request": { + "method": "POST", + "url": "Immunization" + } + }, + { + "fullUrl": "urn:uuid:0c672ca5-ae3f-4bce-9e95-1ab3a6a5d0c8", + "resource": { + "resourceType": "Immunization", + "id": "0c672ca5-ae3f-4bce-9e95-1ab3a6a5d0c8", + "status": "completed", + "vaccineCode": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "133", + "display": "Pneumococcal conjugate PCV 13" + } + ], + "text": "Pneumococcal conjugate PCV 13" + }, + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "encounter": { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + }, + "occurrenceDateTime": "2020-01-04T14:53:49-05:00", + "primarySource": true + }, + "request": { + "method": "POST", + "url": "Immunization" + } + }, + { + "fullUrl": "urn:uuid:6d13d9c3-cf11-4b93-bfbc-ceca024f682f", + "resource": { + "resourceType": "Claim", + "id": "6d13d9c3-cf11-4b93-bfbc-ceca024f682f", + "status": "active", + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claim-type", + "code": "institutional" + } + ] + }, + "use": "claim", + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69", + "display": "David908 Ritchie586" + }, + "billablePeriod": { + "start": "2020-01-04T14:53:49-05:00", + "end": "2020-01-04T15:08:49-05:00" + }, + "created": "2020-01-04T15:08:49-05:00", + "provider": { + "reference": "urn:uuid:6157c34e-8d56-3512-8471-8f6314f7f248", + "display": "BAY STATE PHYSICAL THERAPY OF RANDOLPH PC" + }, + "priority": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/processpriority", + "code": "normal" + } + ] + }, + "supportingInfo": [ + { + "sequence": 1, + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claiminformationcategory", + "code": "info" + } + ] + }, + "valueReference": { + "reference": "urn:uuid:9563a43c-20de-4877-9584-daf8876e1926" + } + }, + { + "sequence": 2, + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claiminformationcategory", + "code": "info" + } + ] + }, + "valueReference": { + "reference": "urn:uuid:c8f088bf-ab61-46a5-9e23-7677f72d5560" + } + }, + { + "sequence": 3, + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claiminformationcategory", + "code": "info" + } + ] + }, + "valueReference": { + "reference": "urn:uuid:61c6628d-2cd7-4104-8077-2dd9317e03ee" + } + }, + { + "sequence": 4, + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claiminformationcategory", + "code": "info" + } + ] + }, + "valueReference": { + "reference": "urn:uuid:4b8c55c7-082c-4033-994b-e671e8532ab9" + } + }, + { + "sequence": 5, + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claiminformationcategory", + "code": "info" + } + ] + }, + "valueReference": { + "reference": "urn:uuid:0c672ca5-ae3f-4bce-9e95-1ab3a6a5d0c8" + } + } + ], + "insurance": [ + { + "sequence": 1, + "focal": true, + "coverage": { + "display": "Anthem" + } + } + ], + "item": [ + { + "sequence": 1, + "productOrService": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "410620009", + "display": "Well child visit (procedure)" + } + ], + "text": "Well child visit (procedure)" + }, + "encounter": [ + { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + } + ] + }, + { + "sequence": 2, + "informationSequence": [ + 1 + ], + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "49", + "display": "Hib (PRP-OMP)" + } + ], + "text": "Hib (PRP-OMP)" + }, + "net": { + "value": 140.52, + "currency": "USD" + } + }, + { + "sequence": 3, + "informationSequence": [ + 2 + ], + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "119", + "display": "rotavirus, monovalent" + } + ], + "text": "rotavirus, monovalent" + }, + "net": { + "value": 140.52, + "currency": "USD" + } + }, + { + "sequence": 4, + "informationSequence": [ + 3 + ], + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "10", + "display": "IPV" + } + ], + "text": "IPV" + }, + "net": { + "value": 140.52, + "currency": "USD" + } + }, + { + "sequence": 5, + "informationSequence": [ + 4 + ], + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "20", + "display": "DTaP" + } + ], + "text": "DTaP" + }, + "net": { + "value": 140.52, + "currency": "USD" + } + }, + { + "sequence": 6, + "informationSequence": [ + 5 + ], + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "133", + "display": "Pneumococcal conjugate PCV 13" + } + ], + "text": "Pneumococcal conjugate PCV 13" + }, + "net": { + "value": 140.52, + "currency": "USD" + } + } + ], + "total": { + "value": 129.16, + "currency": "USD" + } + }, + "request": { + "method": "POST", + "url": "Claim" + } + }, + { + "fullUrl": "urn:uuid:f321cb98-4076-40e2-9609-351c64e6cf50", + "resource": { + "resourceType": "ExplanationOfBenefit", + "id": "f321cb98-4076-40e2-9609-351c64e6cf50", + "contained": [ + { + "resourceType": "ServiceRequest", + "id": "referral", + "status": "completed", + "intent": "order", + "subject": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "requester": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + }, + "performer": [ + { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + } + ] + }, + { + "resourceType": "Coverage", + "id": "coverage", + "status": "active", + "type": { + "text": "Anthem" + }, + "beneficiary": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "payor": [ + { + "display": "Anthem" + } + ] + } + ], + "identifier": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/clm_id", + "value": "6d13d9c3-cf11-4b93-bfbc-ceca024f682f" + }, + { + "system": "https://bluebutton.cms.gov/resources/identifier/claim-group", + "value": "99999999999" + } + ], + "status": "active", + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claim-type", + "code": "institutional" + } + ] + }, + "use": "claim", + "patient": { + "reference": "urn:uuid:8ea0ad2c-db87-4198-91ac-57608b314b69" + }, + "billablePeriod": { + "start": "2020-01-04T15:08:49-05:00", + "end": "2021-01-04T15:08:49-05:00" + }, + "created": "2020-01-04T15:08:49-05:00", + "insurer": { + "display": "Anthem" + }, + "provider": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + }, + "referral": { + "reference": "#referral" + }, + "claim": { + "reference": "urn:uuid:6d13d9c3-cf11-4b93-bfbc-ceca024f682f" + }, + "outcome": "complete", + "careTeam": [ + { + "sequence": 1, + "provider": { + "reference": "urn:uuid:b98d4677-b4e7-3541-a38e-7d0e573fb4a3" + }, + "role": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claimcareteamrole", + "code": "primary", + "display": "Primary Care Practitioner" + } + ] + } + } + ], + "insurance": [ + { + "focal": true, + "coverage": { + "reference": "#coverage", + "display": "Anthem" + } + } + ], + "item": [ + { + "sequence": 1, + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "410620009", + "display": "Well child visit (procedure)" + } + ], + "text": "Well child visit (procedure)" + }, + "servicedPeriod": { + "start": "2020-01-04T14:53:49-05:00", + "end": "2020-01-04T15:08:49-05:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "encounter": [ + { + "reference": "urn:uuid:beb5bd2f-e787-4852-a541-0e6f88950893" + } + ] + }, + { + "sequence": 2, + "informationSequence": [ + 1 + ], + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "49", + "display": "Hib (PRP-OMP)" + } + ], + "text": "Hib (PRP-OMP)" + }, + "servicedPeriod": { + "start": "2020-01-04T14:53:49-05:00", + "end": "2020-01-04T15:08:49-05:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "net": { + "value": 140.52, + "currency": "USD" + }, + "adjudication": [ + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt", + "display": "Line Beneficiary Coinsurance Amount" + } + ] + }, + "amount": { + "value": 28.104000000000003, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt", + "display": "Line Provider Payment Amount" + } + ] + }, + "amount": { + "value": 112.41600000000001, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt", + "display": "Line Submitted Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt", + "display": "Line Allowed Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt", + "display": "Line Beneficiary Part B Deductible Amount" + } + ] + }, + "amount": { + "value": 0, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd", + "display": "Line Processing Indicator Code" + } + ] + } + } + ] + }, + { + "sequence": 3, + "informationSequence": [ + 2 + ], + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "119", + "display": "rotavirus, monovalent" + } + ], + "text": "rotavirus, monovalent" + }, + "servicedPeriod": { + "start": "2020-01-04T14:53:49-05:00", + "end": "2020-01-04T15:08:49-05:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "net": { + "value": 140.52, + "currency": "USD" + }, + "adjudication": [ + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt", + "display": "Line Beneficiary Coinsurance Amount" + } + ] + }, + "amount": { + "value": 28.104000000000003, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt", + "display": "Line Provider Payment Amount" + } + ] + }, + "amount": { + "value": 112.41600000000001, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt", + "display": "Line Submitted Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt", + "display": "Line Allowed Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt", + "display": "Line Beneficiary Part B Deductible Amount" + } + ] + }, + "amount": { + "value": 0, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd", + "display": "Line Processing Indicator Code" + } + ] + } + } + ] + }, + { + "sequence": 4, + "informationSequence": [ + 3 + ], + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "10", + "display": "IPV" + } + ], + "text": "IPV" + }, + "servicedPeriod": { + "start": "2020-01-04T14:53:49-05:00", + "end": "2020-01-04T15:08:49-05:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "net": { + "value": 140.52, + "currency": "USD" + }, + "adjudication": [ + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt", + "display": "Line Beneficiary Coinsurance Amount" + } + ] + }, + "amount": { + "value": 28.104000000000003, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt", + "display": "Line Provider Payment Amount" + } + ] + }, + "amount": { + "value": 112.41600000000001, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt", + "display": "Line Submitted Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt", + "display": "Line Allowed Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt", + "display": "Line Beneficiary Part B Deductible Amount" + } + ] + }, + "amount": { + "value": 0, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd", + "display": "Line Processing Indicator Code" + } + ] + } + } + ] + }, + { + "sequence": 5, + "informationSequence": [ + 4 + ], + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "20", + "display": "DTaP" + } + ], + "text": "DTaP" + }, + "servicedPeriod": { + "start": "2020-01-04T14:53:49-05:00", + "end": "2020-01-04T15:08:49-05:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "net": { + "value": 140.52, + "currency": "USD" + }, + "adjudication": [ + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt", + "display": "Line Beneficiary Coinsurance Amount" + } + ] + }, + "amount": { + "value": 28.104000000000003, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt", + "display": "Line Provider Payment Amount" + } + ] + }, + "amount": { + "value": 112.41600000000001, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt", + "display": "Line Submitted Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt", + "display": "Line Allowed Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt", + "display": "Line Beneficiary Part B Deductible Amount" + } + ] + }, + "amount": { + "value": 0, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd", + "display": "Line Processing Indicator Code" + } + ] + } + } + ] + }, + { + "sequence": 6, + "informationSequence": [ + 5 + ], + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd", + "code": "1", + "display": "Medical care" + } + ] + }, + "productOrService": { + "coding": [ + { + "system": "http://hl7.org/fhir/sid/cvx", + "code": "133", + "display": "Pneumococcal conjugate PCV 13" + } + ], + "text": "Pneumococcal conjugate PCV 13" + }, + "servicedPeriod": { + "start": "2020-01-04T14:53:49-05:00", + "end": "2020-01-04T15:08:49-05:00" + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/ex-serviceplace", + "code": "19", + "display": "Off Campus-Outpatient Hospital" + } + ] + }, + "net": { + "value": 140.52, + "currency": "USD" + }, + "adjudication": [ + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt", + "display": "Line Beneficiary Coinsurance Amount" + } + ] + }, + "amount": { + "value": 28.104000000000003, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt", + "display": "Line Provider Payment Amount" + } + ] + }, + "amount": { + "value": 112.41600000000001, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt", + "display": "Line Submitted Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt", + "display": "Line Allowed Charge Amount" + } + ] + }, + "amount": { + "value": 140.52, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt", + "display": "Line Beneficiary Part B Deductible Amount" + } + ] + }, + "amount": { + "value": 0, + "currency": "USD" + } + }, + { + "category": { + "coding": [ + { + "system": "https://bluebutton.cms.gov/resources/codesystem/adjudication", + "code": "https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd", + "display": "Line Processing Indicator Code" + } + ] + } + } + ] + } + ], + "total": [ + { + "category": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "submitted", + "display": "Submitted Amount" + } + ], + "text": "Submitted Amount" + }, + "amount": { + "value": 129.16, + "currency": "USD" + } + } + ], + "payment": { + "amount": { + "value": 562.08, + "currency": "USD" + } + } + }, + "request": { + "method": "POST", + "url": "ExplanationOfBenefit" + } + } + ] +} From e64d67f4290cf63dd695479d1cbefd7de86b90bc Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Fri, 10 Apr 2020 16:58:00 -0400 Subject: [PATCH 05/31] Consolidate changes to Command Line Tool and add/improve tests. --- .../FhirResourceDaoObservationDstu3.java | 18 +++-------- .../ObservationLastNIndexPersistDstu3Svc.java | 6 ---- .../ObservationLastNIndexPersistR4Svc.java | 6 ---- .../ObservationLastNIndexPersistR5Svc.java | 6 ---- ...ervationIndexedSearchParamLastNEntity.java | 2 +- .../dao/r4/FhirResourceDaoObservationR4.java | 30 +++---------------- .../dao/r5/FhirResourceDaoObservationR5.java | 20 ++++--------- ...seJpaResourceProviderObservationDstu2.java | 17 +++++++---- ...seJpaResourceProviderObservationDstu3.java | 17 +++++++---- .../BaseJpaResourceProviderObservationR4.java | 15 ++++++---- .../BaseJpaResourceProviderObservationR5.java | 17 +++++++---- .../search/lastn/ElasticsearchSvcImpl.java | 6 +--- ...bservationIndexedSearchParamLastNTest.java | 4 +-- ...bservationIndexedSearchParamLastNTest.java | 6 ++-- ...sticsearchSvcMultipleObservationsTest.java | 2 +- .../fhir/jpa/model/entity/ResourceLink.java | 4 +++ 16 files changed, 68 insertions(+), 108 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java index 3cb0bb1f2f0..993ce07bdf2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java @@ -24,7 +24,6 @@ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistDstu3Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; -import ca.uhn.fhir.jpa.model.entity.ResourceLink; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; @@ -41,9 +40,9 @@ import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.hl7.fhir.dstu3.model.Observation; +import org.hl7.fhir.dstu3.model.Reference; import org.springframework.beans.factory.annotation.Autowired; -import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.util.Collection; import java.util.Collections; @@ -93,18 +92,9 @@ public class FhirResourceDaoObservationDstu3 extends BaseHapiFhirResourceDao myResourceLinks = retVal.getResourceLinks(); - Long subjectID = null; - for (ResourceLink resourceLink : myResourceLinks) { - if(resourceLink.getSourcePath().equals("Observation.subject")) { - subjectID = resourceLink.getTargetResourcePid(); - } - } - if (subjectID != null) { - myObservationLastNIndexPersistDstu3Svc.indexObservation(observation, subjectID.toString()); - } else { - myObservationLastNIndexPersistDstu3Svc.indexObservation(observation); - } + Reference subjectReference = observation.getSubject(); + String subjectID = subjectReference.getIdElement().getValue(); + myObservationLastNIndexPersistDstu3Svc.indexObservation(observation, subjectID); } return retVal; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java index 66ccd325db4..f02743db2fd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java @@ -78,10 +78,4 @@ public class ObservationLastNIndexPersistDstu3Svc { } - // TODO: Remove this once Unit tests are updated. - public void indexObservation(Observation theObservation) { - String subjectId = "Patient/" + theObservation.getSubject().getReference(); - - indexObservation(theObservation, subjectId); - } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java index 09f87f8e9b3..ec7a4afe6f9 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java @@ -79,10 +79,4 @@ public class ObservationLastNIndexPersistR4Svc { } - // TODO: Remove this once Unit tests are updated. - public void indexObservation(Observation theObservation) { - String subjectId = "Patient/" + theObservation.getSubject().getReference(); - - indexObservation(theObservation, subjectId); - } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java index e37d7a154b2..4365740fc91 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java @@ -78,10 +78,4 @@ public class ObservationLastNIndexPersistR5Svc { } - // TODO: Remove this once Unit tests are updated. - public void indexObservation(Observation theObservation) { - String subjectId = "Patient/" + theObservation.getSubject().getReference(); - - indexObservation(theObservation, subjectId); - } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java index 70f8996089a..c56753063ee 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java @@ -17,7 +17,7 @@ public class ObservationIndexedSearchParamLastNEntity { private Long myId; @Field(name = "subject", analyze = Analyze.NO) - @Column(name = "LASTN_SUBJECT_ID", nullable = false) + @Column(name = "LASTN_SUBJECT_ID", nullable = true) private String mySubject; @ManyToOne(fetch = FetchType.LAZY) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index 18aa3098b2f..cc3d220c82c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -22,33 +22,20 @@ package ca.uhn.fhir.jpa.dao.r4; import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoPatient; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR4Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; -import ca.uhn.fhir.jpa.model.entity.ResourceLink; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; -import ca.uhn.fhir.model.api.IResource; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; -import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.param.DateRangeParam; -import ca.uhn.fhir.rest.param.StringAndListParam; -import ca.uhn.fhir.rest.param.StringParam; import org.hl7.fhir.instance.model.api.IBaseResource; -import org.hl7.fhir.instance.model.api.IIdType; -import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.hl7.fhir.r4.model.Observation; -import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.Reference; import org.springframework.beans.factory.annotation.Autowired; -import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.util.Collection; -import java.util.Collections; import java.util.Date; public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { @@ -75,18 +62,9 @@ public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDao myResourceLinks = retVal.getResourceLinks(); - Long subjectID = null; - for (ResourceLink resourceLink : myResourceLinks) { - if(resourceLink.getSourcePath().equals("Observation.subject")) { - subjectID = resourceLink.getTargetResourcePid(); - } - } - if (subjectID != null) { - myObservationLastNIndexPersistR4Svc.indexObservation(observation, subjectID.toString()); - } else { - myObservationLastNIndexPersistR4Svc.indexObservation(observation); - } + Reference subjectReference = observation.getSubject(); + String subjectID = subjectReference.getIdElement().getValue(); + myObservationLastNIndexPersistR4Svc.indexObservation(observation, subjectID); } return retVal; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java index 665f425ea20..d39d69f7621 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java @@ -24,7 +24,6 @@ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR5Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; -import ca.uhn.fhir.jpa.model.entity.ResourceLink; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; @@ -40,12 +39,11 @@ import ca.uhn.fhir.rest.param.StringParam; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.hl7.fhir.r5.model.Reference; import org.hl7.fhir.r5.model.Observation; import org.springframework.beans.factory.annotation.Autowired; -import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -93,20 +91,12 @@ public class FhirResourceDaoObservationR5 extends BaseHapiFhirResourceDao myResourceLinks = retVal.getResourceLinks(); - Long subjectID = null; - for (ResourceLink resourceLink : myResourceLinks) { - if(resourceLink.getSourcePath().equals("Observation.subject")) { - subjectID = resourceLink.getTargetResourcePid(); - } - } - if (subjectID != null) { - myObservationLastNIndexPersistR5Svc.indexObservation(observation, subjectID.toString()); - } else { - myObservationLastNIndexPersistR5Svc.indexObservation(observation); - } + Reference subjectReference = observation.getSubject(); + String subjectID = subjectReference.getIdElement().getValue(); + myObservationLastNIndexPersistR5Svc.indexObservation(observation, subjectID); } + return retVal; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java index ff9956b2c32..89a68df138a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java @@ -9,6 +9,7 @@ import ca.uhn.fhir.model.dstu2.resource.Observation; import ca.uhn.fhir.model.valueset.BundleTypeEnum; import ca.uhn.fhir.rest.annotation.*; import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.SummaryEnum; import ca.uhn.fhir.rest.api.server.IBundleProvider; @@ -73,10 +74,6 @@ public class BaseJpaResourceProviderObservationDstu2 extends JpaResourceProvider @OperationParam(name="date") DateRangeParam theDate, - @Description(shortDefinition="The maximum number of observations to return for each each observation code") - @OperationParam(name="max", max=1, min=0) - NumberParam theMax, - @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -127,19 +124,27 @@ public class BaseJpaResourceProviderObservationDstu2 extends JpaResourceProvider paramMap.add("category", theCategory); paramMap.add("code", theCode); paramMap.add("date", theDate); - paramMap.add("max", theMax); paramMap.add("patient", thePatient); paramMap.add("subject", theSubject); paramMap.setRevIncludes(theRevIncludes); paramMap.setLastUpdated(theLastUpdated); paramMap.setIncludes(theIncludes); paramMap.setLastN(true); + if (theSort == null) { + SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); + SortSpec observationCode = new SortSpec("code").setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); + if (thePatient != null && theSubject == null) { + theSort = new SortSpec("patient").setChain(observationCode); + } else { + theSort = new SortSpec("subject").setChain(observationCode); + } + } paramMap.setSort(theSort); paramMap.setCount(theCount); paramMap.setSummaryMode(theSummaryMode); paramMap.setSearchTotalMode(theSearchTotalMode); - return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); + return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); } finally { endRequest(theServletRequest); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java index eb852776af4..2e2e60e1c89 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java @@ -8,6 +8,7 @@ import ca.uhn.fhir.model.api.annotation.Description; import ca.uhn.fhir.model.valueset.BundleTypeEnum; import ca.uhn.fhir.rest.annotation.*; import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.SummaryEnum; import ca.uhn.fhir.rest.api.server.IBundleProvider; @@ -73,10 +74,6 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider @OperationParam(name="date") DateRangeParam theDate, - @Description(shortDefinition="The maximum number of observations to return for each each observation code") - @OperationParam(name="max", max=1, min=0) - NumberParam theMax, - @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -127,19 +124,27 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider paramMap.add("category", theCategory); paramMap.add("code", theCode); paramMap.add("date", theDate); - paramMap.add("max", theMax); paramMap.add("patient", thePatient); paramMap.add("subject", theSubject); paramMap.setRevIncludes(theRevIncludes); paramMap.setLastUpdated(theLastUpdated); paramMap.setIncludes(theIncludes); paramMap.setLastN(true); + if (theSort == null) { + SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); + SortSpec observationCode = new SortSpec("code").setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); + if (thePatient != null && theSubject == null) { + theSort = new SortSpec("patient").setChain(observationCode); + } else { + theSort = new SortSpec("subject").setChain(observationCode); + } + } paramMap.setSort(theSort); paramMap.setCount(theCount); paramMap.setSummaryMode(theSummaryMode); paramMap.setSearchTotalMode(theSearchTotalMode); - return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); + return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); } finally { endRequest(theServletRequest); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java index d8fb025c1be..c1c7b5dd430 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java @@ -8,6 +8,7 @@ import ca.uhn.fhir.model.api.annotation.Description; import ca.uhn.fhir.model.valueset.BundleTypeEnum; import ca.uhn.fhir.rest.annotation.*; import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.SummaryEnum; import ca.uhn.fhir.rest.api.server.IBundleProvider; @@ -73,10 +74,6 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< @OperationParam(name="date") DateRangeParam theDate, -// @Description(shortDefinition="The maximum number of observations to return for each each observation code") -// @OperationParam(name="max", max=1, min=0) -// NumberParam theMax, - @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -127,13 +124,21 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< paramMap.add("category", theCategory); paramMap.add("code", theCode); paramMap.add("date", theDate); -// paramMap.add("max", theMax); paramMap.add("patient", thePatient); paramMap.add("subject", theSubject); paramMap.setRevIncludes(theRevIncludes); paramMap.setLastUpdated(theLastUpdated); paramMap.setIncludes(theIncludes); paramMap.setLastN(true); + if (theSort == null) { + SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); + SortSpec observationCode = new SortSpec("code").setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); + if (thePatient != null && theSubject == null) { + theSort = new SortSpec("patient").setChain(observationCode); + } else { + theSort = new SortSpec("subject").setChain(observationCode); + } + } paramMap.setSort(theSort); paramMap.setCount(theCount); paramMap.setSummaryMode(theSummaryMode); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java index 85a4dcd7c2a..68bf078f778 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java @@ -8,6 +8,7 @@ import ca.uhn.fhir.model.api.annotation.Description; import ca.uhn.fhir.model.valueset.BundleTypeEnum; import ca.uhn.fhir.rest.annotation.*; import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.SummaryEnum; import ca.uhn.fhir.rest.api.server.IBundleProvider; @@ -73,10 +74,6 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< @OperationParam(name="date") DateRangeParam theDate, - @Description(shortDefinition="The maximum number of observations to return for each each observation code") - @OperationParam(name="max", max=1, min=0) - NumberParam theMax, - @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -127,19 +124,27 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< paramMap.add("category", theCategory); paramMap.add("code", theCode); paramMap.add("date", theDate); - paramMap.add("max", theMax); paramMap.add("patient", thePatient); paramMap.add("subject", theSubject); paramMap.setRevIncludes(theRevIncludes); paramMap.setLastUpdated(theLastUpdated); paramMap.setIncludes(theIncludes); paramMap.setLastN(true); + if (theSort == null) { + SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); + SortSpec observationCode = new SortSpec("code").setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); + if (thePatient != null && theSubject == null) { + theSort = new SortSpec("patient").setChain(observationCode); + } else { + theSort = new SortSpec("subject").setChain(observationCode); + } + } paramMap.setSort(theSort); paramMap.setCount(theCount); paramMap.setSummaryMode(theSummaryMode); paramMap.setSearchTotalMode(theSearchTotalMode); - return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); + return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); } finally { endRequest(theServletRequest); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index fa8061c5212..ef5f2c4b0cd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -415,11 +415,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { if (nextOr instanceof ReferenceParam) { ReferenceParam ref = (ReferenceParam) nextOr; if (isBlank(ref.getChain())) { - if (ref.getResourceType() != null) { - referenceList.add(ref.getResourceType() + "/" + ref.getValue()); - } else { - referenceList.add(ref.getValue()); - } + referenceList.add(ref.getValue()); } } else { throw new IllegalArgumentException("Invalid token type (expecting ReferenceParam): " + nextOr.getClass()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java index 9413e5ae26d..1ba180c5ff7 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java @@ -124,7 +124,7 @@ public class IntegratedObservationIndexedSearchParamLastNTest { // codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); myObservation.setCode(codeableConceptField); - myObservationLastNIndexPersistR4Svc.indexObservation(myObservation); + myObservationLastNIndexPersistR4Svc.indexObservation(myObservation, "4567"); SearchParameterMap searchParameterMap = new SearchParameterMap(); ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); @@ -213,7 +213,7 @@ public class IntegratedObservationIndexedSearchParamLastNTest { Date effectiveDtm = observationDate.getTime(); observation.setEffective(new DateTimeType(effectiveDtm)); - myObservationLastNIndexPersistR4Svc.indexObservation(observation); + myObservationLastNIndexPersistR4Svc.indexObservation(observation, String.valueOf(patientCount)); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java index 1ea31e450bb..d117a41de72 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java @@ -94,12 +94,12 @@ public class PersistObservationIndexedSearchParamLastNTest { codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); myObservation.setCode(codeableConceptField); - myObservationLastNIndexPersistR4Svc.indexObservation(myObservation); + myObservationLastNIndexPersistR4Svc.indexObservation(myObservation, "4567"); List persistedObservationEntities = myResourceIndexedObservationLastNDao.findAll(); assertEquals(1, persistedObservationEntities.size()); ObservationIndexedSearchParamLastNEntity persistedObservationEntity = persistedObservationEntities.get(0); - assertEquals("Patient/"+subjectId.getReference(), persistedObservationEntity.getSubject()); + assertEquals(subjectId.getReference(), persistedObservationEntity.getSubject()); assertEquals(resourcePID, persistedObservationEntity.getIdentifier()); assertEquals(effectiveDtm, persistedObservationEntity.getEffectiveDtm()); @@ -171,7 +171,7 @@ public class PersistObservationIndexedSearchParamLastNTest { Date effectiveDtm = observationDate.getTime(); observation.setEffective(new DateTimeType(effectiveDtm)); - myObservationLastNIndexPersistR4Svc.indexObservation(observation); + myObservationLastNIndexPersistR4Svc.indexObservation(observation, subjectId); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java index 58d69592b8d..50da9dbe3da 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java @@ -260,7 +260,7 @@ public class LastNElasticsearchSvcMultipleObservationsTest { for (int patientCount = 0; patientCount < 10 ; patientCount++) { - String subject = "Patient/"+patientCount; + String subject = String.valueOf(patientCount); for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java index 44438c28626..39ecf5f5a8d 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java @@ -162,6 +162,10 @@ public class ResourceLink extends BaseResourceIndex { myTargetResourceUrl = theTargetResourceUrl.getValue(); } + public String getTargetResourceUrl() { + return myTargetResourceUrl; + } + public void setTargetResourceUrlCanonical(String theTargetResourceUrl) { Validate.notBlank(theTargetResourceUrl); From af11763dee300d6189332af4e1fd637e08c9ebde Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Mon, 13 Apr 2020 08:56:56 -0400 Subject: [PATCH 06/31] Fixes to command line tool, clean-up and minor re-factoring. --- .../ca/uhn/fhir/jpa/demo/CommonConfig.java | 26 +- .../src/main/java/ca/uhn/fhir/fhir/App.java | 13 - .../test/java/ca/uhn/fhir/fhir/AppTest.java | 20 - .../ca/uhn/fhir/jpa/config/BaseConfig.java | 3 +- ...exedCodeCodeableConceptSearchParamDao.java | 2 +- ...vationIndexedCodeCodingSearchParamDao.java | 2 +- ...ObservationIndexedSearchParamLastNDao.java | 2 +- .../ObservationLastNIndexPersistDstu3Svc.java | 3 +- .../ObservationLastNIndexPersistR4Svc.java | 3 +- .../ObservationLastNIndexPersistR5Svc.java | 3 +- .../ObservationLastNIndexPersistSvc.java | 80 ---- ...nIndexedCategoryCodeableConceptEntity.java | 2 +- ...bservationIndexedCategoryCodingEntity.java | 2 +- ...ationIndexedCodeCodeableConceptEntity.java | 4 +- .../ObservationIndexedCodeCodingEntity.java | 2 +- ...ervationIndexedSearchParamLastNEntity.java | 2 +- .../lastn/ElasticsearchBulkIndexSvcImpl.java | 4 +- .../search/lastn/ElasticsearchSvcImpl.java | 4 +- .../fhir/jpa/search/lastn/IndexConstants.java | 4 +- ...bservationIndexedSearchParamLastNTest.java | 4 +- .../lastn/ElasticsearchPerformanceTests.java | 195 ---------- .../ElasticsearchV5PerformanceTests.java | 153 -------- ...ElasticsearchSvcSingleObservationTest.java | 11 +- ...icsearchV5SvcMultipleObservationsTest.java | 312 ---------------- ...asticsearchV5SvcSingleObservationTest.java | 346 ------------------ 25 files changed, 41 insertions(+), 1161 deletions(-) delete mode 100644 hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java delete mode 100644 hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{dao/lastn => model}/entity/ObservationIndexedCategoryCodeableConceptEntity.java (97%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{dao/lastn => model}/entity/ObservationIndexedCategoryCodingEntity.java (95%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{dao/lastn => model}/entity/ObservationIndexedCodeCodeableConceptEntity.java (96%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{dao/lastn => model}/entity/ObservationIndexedCodeCodingEntity.java (97%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{dao/lastn => model}/entity/ObservationIndexedSearchParamLastNEntity.java (98%) delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java index 82b9a9b98dd..c23c27568bc 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java @@ -233,18 +233,20 @@ public class CommonConfig { public EmbeddedElastic embeddedElasticSearch() { String ELASTIC_VERSION = "6.5.4"; - EmbeddedElastic embeddedElastic; - try { - embeddedElastic = EmbeddedElastic.builder() - .withElasticVersion(ELASTIC_VERSION) - .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) - .withSetting(PopularProperties.HTTP_PORT, 0) - .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) - .withStartTimeout(60, TimeUnit.SECONDS) - .build() - .start(); - } catch (IOException | InterruptedException e) { - throw new ConfigurationException(e); + EmbeddedElastic embeddedElastic = null; + if(!ContextHolder.isExternalElasticsearch()) { + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } } return embeddedElastic; diff --git a/hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java b/hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java deleted file mode 100644 index 9b658e5bf48..00000000000 --- a/hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java +++ /dev/null @@ -1,13 +0,0 @@ -package ca.uhn.fhir.fhir; - -/** - * Hello world! - * - */ -public class App -{ - public static void main( String[] args ) - { - System.out.println( "Hello World!" ); - } -} diff --git a/hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java b/hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java deleted file mode 100644 index 83fcc8b98cb..00000000000 --- a/hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package ca.uhn.fhir.fhir; - -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -/** - * Unit test for simple App. - */ -public class AppTest -{ - /** - * Rigorous Test :-) - */ - @Test - public void shouldAnswerWithTrue() - { - assertTrue( true ); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index 5318c8cbaa2..11163796b33 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -285,8 +285,7 @@ public abstract class BaseConfig { public static void configureEntityManagerFactory(LocalContainerEntityManagerFactoryBean theFactory, FhirContext theCtx) { theFactory.setJpaDialect(hibernateJpaDialect(theCtx.getLocalizer())); - //TODO: Consider moving the jpa.dao.lastn.entity.* classes into jpa.entity at some point. - theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.dao.lastn.entity"); + theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity"); theFactory.setPersistenceProvider(new HibernatePersistenceProvider()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java index 2bde707c764..84c6c0e680f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java index d5d46f47e16..c4ca00b856a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodingEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodingEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java index 7a33dec68cd..04d3780058c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java index f02743db2fd..0dfdea37f18 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java @@ -3,7 +3,7 @@ package ca.uhn.fhir.jpa.dao.lastn; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import ca.uhn.fhir.jpa.model.entity.*; import org.hl7.fhir.dstu3.model.CodeableConcept; import org.hl7.fhir.dstu3.model.Coding; import org.hl7.fhir.dstu3.model.Observation; @@ -25,7 +25,6 @@ public class ObservationLastNIndexPersistDstu3Svc { @Autowired IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; - // TODO: Change theSubjectId to be a Long public void indexObservation(Observation theObservation, String theSubjectId) { ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); String resourcePID = theObservation.getIdElement().getIdPart(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java index ec7a4afe6f9..e6336fcf135 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java @@ -3,7 +3,7 @@ package ca.uhn.fhir.jpa.dao.lastn; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import ca.uhn.fhir.jpa.model.entity.*; import org.hl7.fhir.r4.model.CodeableConcept; import org.hl7.fhir.r4.model.Coding; import org.hl7.fhir.r4.model.Observation; @@ -25,7 +25,6 @@ public class ObservationLastNIndexPersistR4Svc { @Autowired IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; - // TODO: Change theSubjectId to be a Long public void indexObservation(Observation theObservation, String theSubjectId) { ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); String resourcePID = theObservation.getIdElement().getIdPart(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java index 4365740fc91..8650b2d2925 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java @@ -3,7 +3,7 @@ package ca.uhn.fhir.jpa.dao.lastn; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import ca.uhn.fhir.jpa.model.entity.*; import org.hl7.fhir.r5.model.CodeableConcept; import org.hl7.fhir.r5.model.Coding; import org.hl7.fhir.r5.model.Observation; @@ -25,7 +25,6 @@ public class ObservationLastNIndexPersistR5Svc { @Autowired IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; - // TODO: Change theSubjectId to be a Long public void indexObservation(Observation theObservation, String theSubjectId) { ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); String resourcePID = theObservation.getIdElement().getIdPart(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java deleted file mode 100644 index 3a4a2c4ab07..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java +++ /dev/null @@ -1,80 +0,0 @@ -package ca.uhn.fhir.jpa.dao.lastn; - -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.*; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import org.hl7.fhir.r4.model.CodeableConcept; -import org.hl7.fhir.r4.model.Coding; -import org.hl7.fhir.r4.model.Observation; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.*; - -//@Component -public class ObservationLastNIndexPersistSvc { - - @Autowired - IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; - - @Autowired - IObservationIndexedCodeCodeableConceptSearchParamDao myObservationIndexedCodeableConceptSearchParamDao; - - @Autowired - IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; - - public void indexObservation(Observation theObservation) { - ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); - String resourcePID = theObservation.getId(); - indexedObservation.setIdentifier(resourcePID); - // TODO: Need to improve this as there are multiple use cases involving subject. - String subjectId = "Patient/" + theObservation.getSubject().getReference(); - indexedObservation.setSubject(subjectId); - Date effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); - indexedObservation.setEffectiveDtm(effectiveDtm); - - // Build CodeableConcept entities for Observation.Category - Set categoryConcepts = new HashSet<>(); - for(CodeableConcept categoryCodeableConcept : theObservation.getCategory()) { - // Build Coding entities for each category CodeableConcept - Set categoryCodingEntities = new HashSet<>(); - ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConceptEntity = new ObservationIndexedCategoryCodeableConceptEntity(categoryCodeableConcept.getText()); - for(Coding categoryCoding : categoryCodeableConcept.getCoding()){ - categoryCodingEntities.add(new ObservationIndexedCategoryCodingEntity(categoryCoding.getSystem(), categoryCoding.getCode(), categoryCoding.getDisplay())); - } - categoryCodeableConceptEntity.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); - categoryConcepts.add(categoryCodeableConceptEntity); - } - indexedObservation.setCategoryCodeableConcepts(categoryConcepts); - - // Build CodeableConcept entity for Observation.Code. - CodeableConcept codeCodeableConcept = theObservation.getCode(); - String observationCodeNormalizedId = null; - - // Determine if a Normalized ID was created previously for Observation Code - for (Coding codeCoding : codeCodeableConcept.getCoding()) { - if (codeCoding.hasCode() && codeCoding.hasSystem()) { - observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); - } else { - observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); - } - } - // Generate a new a normalized ID if necessary - if (observationCodeNormalizedId == null) { - observationCodeNormalizedId = UUID.randomUUID().toString(); - } - - // Create/update normalized Observation Code index record - ObservationIndexedCodeCodeableConceptEntity codeableConceptField = new ObservationIndexedCodeCodeableConceptEntity(codeCodeableConcept.getText(), observationCodeNormalizedId); - for (Coding codeCoding : codeCodeableConcept.getCoding()) { - codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); - } - myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); - - indexedObservation.setObservationCode(codeableConceptField); - indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); - myResourceIndexedObservationLastNDao.save(indexedObservation); - - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodeableConceptEntity.java similarity index 97% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodeableConceptEntity.java index 7d5d98525af..cf5df932ef4 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodeableConceptEntity.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; import org.hibernate.search.annotations.DocumentId; import org.hibernate.search.annotations.Field; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodingEntity.java similarity index 95% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodingEntity.java index 9bc95ed657c..a29ac733005 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodingEntity.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; import ca.uhn.fhir.jpa.dao.lastn.util.CodeSystemHash; import org.hibernate.search.annotations.Analyze; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java similarity index 96% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java index cfe95e2f93d..a2035cc0540 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; import org.hibernate.search.annotations.DocumentId; import org.hibernate.search.annotations.Field; @@ -6,8 +6,6 @@ import org.hibernate.search.annotations.Indexed; import org.hibernate.search.annotations.IndexedEmbedded; import javax.persistence.*; -import java.util.HashSet; -import java.util.Set; @Entity @Indexed(index = "code_index") diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java similarity index 97% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java index ecdbfd6e7e3..8b28dc1d9af 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; import ca.uhn.fhir.jpa.dao.lastn.util.CodeSystemHash; import org.hibernate.search.annotations.Analyze; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java similarity index 98% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java index c56753063ee..8948f2e3575 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; import org.hibernate.search.annotations.*; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java index 2444195c631..d0ace200a6a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java @@ -38,7 +38,7 @@ public class ElasticsearchBulkIndexSvcImpl { } String observationMapping = "{\n" + " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + + " \"ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + " \"properties\" : {\n" + " \"codeconceptid\" : {\n" + " \"type\" : \"keyword\",\n" + @@ -100,7 +100,7 @@ public class ElasticsearchBulkIndexSvcImpl { } String codeMapping = "{\n" + " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + + " \"ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + " \"properties\" : {\n" + " \"codeable_concept_id\" : {\n" + " \"type\" : \"keyword\",\n" + diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index ef5f2c4b0cd..a04e070184b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -73,7 +73,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } String observationMapping = "{\n" + " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + + " \"ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + " \"properties\" : {\n" + " \"codeconceptid\" : {\n" + " \"type\" : \"keyword\",\n" + @@ -135,7 +135,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } String codeMapping = "{\n" + " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + + " \"ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + " \"properties\" : {\n" + " \"codeable_concept_id\" : {\n" + " \"type\" : \"keyword\",\n" + diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java index 94276924896..e605bd079ca 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java @@ -4,6 +4,6 @@ public class IndexConstants { public static final String OBSERVATION_INDEX = "observation_index"; public static final String CODE_INDEX = "code_index"; - public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; - public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity"; + public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity"; + public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity"; } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java index d117a41de72..62abeeb0eca 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java @@ -2,8 +2,8 @@ package ca.uhn.fhir.jpa.dao.lastn; import ca.uhn.fhir.jpa.dao.lastn.config.TestObservationIndexSearchConfig; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; import org.hl7.fhir.r4.model.*; import org.junit.After; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java deleted file mode 100644 index 2e4c751093b..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java +++ /dev/null @@ -1,195 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn; - -import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; -import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; -import ca.uhn.fhir.jpa.search.lastn.util.SimpleStopWatch; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.param.ReferenceParam; -import ca.uhn.fhir.rest.param.TokenParam; -import com.fasterxml.jackson.databind.ObjectMapper; -/* -import org.shadehapi.elasticsearch.action.search.SearchRequest; -import org.shadehapi.elasticsearch.action.search.SearchResponse; -import org.shadehapi.elasticsearch.index.query.QueryBuilders; -import org.shadehapi.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.shadehapi.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; -import org.shadehapi.elasticsearch.search.SearchHit; -import org.shadehapi.elasticsearch.search.aggregations.AggregationBuilders; -import org.shadehapi.elasticsearch.search.aggregations.Aggregations; -import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; -import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.shadehapi.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits; -import org.shadehapi.elasticsearch.search.aggregations.support.ValueType; -import org.shadehapi.elasticsearch.search.builder.SearchSourceBuilder; -import org.shadehapi.elasticsearch.search.sort.SortOrder; - */ -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -@Ignore -public class ElasticsearchPerformanceTests { - - private ElasticsearchSvcImpl elasticsearchSvc = new ElasticsearchSvcImpl("localhost", 9301, "elastic", "changeme"); - private List patientIds = new ArrayList<>(); - - @Before - public void before() throws IOException { -/* SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - RandomScoreFunctionBuilder randomScoreFunctionBuilder = new RandomScoreFunctionBuilder(); - Date now = new Date(); - randomScoreFunctionBuilder.seed(now.getTime()); - FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(randomScoreFunctionBuilder); - searchSourceBuilder.query(functionScoreQueryBuilder); - searchSourceBuilder.size(0); - - // Aggregation by order codes - TermsAggregationBuilder observationCodeValuesBuilder = new TermsAggregationBuilder("group_by_patient", ValueType.STRING).field("subject"); - observationCodeValuesBuilder.size(1000); - // Top Hits Aggregation - String[] topHitsInclude = {"subject"}; - observationCodeValuesBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") - .sort("effectivedtm", SortOrder.DESC) - .fetchSource(topHitsInclude, null).size(1)); - - searchSourceBuilder.aggregation(observationCodeValuesBuilder); - searchRequest.source(searchSourceBuilder); - - SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - patientIds = buildResults(response); -*/ - } - -// @Test - public void testObservationCodesQueryPerformance() throws IOException { -// SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(10000); - long totalElapsedTimes = 0L; - long totalElapsedSearchAndBuildTimes = 0L; - for(int i=0; i<1000; i++) { - SimpleStopWatch stopWatch = new SimpleStopWatch(); -// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - totalElapsedTimes += stopWatch.getElapsedTime(); -// List codes = elasticsearchSvc.buildCodeResult(response); - totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); -// assertEquals(1000L, codes.size()); - } - System.out.println("Codes query Average elapsed time = " + totalElapsedTimes/1000 + " ms"); - System.out.println("Codes query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/1000 + " ms"); - } - - @Test - public void testObservationsQueryPerformance() throws IOException { - long totalElapsedTimes = 0L; - long totalElapsedSearchAndBuildTimes = 0L; - for(String patientId : patientIds) { - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); - searchParameterMap.add("category", categoryParam); - -// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); - SimpleStopWatch stopWatch = new SimpleStopWatch(); -// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - totalElapsedTimes += stopWatch.getElapsedTime(); -// List observations = elasticsearchSvc.buildObservationTermsResults(response); - totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); -// assertEquals(25, observations.size()); - } - System.out.println("Observations query Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); - System.out.println("Observations query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); - } - -// @Test - public void testSingleObservationsQuery() throws IOException { - long totalElapsedTimes = 0L; - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "5b2091a5-a50e-447b-aff4-c34b69eb43d5"); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); - searchParameterMap.add("category", categoryParam); - -// SearchRequest searchRequest = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); - SimpleStopWatch stopWatch = new SimpleStopWatch(); -// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - totalElapsedTimes += stopWatch.getElapsedTime(); -// List observations = elasticsearchSvc.buildObservationCompositeResults(response); -// assertEquals(25, observations.size()); - System.out.println("Average elapsed time = " + totalElapsedTimes/1000 + " ms"); - } - -// @Test - public void testLastNObservationsQueryTermsPerformance() throws IOException { - long totalElapsedTimes = 0L; - long totalElapsedSearchAndBuildTimes = 0L; - for(String patientId : patientIds) { - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); - searchParameterMap.add("category", categoryParam); - -// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 5); - SimpleStopWatch stopWatch = new SimpleStopWatch(); -// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - totalElapsedTimes += stopWatch.getElapsedTime(); -// elasticsearchSvc.buildObservationTermsResults(response); - totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); - } - System.out.println("LastN Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); - System.out.println("LastN Average elapsed search and Build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); - } - -// @Test - public void testLastNObservationsQueryCompPerformance() throws IOException { - long totalElapsedTimes = 0L; - long totalElapsedSearchAndBuildTimes = 0L; - for(String patientId : patientIds) { - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); - searchParameterMap.add("category", categoryParam); - -// SearchRequest searchRequest = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 5); - SimpleStopWatch stopWatch = new SimpleStopWatch(); -// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - totalElapsedTimes += stopWatch.getElapsedTime(); -// elasticsearchSvc.buildObservationCompositeResults(response); - totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); - } - System.out.println("LastN Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); - System.out.println("LastN Average elapsed search and Build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); - } - -/* private List buildResults(SearchResponse theSearchResponse) throws IOException { - Aggregations responseAggregations = theSearchResponse.getAggregations(); - ParsedTerms aggregatedObservationCodes = responseAggregations.get("group_by_patient"); - aggregatedObservationCodes.getBuckets(); - List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); - ObjectMapper objectMapper = new ObjectMapper(); - List identifiers = new ArrayList<>(10000); - for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { - Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); - ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); - SearchHit[] topHits = parsedTopHits.getHits().getHits(); - for (SearchHit topHit : topHits) { - String sources = topHit.getSourceAsString(); - ObservationJson code = objectMapper.readValue(sources,ObservationJson.class); - identifiers.add(code.getSubject().substring(8)); - } - } - return identifiers; - } -*/ - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java deleted file mode 100644 index 96a71aeacc8..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java +++ /dev/null @@ -1,153 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn; - -import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; -import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; -import ca.uhn.fhir.jpa.search.lastn.util.SimpleStopWatch; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.param.ReferenceParam; -import ca.uhn.fhir.rest.param.TokenParam; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortOrder; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -@Ignore -public class ElasticsearchV5PerformanceTests { -/* - private ElasticsearchV5SvcImpl elasticsearchSvc = new ElasticsearchV5SvcImpl("localhost", 9301, "elastic", "changeme"); - private List patientIds = new ArrayList<>(); - - @Before - public void before() throws IOException { - SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - RandomScoreFunctionBuilder randomScoreFunctionBuilder = new RandomScoreFunctionBuilder(); - Date now = new Date(); - randomScoreFunctionBuilder.seed(now.getTime()); - FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(randomScoreFunctionBuilder); - searchSourceBuilder.query(functionScoreQueryBuilder); - searchSourceBuilder.size(0); - - // Aggregation by order codes - TermsAggregationBuilder observationCodeValuesBuilder = new TermsAggregationBuilder("group_by_patient", ValueType.STRING).field("subject"); - observationCodeValuesBuilder.size(1000); - // Top Hits Aggregation - String[] topHitsInclude = {"subject"}; - observationCodeValuesBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") - .sort("effectivedtm", SortOrder.DESC) - .fetchSource(topHitsInclude, null).size(1)); - - searchSourceBuilder.aggregation(observationCodeValuesBuilder); - searchRequest.source(searchSourceBuilder); - -// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); -// patientIds = buildResults(response); - - } - - @Test - public void testObservationCodesQueryPerformance() throws IOException { -// SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(10000); - long totalElapsedTimes = 0L; - long totalElapsedSearchAndBuildTimes = 0L; - for(int i=0; i<1000; i++) { - SimpleStopWatch stopWatch = new SimpleStopWatch(); -// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - totalElapsedTimes += stopWatch.getElapsedTime(); -// List codes = elasticsearchSvc.buildCodeResult(response); - totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); -// assertEquals(1000L, codes.size()); - } - System.out.println("Codes query Average elapsed time = " + totalElapsedTimes/1000 + " ms"); - System.out.println("Codes query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/1000 + " ms"); - } - - @Test - public void testObservationsQueryPerformance() throws IOException { - long totalElapsedTimes = 0L; - long totalElapsedSearchAndBuildTimes = 0L; - for(String patientId : patientIds) { - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); - searchParameterMap.add("category", categoryParam); - -// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); - SimpleStopWatch stopWatch = new SimpleStopWatch(); -// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - totalElapsedTimes += stopWatch.getElapsedTime(); -// List observations = elasticsearchSvc.buildObservationTermsResults(response); - totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); -// assertEquals(25, observations.size()); - } - System.out.println("Observations query Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); - System.out.println("Observations query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); - } - - @Test - public void testLastNObservationsQueryTermsPerformance() throws IOException { - long totalElapsedTimes = 0L; - long totalElapsedSearchAndBuildTimes = 0L; - for(String patientId : patientIds) { - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); - searchParameterMap.add("category", categoryParam); - -// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 5); - SimpleStopWatch stopWatch = new SimpleStopWatch(); -// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - totalElapsedTimes += stopWatch.getElapsedTime(); -// elasticsearchSvc.buildObservationTermsResults(response); - totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); - } - System.out.println("LastN Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); - System.out.println("LastN Average elapsed search and Build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); - } - - private List buildResults(SearchResponse theSearchResponse) throws IOException { - Aggregations responseAggregations = theSearchResponse.getAggregations(); - ParsedTerms aggregatedObservationCodes = responseAggregations.get("group_by_patient"); - aggregatedObservationCodes.getBuckets(); - List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); - ObjectMapper objectMapper = new ObjectMapper(); - List identifiers = new ArrayList<>(10000); - for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { - Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); - ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); - SearchHit[] topHits = parsedTopHits.getHits().getHits(); - for (SearchHit topHit : topHits) { - String sources = topHit.getSourceAsString(); - ObservationJson code = objectMapper.readValue(sources,ObservationJson.class); - identifiers.add(code.getSubject().substring(8)); - } - } - return identifiers; - } -*/ - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java index 59011bc48f8..3c210d42453 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java @@ -37,8 +37,9 @@ public class LastNElasticsearchSvcSingleObservationTest { static ObjectMapper ourMapperNonPrettyPrint; final String RESOURCEPID = "123"; - final String SUBJECTID = "4567"; - final String SUBJECTTYPEANDID = "Patient/4567"; +// final String SUBJECTID = "4567"; +// final String SUBJECTTYPEANDID = "Patient/4567"; + final String SUBJECTID = "Patient/4567"; final Date EFFECTIVEDTM = new Date(); final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category"; final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; @@ -150,7 +151,8 @@ public class LastNElasticsearchSvcSingleObservationTest { ObservationJson observation = observations.get(0); assertEquals(RESOURCEPID, observation.getIdentifier()); - assertEquals(SUBJECTTYPEANDID, observation.getSubject()); +// assertEquals(SUBJECTTYPEANDID, observation.getSubject()); + assertEquals(SUBJECTID, observation.getSubject()); assertEquals(RESOURCEPID, observation.getIdentifier()); assertEquals(EFFECTIVEDTM, observation.getEffectiveDtm()); assertEquals(OBSERVATIONSINGLECODEID, observation.getCode_concept_id()); @@ -322,7 +324,8 @@ public class LastNElasticsearchSvcSingleObservationTest { private void createSingleObservation() throws IOException { ObservationJson indexedObservation = new ObservationJson(); indexedObservation.setIdentifier(RESOURCEPID); - indexedObservation.setSubject(SUBJECTTYPEANDID); +// indexedObservation.setSubject(SUBJECTTYPEANDID); + indexedObservation.setSubject(SUBJECTID); indexedObservation.setEffectiveDtm(EFFECTIVEDTM); // Add three CodeableConcepts for category diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java deleted file mode 100644 index a3ded8e1a86..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java +++ /dev/null @@ -1,312 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn; - -import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchV5Config; -import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; -import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.param.ReferenceParam; -import ca.uhn.fhir.rest.param.TokenParam; -import ca.uhn.fhir.rest.param.TokenParamModifier; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import org.hl7.fhir.r4.model.CodeableConcept; -import org.hl7.fhir.r4.model.Coding; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import java.io.IOException; -import java.util.*; - -import static org.junit.Assert.*; - -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestElasticsearchV5Config.class } ) -public class LastNElasticsearchV5SvcMultipleObservationsTest { - - @Autowired -// private ElasticsearchV5SvcImpl elasticsearchSvc; - - private static ObjectMapper ourMapperNonPrettyPrint; - - private Map>> createdPatientObservationMap = new HashMap<>(); - - - @BeforeClass - public static void beforeClass() { - ourMapperNonPrettyPrint = new ObjectMapper(); - ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); - ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); - ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); - } - - @Before - public void before() throws IOException { -// createMultiplePatientsAndObservations(); - } - - @After - public void after() throws IOException { -// elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); -// elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); - } -/* - @Test - public void testLastNNoCriteriaQuery() throws IOException { - - // execute Observation ID search (Terms Aggregation) last 3 observations for each patient - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, null, 3); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); - - validateQueryResponse(observationIdsOnly); - - } - - private void validateQueryResponse(List observationIdsOnly) { - assertEquals(60, observationIdsOnly.size()); - - // Observation documents should be grouped by subject, then by observation code, and then sorted by effective date/time - // within each observation code. Verify the grouping by creating a nested Map. - Map>> queriedPatientObservationMap = new HashMap<>(); - ObservationJson previousObservationJson = null; - for (ObservationJson observationJson : observationIdsOnly) { - assertNotNull(observationJson.getIdentifier()); - assertNotNull(observationJson.getSubject()); - assertNotNull(observationJson.getCode_concept_id()); - assertNotNull(observationJson.getEffectiveDtm()); - if (previousObservationJson == null) { - ArrayList observationDates = new ArrayList<>(); - observationDates.add(observationJson.getEffectiveDtm()); - Map> codeObservationMap = new HashMap<>(); - codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); - queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); - } else if (observationJson.getSubject().equals(previousObservationJson.getSubject())) { - if (observationJson.getCode_concept_id().equals(previousObservationJson.getCode_concept_id())) { - queriedPatientObservationMap.get(observationJson.getSubject()).get(observationJson.getCode_concept_id()). - add(observationJson.getEffectiveDtm()); - } else { - Map> codeObservationDateMap = queriedPatientObservationMap.get(observationJson.getSubject()); - // Ensure that code concept was not already retrieved out of order for this subject/patient. - assertFalse(codeObservationDateMap.containsKey(observationJson.getCode_concept_id())); - ArrayList observationDates = new ArrayList<>(); - observationDates.add(observationJson.getEffectiveDtm()); - codeObservationDateMap.put(observationJson.getCode_concept_id(),observationDates); - } - } else { - // Ensure that subject/patient was not already retrieved out of order - assertFalse(queriedPatientObservationMap.containsKey(observationJson.getSubject())); - ArrayList observationDates = new ArrayList<>(); - observationDates.add(observationJson.getEffectiveDtm()); - Map> codeObservationMap = new HashMap<>(); - codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); - queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); - } - previousObservationJson = observationJson; - } - - // Finally check that only the most recent effective date/time values were returned and in the correct order. - for(String subjectId : queriedPatientObservationMap.keySet()) { - Map> queriedObservationCodeMap = queriedPatientObservationMap.get(subjectId); - Map> createdObservationCodeMap = createdPatientObservationMap.get(subjectId); - for(String observationCode : queriedObservationCodeMap.keySet()) { - List queriedObservationDates = queriedObservationCodeMap.get(observationCode); - List createdObservationDates = createdObservationCodeMap.get(observationCode); - for (int dateIdx=0; dateIdx observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); - - assertEquals(10, observationIdsOnly.size()); - - } - - @Test - public void testLastNCodeCodeOnlyCategoryCodeOnly() throws IOException { - // Include subject and patient - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam ("test-heart-rate"); - searchParameterMap.add("category", categoryParam); - TokenParam codeParam = new TokenParam("test-code-1"); - searchParameterMap.add("code", codeParam); - - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); - - assertEquals(5, observationIdsOnly.size()); - - } - - @Test - public void testLastNCodeSystemOnlyCategorySystemOnly() throws IOException { - // Include subject and patient - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", null); - searchParameterMap.add("category", categoryParam); - TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); - searchParameterMap.add("code", codeParam); - - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); - - assertEquals(10, observationIdsOnly.size()); - } - - @Test - public void testLastNCodeCodeTextCategoryTextOnly() throws IOException { - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("test-heart-rate display"); - categoryParam.setModifier(TokenParamModifier.TEXT); - searchParameterMap.add("category", categoryParam); - TokenParam codeParam = new TokenParam("test-code-1 display"); - codeParam.setModifier(TokenParamModifier.TEXT); - searchParameterMap.add("code", codeParam); - - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); - - assertEquals(5, observationIdsOnly.size()); - - } - - private void createMultiplePatientsAndObservations() throws IOException { - // Create CodeableConcepts for two Codes, each with three codings. - String codeableConceptId1 = UUID.randomUUID().toString(); - CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); - codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); - codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); - codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); - CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1); - String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1); - - String codeableConceptId2 = UUID.randomUUID().toString(); - CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); - codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); - codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); - codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); - CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2); - String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2); - - // Create CodeableConcepts for two categories, each with three codings. - List category1 = new ArrayList<>(); - // Create three codings and first category CodeableConcept - category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); - category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); - category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); - List categoryConcepts1 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); - categoryCodeableConcept1.setCoding(category1); - categoryConcepts1.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - List category2 = new ArrayList<>(); - category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); - category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); - category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); - List categoryConcepts2 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); - categoryCodeableConcept2.setCoding(category2); - categoryConcepts2.add(categoryCodeableConcept2); - - for (int patientCount = 0; patientCount < 10 ; patientCount++) { - - String subject = "Patient/"+patientCount; - - for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { - - ObservationJson observationJson = new ObservationJson(); - String identifier = String.valueOf((entryCount + patientCount*10)); - observationJson.setIdentifier(identifier); - observationJson.setSubject(subject); - - if (entryCount%2 == 1) { - observationJson.setCategories(categoryConcepts1); - observationJson.setCode(codeableConceptField1); - observationJson.setCode_concept_id(codeableConceptId1); - assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, codeableConceptId1, codeJson1Document, IndexConstants.CODE_DOCUMENT_TYPE)); - } else { - observationJson.setCategories(categoryConcepts2); - observationJson.setCode(codeableConceptField2); - observationJson.setCode_concept_id(codeableConceptId2); - assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, codeableConceptId2, codeJson2Document, IndexConstants.CODE_DOCUMENT_TYPE)); - } - - Calendar observationDate = new GregorianCalendar(); - observationDate.add(Calendar.HOUR, -10 + entryCount); - Date effectiveDtm = observationDate.getTime(); - observationJson.setEffectiveDtm(effectiveDtm); - - String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationJson); - assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, identifier,observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); - - if (createdPatientObservationMap.containsKey(subject)) { - Map> observationCodeMap = createdPatientObservationMap.get(subject); - if (observationCodeMap.containsKey(observationJson.getCode_concept_id())) { - List observationDates = observationCodeMap.get(observationJson.getCode_concept_id()); - // Want dates to be sorted in descending order - observationDates.add(0, effectiveDtm); - // Only keep the three most recent dates for later check. - if(observationDates.size() > 3) { - observationDates.remove(3); - } - } else { - ArrayList observationDates = new ArrayList<>(); - observationDates.add(effectiveDtm); - observationCodeMap.put(observationJson.getCode_concept_id(), observationDates); - } - } else { - ArrayList observationDates = new ArrayList<>(); - observationDates.add(effectiveDtm); - Map> codeObservationMap = new HashMap<>(); - codeObservationMap.put(observationJson.getCode_concept_id(), observationDates); - createdPatientObservationMap.put(subject, codeObservationMap); - } - } - } - - try { - Thread.sleep(2000L); - } catch (InterruptedException theE) { - theE.printStackTrace(); - } - - } -*/ -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java deleted file mode 100644 index 5c2479957aa..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java +++ /dev/null @@ -1,346 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn; - -import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchV5Config; -import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; -import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; -import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.param.ReferenceParam; -import ca.uhn.fhir.rest.param.TokenParam; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import org.hl7.fhir.r4.model.CodeableConcept; -import org.hl7.fhir.r4.model.Coding; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.UUID; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestElasticsearchV5Config.class } ) -public class LastNElasticsearchV5SvcSingleObservationTest { - - @Autowired -// ElasticsearchV5SvcImpl elasticsearchSvc; - - static ObjectMapper ourMapperNonPrettyPrint; - - final String RESOURCEPID = "123"; - final String SUBJECTID = "4567"; - final String SUBJECTTYPEANDID = "Patient/4567"; - final Date EFFECTIVEDTM = new Date(); - final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category"; - final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; - final String CATEGORYSECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-category"; - final String CATEGORYTHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-category"; - final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; - final String FIRSTCATEGORYFIRSTCODINGDISPLAY = "test-heart-rate display"; - final String FIRSTCATEGORYSECONDCODINGCODE = "test-alt-heart-rate"; - final String FIRSTCATEGORYSECONDCODINGDISPLAY = "test-alt-heart-rate display"; - final String FIRSTCATEGORYTHIRDCODINGCODE = "test-2nd-alt-heart-rate"; - final String FIRSTCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-heart-rate display"; - final String SECONDCATEGORYTEXT = "Test Codeable Concept Field for for second category"; - final String SECONDCATEGORYFIRSTCODINGCODE = "test-vital-signs"; - final String SECONDCATEGORYFIRSTCODINGDISPLAY = "test-vital-signs display"; - final String SECONDCATEGORYSECONDCODINGCODE = "test-alt-vitals"; - final String SECONDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; - final String SECONDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals"; - final String SECONDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals display"; - final String THIRDCATEGORYTEXT = "Test Codeable Concept Field for third category"; - final String THIRDCATEGORYFIRSTCODINGCODE = "test-vital-panel"; - final String THIRDCATEGORYFIRSTCODINGDISPLAY = "test-vitals-panel display"; - final String THIRDCATEGORYSECONDCODINGCODE = "test-alt-vitals-panel"; - final String THIRDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; - final String THIRDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals-panel"; - final String THIRDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals-panel display"; - - final String OBSERVATIONSINGLECODEID = UUID.randomUUID().toString(); - final String OBSERVATIONCODETEXT = "Test Codeable Concept Field for Code"; - final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; - final String CODEFIRSTCODINGCODE = "test-code"; - final String CODEFIRSTCODINGDISPLAY = "test-code display"; - final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; - final String CODESECONDCODINGCODE = "test-alt-code"; - final String CODESECONDCODINGDISPLAY = "test-alt-code display"; - final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; - final String CODETHIRDCODINGCODE = "test-second-alt-code"; - final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; - - @BeforeClass - public static void beforeClass() { - ourMapperNonPrettyPrint = new ObjectMapper(); - ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); - ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); - ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); - - } - - // @Before - public void before() throws IOException { -// elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); -// elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); - } - - @After - public void after() throws IOException { -// elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); -// elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); - } - @Test - public void testSingleObservationQuery() throws IOException { - - createSingleObservation(); - - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); - searchParameterMap.add("category", categoryParam); - TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); - searchParameterMap.add("code", codeParam); - - // execute Observation ID search - Terms Aggregation -/* SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 3); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); - - assertEquals(1, observationIdsOnly.size()); - ObservationJson observationIdOnly = observationIdsOnly.get(0); - assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); - - // execute full Observation search - Terms Aggregation - SearchRequest searchRequestAllFields = elasticsearchSvc.buildObservationAllFieldsTermsSearchRequest(1000, searchParameterMap, 3); - SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequestAllFields); - List observations = elasticsearchSvc.buildObservationTermsResults(response); - - validateFullObservationSearch(observations); -*/ - } - - private void validateFullObservationSearch(List observations) throws IOException { - - assertEquals(1, observations.size()); - ObservationJson observation = observations.get(0); - assertEquals(RESOURCEPID, observation.getIdentifier()); - - assertEquals(SUBJECTTYPEANDID, observation.getSubject()); - assertEquals(RESOURCEPID, observation.getIdentifier()); - assertEquals(EFFECTIVEDTM, observation.getEffectiveDtm()); - assertEquals(OBSERVATIONSINGLECODEID, observation.getCode_concept_id()); - - List category_concept_text_values = observation.getCategory_concept_text(); - assertEquals(3,category_concept_text_values.size()); - assertEquals(FIRSTCATEGORYTEXT, category_concept_text_values.get(0)); - assertEquals(SECONDCATEGORYTEXT, category_concept_text_values.get(1)); - assertEquals(THIRDCATEGORYTEXT, category_concept_text_values.get(2)); - - List> category_codings_systems = observation.getCategory_coding_system(); - assertEquals(3,category_codings_systems.size()); - List category_coding_systems = category_codings_systems.get(0); - assertEquals(3, category_coding_systems.size()); - assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); - assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); - assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); - category_codings_systems.get(1); - assertEquals(3, category_coding_systems.size()); - assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); - assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); - assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); - category_codings_systems.get(2); - assertEquals(3, category_coding_systems.size()); - assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); - assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); - assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); - - List> category_codings_codes = observation.getCategory_coding_code(); - assertEquals(3, category_codings_codes.size()); - List category_coding_codes = category_codings_codes.get(0); - assertEquals(3, category_coding_codes.size()); - assertEquals(FIRSTCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); - assertEquals(FIRSTCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); - assertEquals(FIRSTCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); - category_coding_codes = category_codings_codes.get(1); - assertEquals(3, category_coding_codes.size()); - assertEquals(SECONDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); - assertEquals(SECONDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); - assertEquals(SECONDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); - category_coding_codes = category_codings_codes.get(2); - assertEquals(3, category_coding_codes.size()); - assertEquals(THIRDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); - assertEquals(THIRDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); - assertEquals(THIRDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); - - List> category_codings_displays = observation.getCategory_coding_display(); - assertEquals(3, category_codings_displays.size()); - List category_coding_displays = category_codings_displays.get(0); - assertEquals(FIRSTCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); - assertEquals(FIRSTCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); - assertEquals(FIRSTCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); - category_coding_displays = category_codings_displays.get(1); - assertEquals(3, category_coding_displays.size()); - assertEquals(SECONDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); - assertEquals(SECONDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); - assertEquals(SECONDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); - category_coding_displays = category_codings_displays.get(2); - assertEquals(3, category_coding_displays.size()); - assertEquals(THIRDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); - assertEquals(THIRDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); - assertEquals(THIRDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); - - List> category_codings_code_system_hashes = observation.getCategory_coding_code_system_hash(); - assertEquals(3, category_codings_code_system_hashes.size()); - List category_coding_code_system_hashes = category_codings_code_system_hashes.get(0); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); - category_coding_code_system_hashes = category_codings_code_system_hashes.get(1); - assertEquals(3, category_coding_code_system_hashes.size()); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); - category_coding_code_system_hashes = category_codings_code_system_hashes.get(2); - assertEquals(3, category_coding_code_system_hashes.size()); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); - - String code_concept_text_values = observation.getCode_concept_text(); - assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); -/* - List code_coding_systems = observation.getCode_coding_system(); - assertEquals(3,code_coding_systems.size()); - assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems.get(0)); - assertEquals(CODESECONDCODINGSYSTEM, code_coding_systems.get(1)); - assertEquals(CODETHIRDCODINGSYSTEM, code_coding_systems.get(2)); - - List code_coding_codes = observation.getCode_coding_code(); - assertEquals(3, code_coding_codes.size()); - assertEquals(CODEFIRSTCODINGCODE, code_coding_codes.get(0)); - assertEquals(CODESECONDCODINGCODE, code_coding_codes.get(1)); - assertEquals(CODETHIRDCODINGCODE, code_coding_codes.get(2)); - - List code_coding_display = observation.getCode_coding_display(); - assertEquals(3, code_coding_display.size()); - assertEquals(CODEFIRSTCODINGDISPLAY, code_coding_display.get(0)); - assertEquals(CODESECONDCODINGDISPLAY, code_coding_display.get(1)); - assertEquals(CODETHIRDCODINGDISPLAY, code_coding_display.get(2)); - - List code_coding_code_system_hash = observation.getCode_coding_code_system_hash(); - assertEquals(3, code_coding_code_system_hash.size()); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), code_coding_code_system_hash.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), code_coding_code_system_hash.get(2)); -*/ - // Retrieve all Observation codes -/* SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); - SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - List codes = elasticsearchSvc.buildCodeResult(response); - assertEquals(1, codes.size()); - CodeJson persistedObservationCode = codes.get(0); - - String persistedCodeConceptID = persistedObservationCode.getCodeableConceptId(); - assertEquals(OBSERVATIONSINGLECODEID, persistedCodeConceptID); - String persistedCodeConceptText = persistedObservationCode.getCodeableConceptText(); - assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText); - - List persistedCodeCodingSystems = persistedObservationCode.getCoding_system(); - assertEquals(3,persistedCodeCodingSystems.size()); - assertEquals(CODEFIRSTCODINGSYSTEM, persistedCodeCodingSystems.get(0)); - assertEquals(CODESECONDCODINGSYSTEM, persistedCodeCodingSystems.get(1)); - assertEquals(CODETHIRDCODINGSYSTEM, persistedCodeCodingSystems.get(2)); - - List persistedCodeCodingCodes = persistedObservationCode.getCoding_code(); - assertEquals(3, persistedCodeCodingCodes.size()); - assertEquals(CODEFIRSTCODINGCODE, persistedCodeCodingCodes.get(0)); - assertEquals(CODESECONDCODINGCODE, persistedCodeCodingCodes.get(1)); - assertEquals(CODETHIRDCODINGCODE, persistedCodeCodingCodes.get(2)); - - List persistedCodeCodingDisplays = persistedObservationCode.getCoding_display(); - assertEquals(3, persistedCodeCodingDisplays.size()); - assertEquals(CODEFIRSTCODINGDISPLAY, persistedCodeCodingDisplays.get(0)); - assertEquals(CODESECONDCODINGDISPLAY, persistedCodeCodingDisplays.get(1)); - assertEquals(CODETHIRDCODINGDISPLAY, persistedCodeCodingDisplays.get(2)); - - List persistedCodeCodingCodeSystemHashes = persistedObservationCode.getCoding_code_system_hash(); - assertEquals(3, persistedCodeCodingCodeSystemHashes.size()); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(2)); -*/ - - } - - private void createSingleObservation() throws IOException { - ObservationJson indexedObservation = new ObservationJson(); - indexedObservation.setIdentifier(RESOURCEPID); - indexedObservation.setSubject(SUBJECTTYPEANDID); - indexedObservation.setEffectiveDtm(EFFECTIVEDTM); - - // Add three CodeableConcepts for category - List categoryConcepts = new ArrayList<>(); - // Create three codings and first category CodeableConcept - List category1 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText(FIRSTCATEGORYTEXT); - category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY)); - category1.add(new Coding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY)); - category1.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY)); - categoryCodeableConcept1.setCoding(category1); - categoryConcepts.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - List category2 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText(SECONDCATEGORYTEXT); - category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY)); - category2.add(new Coding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY)); - category2.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY)); - categoryCodeableConcept2.setCoding(category2); - categoryConcepts.add(categoryCodeableConcept2); - // Create three codings and third category CodeableConcept - List category3 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText(THIRDCATEGORYTEXT); - category3.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY)); - category3.add(new Coding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY)); - category3.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY)); - categoryCodeableConcept3.setCoding(category3); - categoryConcepts.add(categoryCodeableConcept3); - indexedObservation.setCategories(categoryConcepts); - - // Create CodeableConcept for Code with three codings. - indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID); - CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT); - codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY)); - codeableConceptField.addCoding(new Coding(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE, CODESECONDCODINGDISPLAY)); - codeableConceptField.addCoding(new Coding(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE, CODETHIRDCODINGDISPLAY)); - indexedObservation.setCode(codeableConceptField); - - String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); -// assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); - - CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); - String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); -// assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); - - try { - Thread.sleep(1000L); - } catch (InterruptedException theE) { - theE.printStackTrace(); - } - - } - -} From 0f9bdbd3afd5546a72afd336a7a525d9095250f9 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Mon, 13 Apr 2020 16:14:38 -0400 Subject: [PATCH 07/31] Undid previous re-factoring and fixed a bug introduced in last commit. --- .../src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java | 3 ++- .../IObservationIndexedCodeCodeableConceptSearchParamDao.java | 2 +- .../dao/data/IObservationIndexedCodeCodingSearchParamDao.java | 2 +- .../jpa/dao/data/IObservationIndexedSearchParamLastNDao.java | 2 +- .../fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java | 3 +-- .../jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java | 2 +- .../fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java | 2 +- .../fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java | 2 +- .../ObservationIndexedCategoryCodeableConceptEntity.java | 2 +- .../lastn}/entity/ObservationIndexedCategoryCodingEntity.java | 2 +- .../entity/ObservationIndexedCodeCodeableConceptEntity.java | 2 +- .../lastn}/entity/ObservationIndexedCodeCodingEntity.java | 2 +- .../entity/ObservationIndexedSearchParamLastNEntity.java | 2 +- .../ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java | 2 +- .../ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java | 2 +- .../fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java | 4 ++-- .../ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java | 4 ++-- .../java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java | 4 ++-- .../lastn/PersistObservationIndexedSearchParamLastNTest.java | 4 ++-- 19 files changed, 24 insertions(+), 24 deletions(-) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{model => dao/lastn}/entity/ObservationIndexedCategoryCodeableConceptEntity.java (97%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{model => dao/lastn}/entity/ObservationIndexedCategoryCodingEntity.java (95%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{model => dao/lastn}/entity/ObservationIndexedCodeCodeableConceptEntity.java (98%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{model => dao/lastn}/entity/ObservationIndexedCodeCodingEntity.java (97%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/{model => dao/lastn}/entity/ObservationIndexedSearchParamLastNEntity.java (98%) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index 11163796b33..a52bd1cf903 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -285,7 +285,8 @@ public abstract class BaseConfig { public static void configureEntityManagerFactory(LocalContainerEntityManagerFactoryBean theFactory, FhirContext theCtx) { theFactory.setJpaDialect(hibernateJpaDialect(theCtx.getLocalizer())); - theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity"); + // TODO: Looking at moving the lastn entities into jpa.model.entity package. + theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.dao.lastn.entity"); theFactory.setPersistenceProvider(new HibernatePersistenceProvider()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java index 84c6c0e680f..2bde707c764 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity; +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java index c4ca00b856a..d5d46f47e16 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodingEntity; +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodingEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java index 04d3780058c..7a33dec68cd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity; +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java index 993ce07bdf2..320ee57caa8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java @@ -44,7 +44,6 @@ import org.hl7.fhir.dstu3.model.Reference; import org.springframework.beans.factory.annotation.Autowired; import javax.servlet.http.HttpServletResponse; -import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -93,7 +92,7 @@ public class FhirResourceDaoObservationDstu3 extends BaseHapiFhirResourceDao Date: Mon, 13 Apr 2020 20:01:34 -0400 Subject: [PATCH 08/31] Fixed conflicts after merging latest from Master. --- .../ca/uhn/fhir/cli/RunServerCommand.java | 1 - .../ca/uhn/fhir/jpa/demo/CommonConfig.java | 1 - .../ca/uhn/fhir/jpa/demo/JpaServerDemo.java | 1 + .../api}/dao/IFhirResourceDaoObservation.java | 2 +- .../FhirResourceDaoObservationDstu3.java | 2 +- .../dao/r4/FhirResourceDaoObservationR4.java | 2 +- .../dao/r5/FhirResourceDaoObservationR5.java | 2 +- ...seJpaResourceProviderObservationDstu2.java | 2 +- ...seJpaResourceProviderObservationDstu3.java | 2 +- .../BaseJpaResourceProviderObservationR4.java | 2 +- .../BaseJpaResourceProviderObservationR5.java | 2 +- ...bservationIndexedSearchParamLastNTest.java | 2 +- .../dao/r4/FhirResourceDaoR4LastNTest.java | 22 +++---------------- 13 files changed, 13 insertions(+), 30 deletions(-) rename {hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa => hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api}/dao/IFhirResourceDaoObservation.java (97%) diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java index dd21da3b3d5..65d4d65ead0 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java +++ b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java @@ -20,7 +20,6 @@ package ca.uhn.fhir.cli; * #L% */ -import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.demo.*; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.demo.ContextHolder; diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java index 24a3e4b2325..14d4df8c519 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java @@ -21,7 +21,6 @@ package ca.uhn.fhir.jpa.demo; */ import ca.uhn.fhir.context.ConfigurationException; -import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.model.entity.ModelConfig; import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory; diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java index 8f61decd28a..3e87d92b78a 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java @@ -122,6 +122,7 @@ public class JpaServerDemo extends RestfulServer { myAppCtx.getBean(DaoConfig.class)); confProvider.setImplementationDescription("Example Server"); setServerConformanceProvider(confProvider); + } else if (fhirVersion == FhirVersionEnum.DSTU3) { IFhirSystemDao systemDao = myAppCtx .getBean("mySystemDaoDstu3", IFhirSystemDao.class); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDaoObservation.java similarity index 97% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFhirResourceDaoObservation.java rename to hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDaoObservation.java index fd38cd2e807..fa723120ce6 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFhirResourceDaoObservation.java +++ b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDaoObservation.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao; +package ca.uhn.fhir.jpa.api.dao; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.rest.api.server.IBundleProvider; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java index 320ee57caa8..496eac6240d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java @@ -21,7 +21,7 @@ package ca.uhn.fhir.jpa.dao.dstu3; */ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistDstu3Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index ada783ee346..6750644e943 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -21,7 +21,7 @@ package ca.uhn.fhir.jpa.dao.r4; */ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR4Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java index eacb0df0711..c1168660e63 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java @@ -21,7 +21,7 @@ package ca.uhn.fhir.jpa.dao.r5; */ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR5Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java index 89a68df138a..9a61f635368 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.provider; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.Include; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java index 2e2e60e1c89..039ee2059c3 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.provider.dstu3; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.Include; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java index c1c7b5dd430..1f79f3692e8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.provider.r4; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.Include; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java index 68bf078f778..53f75ba210b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.provider.r5; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.Include; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java index 1ba180c5ff7..1bafb4f7e86 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java @@ -1,7 +1,7 @@ package ca.uhn.fhir.jpa.dao.lastn; import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.jpa.dao.IFhirSystemDao; +import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao; import ca.uhn.fhir.jpa.dao.lastn.config.TestIntegratedObservationIndexSearchConfig; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java index 86f54337192..30797965ebb 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java @@ -1,32 +1,19 @@ package ca.uhn.fhir.jpa.dao.r4; import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.interceptor.executor.InterceptorService; -import ca.uhn.fhir.jpa.dao.*; -import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl.Suggestion; +import ca.uhn.fhir.jpa.api.dao.*; +import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.dao.lastn.config.TestIntegratedObservationIndexSearchConfig; -import ca.uhn.fhir.jpa.model.util.JpaConstants; -import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; +import ca.uhn.fhir.jpa.dao.BaseJpaTest; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.api.Constants; -import ca.uhn.fhir.rest.api.server.IBundleProvider; -import ca.uhn.fhir.rest.param.*; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; -import ca.uhn.fhir.util.BundleUtil; import ca.uhn.fhir.util.TestUtil; -import org.hl7.fhir.instance.model.api.IBaseBundle; -import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.*; -import org.hl7.fhir.r4.model.Observation.ObservationStatus; import org.junit.AfterClass; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Answers; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.test.context.ContextConfiguration; @@ -36,10 +23,7 @@ import org.springframework.transaction.PlatformTransactionManager; import javax.servlet.http.HttpServletRequest; import java.util.*; -import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @RunWith(SpringJUnit4ClassRunner.class) From 4a89860f9b343d5c6553c4c60161782e8187b5c2 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Fri, 17 Apr 2020 17:38:42 -0400 Subject: [PATCH 09/31] Additional tests and cleanup. --- .../ca/uhn/fhir/cli/RunServerCommand.java | 15 - .../ca/uhn/fhir/jpa/demo/CommonConfig.java | 120 +-- .../ca/uhn/fhir/jpa/demo/ContextHolder.java | 18 - .../ca/uhn/fhir/jpa/demo/JpaServerDemo.java | 2 - hapi-fhir-jpaserver-base/pom.xml | 5 - .../ca/uhn/fhir/jpa/config/BaseConfig.java | 2 +- .../BaseHapiFhirResourceDaoObservation.java | 45 + .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 60 +- .../ObservationLastNIndexPersistDstu3Svc.java | 1 + .../ObservationLastNIndexPersistR5Svc.java | 1 + .../dao/r4/FhirResourceDaoObservationR4.java | 8 +- .../BaseJpaResourceProviderObservationR4.java | 4 +- .../lastn/ElasticsearchBulkIndexSvcImpl.java | 188 ---- .../search/lastn/ElasticsearchSvcImpl.java | 899 ++++++++---------- .../jpa/search/lastn/IElasticsearchSvc.java | 12 +- .../fhir/jpa/search/lastn/IndexConstants.java | 14 +- ...ulticodeObservationsIntoElasticSearch.java | 122 --- .../OneMillionPatientsIntoElasticSearch.java | 128 --- ...ThousandObservationsIntoElasticSearch.java | 116 --- .../UploadSampleDatasetIntoElasticSearch.java | 153 --- .../config/TestR4ConfigWithElasticSearch.java | 11 +- .../TestR4ConfigWithElasticsearchClient.java | 16 + ...bservationIndexedSearchParamLastNTest.java | 43 +- ...bservationIndexedSearchParamLastNTest.java | 4 +- ...ntegratedObservationIndexSearchConfig.java | 23 - .../TestObservationIndexSearchConfig.java | 93 -- .../dao/r4/FhirResourceDaoR4LastNTest.java | 134 --- .../r4/FhirResourceDaoR4SearchLastNTest.java | 472 +++++++++ ...sticsearchSvcMultipleObservationsTest.java | 545 ++++++----- ...ElasticsearchSvcSingleObservationTest.java | 511 +++++----- .../config/TestElasticsearchV5Config.java | 53 -- 31 files changed, 1577 insertions(+), 2241 deletions(-) create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNTest.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java index 65d4d65ead0..c79ce66547a 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java +++ b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java @@ -20,7 +20,6 @@ package ca.uhn.fhir.cli; * #L% */ -import ca.uhn.fhir.jpa.demo.*; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.demo.ContextHolder; import ca.uhn.fhir.jpa.demo.FhirServerConfig; @@ -48,10 +47,8 @@ public class RunServerCommand extends BaseCommand { private static final String OPTION_LOWMEM = "lowmem"; private static final String OPTION_ALLOW_EXTERNAL_REFS = "allow-external-refs"; private static final String OPTION_REUSE_SEARCH_RESULTS_MILLIS = "reuse-search-results-milliseconds"; - private static final String OPTION_EXTERNAL_ELASTICSEARCH = "external-elasticsearch"; private static final int DEFAULT_PORT = 8080; private static final String OPTION_P = "p"; - private static final String OPTION_POSTGRES = "postgresql"; // TODO: Don't use qualified names for loggers in HAPI CLI. private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(RunServerCommand.class); @@ -73,8 +70,6 @@ public class RunServerCommand extends BaseCommand { options.addOption(null, OPTION_LOWMEM, false, "If this flag is set, the server will operate in low memory mode (some features disabled)"); options.addOption(null, OPTION_ALLOW_EXTERNAL_REFS, false, "If this flag is set, the server will allow resources to be persisted contaning external resource references"); options.addOption(null, OPTION_DISABLE_REFERENTIAL_INTEGRITY, false, "If this flag is set, the server will not enforce referential integrity"); - options.addOption(null, OPTION_EXTERNAL_ELASTICSEARCH, false, "If this flag is set, the server will attempt to use a local elasticsearch server listening on port 9301"); - options.addOption(null, OPTION_POSTGRES, false, "If this flag is set, the server will attempt to use a local postgresql DB instance listening on port 5432"); addOptionalOption(options, "u", "url", "Url", "If this option is set, specifies the JDBC URL to use for the database connection"); addOptionalOption(options, "d", "default-size", "PageSize", "If this option is set, specifies the default page size for number of query results"); @@ -115,18 +110,8 @@ public class RunServerCommand extends BaseCommand { ContextHolder.setDisableReferentialIntegrity(true); } - if (theCommandLine.hasOption(OPTION_EXTERNAL_ELASTICSEARCH)) { - ourLog.info("Server is configured to use external elasticsearch"); - ContextHolder.setExternalElasticsearch(true); - } - ContextHolder.setDatabaseUrl(theCommandLine.getOptionValue("u")); - if (theCommandLine.hasOption(OPTION_POSTGRES)) { - ourLog.info("Server is configured to use PostgreSQL database"); - ContextHolder.setPostgreSql(true); - } - String defaultPageSize = theCommandLine.getOptionValue("d"); String maxPageSize = theCommandLine.getOptionValue("m"); if (defaultPageSize != null) { diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java index 14d4df8c519..324aa98c2de 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java @@ -46,7 +46,6 @@ import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isNotBlank; @SuppressWarnings("Duplicates") -// TODO: Merge this with new CommonPostgreSQLConfig or find way to avoid conflicts with it. @Configuration public class CommonConfig { @@ -76,42 +75,6 @@ public class CommonConfig { */ @Bean(destroyMethod = "close") public DataSource dataSource() { - if (ContextHolder.isPostGreSql()) { - return getPostgreSqlDataSource(); - } else { - return getH2DataSource(); - } - } - - /** - * The following method creates a PostgreSQL database connection. The 'url' property value of "jdbc:postgresql://localhost:5432/hapi" indicates that the server should save resources in a - * PostgreSQL database named "hapi". - * - * A URL to a remote database could also be placed here, along with login credentials and other properties supported by BasicDataSource. - */ - private DataSource getPostgreSqlDataSource() { - String dbUrl = "jdbc:postgresql://localhost:5432/hapi"; - String dbUsername = "hapi"; - String dbPassword = "HapiFHIR"; - if (isNotBlank(ContextHolder.getDatabaseUrl())) { - dbUrl = ContextHolder.getDatabaseUrl(); - } - - BasicDataSource retVal = new BasicDataSource(); - retVal.setDriverClassName("org.postgresql.Driver"); - retVal.setUrl(dbUrl); - retVal.setUsername(dbUsername); - retVal.setPassword(dbPassword); - return retVal; - } - - /** - * The following method creates an H2 database connection. The 'url' property value of "jdbc:h2:file:target./jpaserver_h2_files" indicates that the server should save resources in a - * directory called "jpaserver_h2_files". - * - * A URL to a remote database could also be placed here, along with login credentials and other properties supported by BasicDataSource. - */ - private DataSource getH2DataSource() { String url = "jdbc:h2:file:./target/jpaserver_h2_files"; if (isNotBlank(ContextHolder.getDatabaseUrl())) { url = ContextHolder.getDatabaseUrl(); @@ -127,14 +90,6 @@ public class CommonConfig { @Bean public Properties jpaProperties() { - if (ContextHolder.isPostGreSql()) { - return getPostGreSqlJpaProperties(); - } else { - return getH2JpaProperties(); - } - } - - private Properties getH2JpaProperties() { Properties extraProperties = new Properties(); extraProperties.put("hibernate.dialect", H2Dialect.class.getName()); extraProperties.put("hibernate.format_sql", "true"); @@ -158,44 +113,12 @@ public class CommonConfig { return configureElasticearch(extraProperties); } - private Properties getPostGreSqlJpaProperties() { - - Properties extraProperties = new Properties(); - extraProperties.put("hibernate.dialect", org.hibernate.dialect.PostgreSQL94Dialect.class.getName()); - extraProperties.put("hibernate.format_sql", "false"); - extraProperties.put("hibernate.show_sql", "false"); - extraProperties.put("hibernate.hbm2ddl.auto", "update"); - extraProperties.put("hibernate.jdbc.batch_size", "20"); - extraProperties.put("hibernate.cache.use_query_cache", "false"); - extraProperties.put("hibernate.cache.use_second_level_cache", "false"); - extraProperties.put("hibernate.cache.use_structured_entries", "false"); - extraProperties.put("hibernate.cache.use_minimal_puts", "false"); - extraProperties.put("hibernate.search.model_mapping", LuceneSearchMappingFactory.class.getName()); - extraProperties.put("hibernate.search.default.directory_provider", "local-heap"); - extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); - extraProperties.put("hibernate.search.default.worker.execution", "sync"); - - if (System.getProperty("lowmem") != null) { - extraProperties.put("hibernate.search.autoregister_listeners", "false"); - } - - return configureElasticearch(extraProperties); - } - private Properties configureElasticearch(Properties theExtraProperties) { String elasticsearchHost = "localhost"; String elasticsearchUserId = ""; String elasticsearchPassword = ""; - Integer elasticsearchPort; - - if(ContextHolder.isExternalElasticsearch()) { - elasticsearchUserId = "elastic"; - elasticsearchPassword = "changeme"; - elasticsearchPort = 9301; - } else { - elasticsearchPort = embeddedElasticSearch().getHttpPort(); - } + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); new ElasticsearchHibernatePropertiesBuilder() .setDebugRefreshAfterWrite(true) @@ -212,41 +135,22 @@ public class CommonConfig { } - @Bean() - public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { - String elasticsearchHost = "localhost"; - String elasticsearchUserId = ""; - String elasticsearchPassword = ""; - Integer elasticsearchPort; - - if(ContextHolder.isExternalElasticsearch()) { - elasticsearchUserId = "elastic"; - elasticsearchPassword = "changeme"; - elasticsearchPort = 9301; - } else { - elasticsearchPort = embeddedElasticSearch().getHttpPort(); - } - return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); - } - @Bean public EmbeddedElastic embeddedElasticSearch() { String ELASTIC_VERSION = "6.5.4"; EmbeddedElastic embeddedElastic = null; - if(!ContextHolder.isExternalElasticsearch()) { - try { - embeddedElastic = EmbeddedElastic.builder() - .withElasticVersion(ELASTIC_VERSION) - .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) - .withSetting(PopularProperties.HTTP_PORT, 0) - .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) - .withStartTimeout(60, TimeUnit.SECONDS) - .build() - .start(); - } catch (IOException | InterruptedException e) { - throw new ConfigurationException(e); - } + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); } return embeddedElastic; diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java index f9d52989a8e..5968d9ef739 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java @@ -33,8 +33,6 @@ public class ContextHolder { private static String ourPath; private static Long ourReuseSearchResultsMillis; private static String ourDatabaseUrl; - private static boolean myExternalElasticsearch = false; - private static boolean myPostGreSql = false; private static Integer myDefaultPageSize = 10; private static Integer myMaxPageSize = 50; @@ -105,22 +103,6 @@ public class ContextHolder { ourDatabaseUrl = theDatabaseUrl; } - public static void setExternalElasticsearch(Boolean theExternalElasticsearch) { - myExternalElasticsearch = theExternalElasticsearch; - } - - public static Boolean isExternalElasticsearch() { - return myExternalElasticsearch; - } - - public static void setPostgreSql(boolean thePostGreSql) { - myPostGreSql = thePostGreSql; - } - - public static boolean isPostGreSql() { - return myPostGreSql; - } - public static void setDefaultPageSize(Integer theDefaultPageSize) { myDefaultPageSize = theDefaultPageSize; } diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java index 3e87d92b78a..40b8e8f1e57 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java @@ -122,7 +122,6 @@ public class JpaServerDemo extends RestfulServer { myAppCtx.getBean(DaoConfig.class)); confProvider.setImplementationDescription("Example Server"); setServerConformanceProvider(confProvider); - } else if (fhirVersion == FhirVersionEnum.DSTU3) { IFhirSystemDao systemDao = myAppCtx .getBean("mySystemDaoDstu3", IFhirSystemDao.class); @@ -161,7 +160,6 @@ public class JpaServerDemo extends RestfulServer { /* * This is a simple paging strategy that keeps the last 10 searches in memory */ - // TODO: Make this configurable via the ContextHolder setPagingProvider(new FifoMemoryPagingProvider(10).setDefaultPageSize(ContextHolder.getDefaultPageSize()).setMaximumPageSize(ContextHolder.getMaxPageSize())); // Register a CORS filter diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index b4a33a33486..af0cee2d749 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -618,11 +618,6 @@ 1.0-SNAPSHOT shaded6 - diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index 005650f5989..f991b36f56c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -270,7 +270,7 @@ public abstract class BaseConfig { public static void configureEntityManagerFactory(LocalContainerEntityManagerFactoryBean theFactory, FhirContext theCtx) { theFactory.setJpaDialect(hibernateJpaDialect(theCtx.getLocalizer())); - // TODO: Looking at moving the lastn entities into jpa.model.entity package. + // TODO: Looking at moving the lastn entities into jpa.model.entity package. Note that moving the lastn entities may require re-building elasticsearch indexes. theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.dao.lastn.entity"); theFactory.setPersistenceProvider(new HibernatePersistenceProvider()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java new file mode 100644 index 00000000000..f0755c825b2 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java @@ -0,0 +1,45 @@ +package ca.uhn.fhir.jpa.dao; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.search.lastn.IndexConstants; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.api.*; +import ca.uhn.fhir.rest.api.server.*; +import org.hl7.fhir.instance.model.api.*; + +public abstract class BaseHapiFhirResourceDaoObservation extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { + + protected void updateSearchParamsForLastn(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails) { + if (!isPagingProviderDatabaseBacked(theRequestDetails)) { + theSearchParameterMap.setLoadSynchronous(true); + } + + theSearchParameterMap.setLastN(true); + if (theSearchParameterMap.getSort() == null) { + SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); + SortSpec observationCode = new SortSpec(IndexConstants.CODE_SEARCH_PARAM).setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); + theSearchParameterMap.setSort(new SortSpec(IndexConstants.SUBJECT_SEARCH_PARAM).setChain(observationCode)); + } + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 5657e0a336f..a3a5a5de9a7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -323,35 +323,45 @@ public class SearchBuilder implements ISearchBuilder { * Fulltext search and lastn */ if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { - List lastnPids = new ArrayList<>(); - List fullTextSearchPids = new ArrayList<>(); + List lastnPids = new ArrayList<>(); + List fullTextSearchPids = new ArrayList<>(); - if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { - if (myFulltextSearchSvc == null) { - if (myParams.containsKey(Constants.PARAM_TEXT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); - } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); - } - } - - if (myParams.getEverythingMode() != null) { - fullTextSearchPids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); - } else { - fullTextSearchPids = myFulltextSearchSvc.search(myResourceName, myParams); - } - } else { - if (myIElasticsearchSvc == null) { - if (myParams.isLastN()) { - throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); - } - } - - if (myParams.isLastN()) { - lastnPids = myIElasticsearchSvc.executeLastN(myParams, theRequest, myIdHelperService); + if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { + if (myFulltextSearchSvc == null) { + if (myParams.containsKey(Constants.PARAM_TEXT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); } } + if (myParams.getEverythingMode() != null) { + fullTextSearchPids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); + } else { + fullTextSearchPids = myFulltextSearchSvc.search(myResourceName, myParams); + } + } else { + if (myIElasticsearchSvc == null) { + if (myParams.isLastN()) { + throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); + } + } + + if (myParams.isLastN()) { + Integer myMaxObservationsPerCode = null; + String[] maxCountParams = theRequest.getParameters().get("max"); + if (maxCountParams != null && maxCountParams.length > 0) { + myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]); + } else { + throw new InvalidRequestException("Max parameter is required for $lastn operation"); + } + List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); + for (String lastnResourceId : lastnResourceIds) { + lastnPids.add(myIdHelperService.resolveResourcePersistentIds(myResourceName, lastnResourceId)); + } + } + } + // List pids; if (fullTextSearchPids.isEmpty()) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java index dd45c9b8908..10b5e1aa697 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java @@ -70,6 +70,7 @@ public class ObservationLastNIndexPersistDstu3Svc { codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); } myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); + codeableConceptField = myObservationIndexedCodeableConceptSearchParamDao.findByCodeableConceptId(observationCodeNormalizedId); indexedObservation.setObservationCode(codeableConceptField); indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java index 434978ae48d..be0b00b58a3 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java @@ -70,6 +70,7 @@ public class ObservationLastNIndexPersistR5Svc { codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); } myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); + codeableConceptField = myObservationIndexedCodeableConceptSearchParamDao.findByCodeableConceptId(observationCodeNormalizedId); indexedObservation.setObservationCode(codeableConceptField); indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index 6750644e943..7ef7972b0b1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -22,6 +22,7 @@ package ca.uhn.fhir.jpa.dao.r4; import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR4Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; @@ -38,16 +39,15 @@ import org.springframework.beans.factory.annotation.Autowired; import javax.servlet.http.HttpServletResponse; import java.util.Date; -public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { +public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDaoObservation { @Autowired ObservationLastNIndexPersistR4Svc myObservationLastNIndexPersistR4Svc; @Override public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse) { - if (!isPagingProviderDatabaseBacked(theRequestDetails)) { - theSearchParameterMap.setLoadSynchronous(true); - } + + updateSearchParamsForLastn(theSearchParameterMap, theRequestDetails); return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java index 1f79f3692e8..a2bdac6f7b4 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java @@ -129,7 +129,7 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< paramMap.setRevIncludes(theRevIncludes); paramMap.setLastUpdated(theLastUpdated); paramMap.setIncludes(theIncludes); - paramMap.setLastN(true); +/* paramMap.setLastN(true); if (theSort == null) { SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); SortSpec observationCode = new SortSpec("code").setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); @@ -138,7 +138,7 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< } else { theSort = new SortSpec("subject").setChain(observationCode); } - } + } */ paramMap.setSort(theSort); paramMap.setCount(theCount); paramMap.setSummaryMode(theSummaryMode); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java deleted file mode 100644 index 2444195c631..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java +++ /dev/null @@ -1,188 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn; -/* -import org.shadehapi.elasticsearch.action.DocWriteRequest; -import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.shadehapi.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.shadehapi.elasticsearch.action.bulk.BulkItemResponse; -import org.shadehapi.elasticsearch.action.bulk.BulkRequest; -import org.shadehapi.elasticsearch.action.bulk.BulkResponse; -import org.shadehapi.elasticsearch.action.index.IndexRequest; -import org.shadehapi.elasticsearch.client.RequestOptions; -import org.shadehapi.elasticsearch.client.RestHighLevelClient; -import org.shadehapi.elasticsearch.common.xcontent.XContentType; -*/ -import java.io.IOException; - -public class ElasticsearchBulkIndexSvcImpl { - -// RestHighLevelClient myRestHighLevelClient; - -// BulkRequest myBulkRequest = null; - - public ElasticsearchBulkIndexSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { - -// myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername,thePassword); - - try { - createObservationIndexIfMissing(); - createCodeIndexIfMissing(); - } catch (IOException theE) { - throw new RuntimeException("Failed to create document index", theE); - } - } - - public void createObservationIndexIfMissing() throws IOException { - if(indexExists(IndexConstants.OBSERVATION_INDEX)) { - return; - } - String observationMapping = "{\n" + - " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + - " \"properties\" : {\n" + - " \"codeconceptid\" : {\n" + - " \"type\" : \"keyword\",\n" + - " \"norms\" : true\n" + - " },\n" + - " \"codeconcepttext\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"codeconceptcodingcode\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codeconceptcodingsystem\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codeconceptcodingcode_system_hash\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codeconceptcodingdisplay\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"categoryconcepttext\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"categoryconceptcodingcode\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"categoryconceptcodingsystem\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"categoryconceptcodingcode_system_hash\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"categoryconceptcodingdisplay\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"effectivedtm\" : {\n" + - " \"type\" : \"date\"\n" + - " },\n" + - " \"identifier\" : {\n" + - " \"type\" : \"keyword\",\n" + - " \"store\" : true\n" + - " },\n" + - " \"subject\" : {\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}\n"; - if(!createIndex(IndexConstants.OBSERVATION_INDEX, observationMapping)) { - throw new RuntimeException("Failed to create observation index"); - } - - } - - public void createCodeIndexIfMissing() throws IOException { - if(indexExists(IndexConstants.CODE_INDEX)) { - return; - } - String codeMapping = "{\n" + - " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + - " \"properties\" : {\n" + - " \"codeable_concept_id\" : {\n" + - " \"type\" : \"keyword\",\n" + - " \"store\" : true\n" + - " },\n" + - " \"codeable_concept_text\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"codingcode\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codingcode_system_hash\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codingdisplay\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"codingsystem\" : {\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}\n"; - if (!createIndex(IndexConstants.CODE_INDEX, codeMapping)) { - throw new RuntimeException("Failed to create code index"); - } - - } - - public boolean createIndex(String theIndexName, String theMapping) throws IOException { -/* CreateIndexRequest request = new CreateIndexRequest(theIndexName); - request.source(theMapping, XContentType.JSON); - CreateIndexResponse createIndexResponse = myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT); - return createIndexResponse.isAcknowledged(); - */ - return false; - } - - public boolean indexExists(String theIndexName) throws IOException { -/* GetIndexRequest request = new GetIndexRequest(); - request.indices(theIndexName); - return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); - - */ - return false; - } - - public void addToBulkIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { -/* IndexRequest request = new IndexRequest(theIndexName); - request.id(theDocumentId); - request.type(theDocumentType); - - request.source(theObservationDocument, XContentType.JSON); - - if (myBulkRequest == null) { - myBulkRequest = new BulkRequest(); - } - myBulkRequest.add(request); */ - } - - public void executeBulkIndex() throws IOException { -/* if (myBulkRequest == null) { - throw new RuntimeException(("No index requests have been added to the bulk request")); - } - BulkResponse bulkResponse = myRestHighLevelClient.bulk(myBulkRequest, RequestOptions.DEFAULT); - for (BulkItemResponse bulkItemResponse : bulkResponse) { - if (bulkItemResponse.getOpType() != DocWriteRequest.OpType.CREATE && bulkItemResponse.getOpType() != DocWriteRequest.OpType.INDEX) { - throw new RuntimeException("Unexpected response for bulk index request: " + bulkItemResponse.getOpType()); - } - } - myBulkRequest = null; */ - } - - public boolean bulkRequestPending() { -// return (myBulkRequest != null); - return false; - } - - public void closeClient() throws IOException { -// myRestHighLevelClient.close(); - } - - -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index ef5f2c4b0cd..2a0a4c790ff 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -1,20 +1,16 @@ package ca.uhn.fhir.jpa.search.lastn; -import ca.uhn.fhir.jpa.dao.index.IdHelperService; -import ca.uhn.fhir.jpa.model.cross.ResourcePersistentId; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.IQueryParameterType; -import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.param.NumberParam; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; -import ca.uhn.fhir.rest.server.RestfulServerUtils; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.shadehapi.elasticsearch.action.admin.indices.get.GetIndexRequest; @@ -51,526 +47,475 @@ import static org.apache.commons.lang3.StringUtils.isBlank; public class ElasticsearchSvcImpl implements IElasticsearchSvc { - RestHighLevelClient myRestHighLevelClient; + RestHighLevelClient myRestHighLevelClient; - ObjectMapper objectMapper = new ObjectMapper(); + ObjectMapper objectMapper = new ObjectMapper(); + + private final String GROUP_BY_SUBJECT = "group_by_subject"; + private final String GROUP_BY_CODE = "group_by_code"; - public ElasticsearchSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { - myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername,thePassword); + public ElasticsearchSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { + myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername, thePassword); - try { - createObservationIndexIfMissing(); - createCodeIndexIfMissing(); - } catch (IOException theE) { - throw new RuntimeException("Failed to create document index", theE); - } - } + try { + createObservationIndexIfMissing(); + createCodeIndexIfMissing(); + } catch (IOException theE) { + throw new RuntimeException("Failed to create document index", theE); + } + } - private void createObservationIndexIfMissing() throws IOException { - if(indexExists(IndexConstants.OBSERVATION_INDEX)) { - return; - } - String observationMapping = "{\n" + - " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + - " \"properties\" : {\n" + - " \"codeconceptid\" : {\n" + - " \"type\" : \"keyword\",\n" + - " \"norms\" : true\n" + - " },\n" + - " \"codeconcepttext\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"codeconceptcodingcode\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codeconceptcodingsystem\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codeconceptcodingcode_system_hash\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codeconceptcodingdisplay\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"categoryconcepttext\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"categoryconceptcodingcode\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"categoryconceptcodingsystem\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"categoryconceptcodingcode_system_hash\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"categoryconceptcodingdisplay\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"effectivedtm\" : {\n" + - " \"type\" : \"date\"\n" + - " },\n" + - " \"identifier\" : {\n" + - " \"type\" : \"keyword\",\n" + - " \"store\" : true\n" + - " },\n" + - " \"subject\" : {\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}\n"; - if(!createIndex(IndexConstants.OBSERVATION_INDEX, observationMapping)) { - throw new RuntimeException("Failed to create observation index"); - } + private void createObservationIndexIfMissing() throws IOException { + if (indexExists(IndexConstants.OBSERVATION_INDEX)) { + return; + } + String observationMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeconceptid\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"norms\" : true\n" + + " },\n" + + " \"codeconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codeconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"effectivedtm\" : {\n" + + " \"type\" : \"date\"\n" + + " },\n" + + " \"identifier\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"subject\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if (!createIndex(IndexConstants.OBSERVATION_INDEX, observationMapping)) { + throw new RuntimeException("Failed to create observation index"); + } - } + } - private void createCodeIndexIfMissing() throws IOException { - if(indexExists(IndexConstants.CODE_INDEX)) { - return; - } - String codeMapping = "{\n" + - " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + - " \"properties\" : {\n" + - " \"codeable_concept_id\" : {\n" + - " \"type\" : \"keyword\",\n" + - " \"store\" : true\n" + - " },\n" + - " \"codeable_concept_text\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"codingcode\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codingcode_system_hash\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"codingdisplay\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + - " \"codingsystem\" : {\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}\n"; - if (!createIndex(IndexConstants.CODE_INDEX, codeMapping)) { - throw new RuntimeException("Failed to create code index"); - } + private void createCodeIndexIfMissing() throws IOException { + if (indexExists(IndexConstants.CODE_INDEX)) { + return; + } + String codeMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeable_concept_id\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"codeable_concept_text\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if (!createIndex(IndexConstants.CODE_INDEX, codeMapping)) { + throw new RuntimeException("Failed to create code index"); + } - } + } - private boolean createIndex(String theIndexName, String theMapping) throws IOException { - CreateIndexRequest request = new CreateIndexRequest(theIndexName); - request.source(theMapping, XContentType.JSON); - CreateIndexResponse createIndexResponse = myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT); - return createIndexResponse.isAcknowledged(); + private boolean createIndex(String theIndexName, String theMapping) throws IOException { + CreateIndexRequest request = new CreateIndexRequest(theIndexName); + request.source(theMapping, XContentType.JSON); + CreateIndexResponse createIndexResponse = myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT); + return createIndexResponse.isAcknowledged(); - } + } - boolean performIndex(String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) throws IOException { - IndexResponse indexResponse = myRestHighLevelClient.index(createIndexRequest(theIndexName, theDocumentId,theIndexDocument,theDocumentType), + @VisibleForTesting + boolean performIndex(String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) throws IOException { + IndexResponse indexResponse = myRestHighLevelClient.index(createIndexRequest(theIndexName, theDocumentId, theIndexDocument, theDocumentType), RequestOptions.DEFAULT); - return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); - } + return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); + } - private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { - IndexRequest request = new IndexRequest(theIndexName); - request.id(theDocumentId); - request.type(theDocumentType); + private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { + IndexRequest request = new IndexRequest(theIndexName); + request.id(theDocumentId); + request.type(theDocumentType); - request.source(theObservationDocument, XContentType.JSON); - return request; - } + request.source(theObservationDocument, XContentType.JSON); + return request; + } - private boolean indexExists(String theIndexName) throws IOException { - GetIndexRequest request = new GetIndexRequest(); - request.indices(theIndexName); - return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); - } + private boolean indexExists(String theIndexName) throws IOException { + GetIndexRequest request = new GetIndexRequest(); + request.indices(theIndexName); + return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); + } - @Override - public List executeLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, IdHelperService theIdHelperService) { - Integer myMaxObservationsPerCode = 1; - String[] maxCountParams = theRequestDetails.getParameters().get("max"); - if (maxCountParams != null && maxCountParams.length > 0) { - myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]); - } - SearchRequest myLastNRequest = buildObservationCompositeSearchRequest(10000, theSearchParameterMap, myMaxObservationsPerCode); - SearchResponse lastnResponse = null; - try { - lastnResponse = executeSearchRequest(myLastNRequest); - List observationIds = buildObservationIdList(lastnResponse, theIdHelperService); - return observationIds; - } catch (IOException theE) { - throw new InvalidRequestException("Unable to execute LastN request", theE); - } - } + @Override + public List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { + String[] topHitsInclude = {"identifier"}; + SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(theMaxObservationsPerCode, topHitsInclude)); + try { + SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); + return buildObservationIdList(lastnResponse); + } catch (IOException theE) { + throw new InvalidRequestException("Unable to execute LastN request", theE); + } + } + + @VisibleForTesting + List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { + SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(theMaxObservationsPerCode, null)); + try { + SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); + return buildObservationDocumentList(lastnResponse); + } catch (IOException theE) { + throw new InvalidRequestException("Unable to execute LastN request", theE); + } + } + + @VisibleForTesting + List queryAllIndexedObservationCodes(int theMaxResultSetSize) throws IOException { + SearchRequest codeSearchRequest = buildObservationCodesSearchRequest(theMaxResultSetSize); + SearchResponse codeSearchResponse = executeSearchRequest(codeSearchRequest); + return buildCodeResult(codeSearchResponse); + } + + @VisibleForTesting + SearchRequest buildObservationCodesSearchRequest(int theMaxResultSetSize) { + SearchRequest searchRequest = new SearchRequest(IndexConstants.CODE_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + // Query + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.size(theMaxResultSetSize); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + private CompositeAggregationBuilder createCompositeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { + TermsAggregationBuilder observationCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE, ValueType.STRING).field("codeconceptid"); + // Top Hits Aggregation + observationCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + .sort("effectivedtm", SortOrder.DESC) + .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); + observationCodeAggregationBuilder.size(10000); + CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); + List> compositeAggSubjectSources = new ArrayList(); + compositeAggSubjectSources.add(subjectValuesBuilder); + CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder(GROUP_BY_SUBJECT, compositeAggSubjectSources); + compositeAggregationSubjectBuilder.subAggregation(observationCodeAggregationBuilder); + compositeAggregationSubjectBuilder.size(10000); + + return compositeAggregationSubjectBuilder; + } + + private SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { + return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + } - SearchRequest buildObservationCodesSearchRequest(int theMaxResultSetSize) { - SearchRequest searchRequest = new SearchRequest(IndexConstants.CODE_INDEX); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - // Query - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - searchSourceBuilder.size(theMaxResultSetSize); - searchRequest.source(searchSourceBuilder); - return searchRequest; - } - - private CompositeAggregationBuilder createCompositeAggregationBuilder(int theMaximumResultSetSize, int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { - TermsAggregationBuilder observationCodeAggregationBuilder = new TermsAggregationBuilder("group_by_code", ValueType.STRING).field("codeconceptid"); - // Top Hits Aggregation - observationCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") - .sort("effectivedtm", SortOrder.DESC) - .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); - observationCodeAggregationBuilder.size(theMaximumResultSetSize); - CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); - List> compositeAggSubjectSources = new ArrayList(); - compositeAggSubjectSources.add(subjectValuesBuilder); - CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder("group_by_subject", compositeAggSubjectSources); - compositeAggregationSubjectBuilder.subAggregation(observationCodeAggregationBuilder); - compositeAggregationSubjectBuilder.size(theMaximumResultSetSize); - - return compositeAggregationSubjectBuilder; - } - - private TermsAggregationBuilder createTermsAggregationBuilder(int theMaximumResultSetSize, int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { - TermsAggregationBuilder observationCodeAggregationBuilder = new TermsAggregationBuilder("group_by_code", ValueType.STRING).field("codeconceptid"); - // Top Hits Aggregation - observationCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") - .sort("effectivedtm", SortOrder.DESC).fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); - observationCodeAggregationBuilder.size(theMaximumResultSetSize); - TermsAggregationBuilder subjectsBuilder = new TermsAggregationBuilder("group_by_subject", ValueType.STRING).field("subject"); - subjectsBuilder.subAggregation(observationCodeAggregationBuilder); - subjectsBuilder.size(theMaximumResultSetSize); - return subjectsBuilder; - } - - public SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { - return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); - } - - public List buildObservationCompositeResults(SearchResponse theSearchResponse) throws IOException { - Aggregations responseAggregations = theSearchResponse.getAggregations(); - ParsedComposite aggregatedSubjects = responseAggregations.get("group_by_subject"); - List subjectBuckets = aggregatedSubjects.getBuckets(); - List codes = new ArrayList<>(); - for(ParsedComposite.ParsedBucket subjectBucket : subjectBuckets) { - Aggregations observationCodeAggregations = subjectBucket.getAggregations(); - ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); - List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); - for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { - Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); - ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); - SearchHit[] topHits = parsedTopHits.getHits().getHits(); - for (SearchHit topHit : topHits) { - String sources = topHit.getSourceAsString(); - ObservationJson code = objectMapper.readValue(sources, ObservationJson.class); - codes.add(code); - } - } - } - return codes; - } - - private List buildObservationIdList(SearchResponse theSearchResponse, IdHelperService theIdHelperService) throws IOException { - Aggregations responseAggregations = theSearchResponse.getAggregations(); - ParsedComposite aggregatedSubjects = responseAggregations.get("group_by_subject"); - List subjectBuckets = aggregatedSubjects.getBuckets(); - List myObservationIds = new ArrayList<>(); - for(ParsedComposite.ParsedBucket subjectBucket : subjectBuckets) { - Aggregations observationCodeAggregations = subjectBucket.getAggregations(); - ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); - List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); - for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { - Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); - ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); - SearchHit[] topHits = parsedTopHits.getHits().getHits(); - for (SearchHit topHit : topHits) { - String sources = topHit.getSourceAsString(); - ObservationJson code = objectMapper.readValue(sources, ObservationJson.class); - - myObservationIds.add(theIdHelperService.resolveResourcePersistentIds("Observation", code.getIdentifier())); + private List buildObservationIdList(SearchResponse theSearchResponse) throws IOException { + List theObservationList = new ArrayList<>(); + for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { + for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) { + for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { + String indexedObservation = lastNMatch.getSourceAsString(); + ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); + theObservationList.add(observationJson.getIdentifier()); } } } - return myObservationIds; + return theObservationList; } - public List buildObservationTermsResults(SearchResponse theSearchResponse) throws IOException { - Aggregations responseAggregations = theSearchResponse.getAggregations(); - ParsedTerms aggregatedSubjects = responseAggregations.get("group_by_subject"); - List subjectBuckets = aggregatedSubjects.getBuckets(); - List codes = new ArrayList<>(); - for (Terms.Bucket subjectBucket : subjectBuckets) { - Aggregations observationCodeAggregations = subjectBucket.getAggregations(); - ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); - List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); - for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { - Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); - ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); - SearchHit[] topHits = parsedTopHits.getHits().getHits(); - for (SearchHit topHit : topHits) { - String sources = topHit.getSourceAsString(); - ObservationJson code = objectMapper.readValue(sources,ObservationJson.class); - codes.add(code); - } - } - } - return codes; - } + private List buildObservationDocumentList(SearchResponse theSearchResponse) throws IOException { + List theObservationList = new ArrayList<>(); + for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { + for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) { + for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { + String indexedObservation = lastNMatch.getSourceAsString(); + ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); + theObservationList.add(observationJson); + } + } + } + return theObservationList; + } - public List buildCodeResult(SearchResponse theSearchResponse) throws JsonProcessingException { - SearchHits codeHits = theSearchResponse.getHits(); - List codes = new ArrayList<>(); - for (SearchHit codeHit : codeHits) { - CodeJson code = objectMapper.readValue(codeHit.getSourceAsString(), CodeJson.class); - codes.add(code); - } - return codes; - } + private List getSubjectBuckets(SearchResponse theSearchResponse) { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedComposite aggregatedSubjects = responseAggregations.get(GROUP_BY_SUBJECT); + return aggregatedSubjects.getBuckets(); + } - public SearchRequest buildObservationAllFieldsCompositeSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { + private List getObservationCodeBuckets(ParsedComposite.ParsedBucket theSubjectBucket) { + Aggregations observationCodeAggregations = theSubjectBucket.getAggregations(); + ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get(GROUP_BY_CODE); + return aggregatedObservationCodes.getBuckets(); + } - return buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, null)); - } + private SearchHit[] getLastNMatches(Terms.Bucket theObservationCodeBucket) { + Aggregations topHitObservationCodes = theObservationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + return parsedTopHits.getHits().getHits(); + } - public SearchRequest buildObservationCompositeSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { - // Return only identifiers - String[] topHitsInclude = {"identifier","subject","effectivedtm","codeconceptid"}; - return buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, topHitsInclude)); - } + private List buildCodeResult(SearchResponse theSearchResponse) throws JsonProcessingException { + SearchHits codeHits = theSearchResponse.getHits(); + List codes = new ArrayList<>(); + for (SearchHit codeHit : codeHits) { + CodeJson code = objectMapper.readValue(codeHit.getSourceAsString(), CodeJson.class); + codes.add(code); + } + return codes; + } - public SearchRequest buildObservationAllFieldsTermsSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { - return buildObservationsSearchRequest(theSearchParameterMap, createTermsAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, null)); - } + private SearchRequest buildObservationsSearchRequest(SearchParameterMap theSearchParameterMap, AggregationBuilder theAggregationBuilder) { + SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + // Query + if (!searchParamsHaveLastNCriteria(theSearchParameterMap)) { + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + } else { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + addSubjectsCriteria(boolQueryBuilder, theSearchParameterMap); + addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap); + addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap); + searchSourceBuilder.query(boolQueryBuilder); + } + searchSourceBuilder.size(0); - public SearchRequest buildObservationTermsSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { - // Return only identifiers - String[] topHitsInclude = {"identifier","subject","effectivedtm","codeconceptid"}; - return buildObservationsSearchRequest(theSearchParameterMap, createTermsAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, topHitsInclude)); - } + // Aggregation by order codes + searchSourceBuilder.aggregation(theAggregationBuilder); + searchRequest.source(searchSourceBuilder); - private SearchRequest buildObservationsSearchRequest(SearchParameterMap theSearchParameterMap, AggregationBuilder theAggregationBuilder) { - SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - // Query - if(!searchParamsHaveLastNCriteria(theSearchParameterMap)) { - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - } else { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - addSubjectsCriteria(boolQueryBuilder,theSearchParameterMap); - addCategoriesCriteria(boolQueryBuilder,theSearchParameterMap); - addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap); - searchSourceBuilder.query(boolQueryBuilder); - } - searchSourceBuilder.size(0); + return searchRequest; + } - // Aggregation by order codes - searchSourceBuilder.aggregation(theAggregationBuilder); - searchRequest.source(searchSourceBuilder); + private Boolean searchParamsHaveLastNCriteria(SearchParameterMap theSearchParameterMap) { + return theSearchParameterMap != null && + (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM) || + theSearchParameterMap.containsKey(IndexConstants.CATEGORY_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.CODE_SEARCH_PARAM)); + } - return searchRequest; - } + private void addSubjectsCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { + ArrayList subjectReferenceCriteria = new ArrayList<>(); + List> andOrParams = new ArrayList<>(); + if (theSearchParameterMap.get(IndexConstants.PATIENT_SEARCH_PARAM) != null) { + andOrParams.addAll(theSearchParameterMap.get(IndexConstants.PATIENT_SEARCH_PARAM)); + } + if (theSearchParameterMap.get(IndexConstants.SUBJECT_SEARCH_PARAM) != null) { + andOrParams.addAll(theSearchParameterMap.get(IndexConstants.SUBJECT_SEARCH_PARAM)); + } + for (List nextAnd : andOrParams) { + subjectReferenceCriteria.addAll(getReferenceValues(nextAnd)); + } + if (subjectReferenceCriteria.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("subject", subjectReferenceCriteria)); + } + } - private Boolean searchParamsHaveLastNCriteria(SearchParameterMap theSearchParameterMap) { - return theSearchParameterMap != null && - (theSearchParameterMap.containsKey("patient") || theSearchParameterMap.containsKey("subject") || - theSearchParameterMap.containsKey("category") || theSearchParameterMap.containsKey("code")); - } + } - private void addSubjectsCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { - if (theSearchParameterMap.containsKey("patient") || theSearchParameterMap.containsKey("subject")) { - ArrayList subjectReferenceCriteria = new ArrayList<>(); - List> andOrParams = new ArrayList<>(); - if (theSearchParameterMap.get("patient") != null) { - andOrParams.addAll(theSearchParameterMap.get("patient")); - } - if (theSearchParameterMap.get("subject") != null) { - andOrParams.addAll(theSearchParameterMap.get("subject")); - } - for (List nextAnd : andOrParams) { - subjectReferenceCriteria.addAll(getReferenceValues(nextAnd)); - } - if (subjectReferenceCriteria.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("subject", subjectReferenceCriteria)); - } - } + private List getReferenceValues(List referenceParams) { + ArrayList referenceList = new ArrayList<>(); - } + for (IQueryParameterType nextOr : referenceParams) { - private List getReferenceValues(List referenceParams) { - ArrayList referenceList = new ArrayList<>(); + if (nextOr instanceof ReferenceParam) { + ReferenceParam ref = (ReferenceParam) nextOr; + if (isBlank(ref.getChain())) { + referenceList.add(ref.getValue()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting ReferenceParam): " + nextOr.getClass()); + } + } + return referenceList; + } - for (IQueryParameterType nextOr : referenceParams) { + private void addCategoriesCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.containsKey(IndexConstants.CATEGORY_SEARCH_PARAM)) { + ArrayList codeSystemHashList = new ArrayList<>(); + ArrayList codeOnlyList = new ArrayList<>(); + ArrayList systemOnlyList = new ArrayList<>(); + ArrayList textOnlyList = new ArrayList<>(); + List> andOrParams = theSearchParameterMap.get(IndexConstants.CATEGORY_SEARCH_PARAM); + for (List nextAnd : andOrParams) { + codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); + codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); + systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); + textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); + } + if (codeSystemHashList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode_system_hash", codeSystemHashList)); + } + if (codeOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode", codeOnlyList)); + } + if (systemOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingsystem", systemOnlyList)); + } + if (textOnlyList.size() > 0) { + BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); + for (String textOnlyParam : textOnlyList) { + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconcepttext", textOnlyParam)); + } + theBoolQueryBuilder.must(myTextBoolQueryBuilder); + } + } - if (nextOr instanceof ReferenceParam) { - ReferenceParam ref = (ReferenceParam) nextOr; - if (isBlank(ref.getChain())) { - referenceList.add(ref.getValue()); - } - } else { - throw new IllegalArgumentException("Invalid token type (expecting ReferenceParam): " + nextOr.getClass()); - } - } - return referenceList; - } + } - private void addCategoriesCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { - if (theSearchParameterMap.containsKey("category")) { - ArrayList codeSystemHashList = new ArrayList<>(); - ArrayList codeOnlyList = new ArrayList<>(); - ArrayList systemOnlyList = new ArrayList<>(); - ArrayList textOnlyList = new ArrayList<>(); - List> andOrParams = theSearchParameterMap.get("category"); - for (List nextAnd : andOrParams) { - codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); - codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); - systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); - textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); - } - if (codeSystemHashList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode_system_hash", codeSystemHashList)); - } - if (codeOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode", codeOnlyList)); - } - if (systemOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingsystem", systemOnlyList)); - } - if (textOnlyList.size() > 0) { - BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); - for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconcepttext", textOnlyParam)); - } - theBoolQueryBuilder.must(myTextBoolQueryBuilder); - } - } + private List getCodingCodeSystemValues(List codeParams) { + ArrayList codeSystemHashList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams) { + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.getSystem() != null && ref.getValue() != null) { + codeSystemHashList.add(String.valueOf(CodeSystemHash.hashCodeSystem(ref.getSystem(), ref.getValue()))); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return codeSystemHashList; + } - } + private List getCodingCodeOnlyValues(List codeParams) { + ArrayList codeOnlyList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams) { - private List getCodingCodeSystemValues(List codeParams) { - ArrayList codeSystemHashList = new ArrayList<>(); - for (IQueryParameterType nextOr : codeParams ) { - if (nextOr instanceof TokenParam) { - TokenParam ref = (TokenParam) nextOr; - if (ref.getSystem() != null && ref.getValue() != null) { - codeSystemHashList.add(String.valueOf(CodeSystemHash.hashCodeSystem(ref.getSystem(), ref.getValue()))); - } - } else { - throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); - } - } - return codeSystemHashList; - } + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.getValue() != null && ref.getSystem() == null && !ref.isText()) { + codeOnlyList.add(ref.getValue()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return codeOnlyList; + } - private List getCodingCodeOnlyValues(List codeParams ) { - ArrayList codeOnlyList = new ArrayList<>(); - for (IQueryParameterType nextOr : codeParams) { + private List getCodingSystemOnlyValues(List codeParams) { + ArrayList systemOnlyList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams) { - if (nextOr instanceof TokenParam) { - TokenParam ref = (TokenParam) nextOr; - if (ref.getValue() != null && ref.getSystem() == null && !ref.isText()) { - codeOnlyList.add(ref.getValue()); - } - } else { - throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); - } - } - return codeOnlyList; - } + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.getValue() == null && ref.getSystem() != null) { + systemOnlyList.add(ref.getSystem()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return systemOnlyList; + } - private List getCodingSystemOnlyValues(List codeParams ) { - ArrayList systemOnlyList = new ArrayList<>(); - for (IQueryParameterType nextOr : codeParams) { + private List getCodingTextOnlyValues(List codeParams) { + ArrayList textOnlyList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams) { - if (nextOr instanceof TokenParam) { - TokenParam ref = (TokenParam) nextOr; - if (ref.getValue() == null && ref.getSystem() != null) { - systemOnlyList.add(ref.getSystem()); - } - } else { - throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); - } - } - return systemOnlyList; - } + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.isText() && ref.getValue() != null) { + textOnlyList.add(ref.getValue()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return textOnlyList; + } - private List getCodingTextOnlyValues(List codeParams ) { - ArrayList textOnlyList = new ArrayList<>(); - for (IQueryParameterType nextOr : codeParams ) { + private void addObservationCodeCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.containsKey(IndexConstants.CODE_SEARCH_PARAM)) { + ArrayList codeSystemHashList = new ArrayList<>(); + ArrayList codeOnlyList = new ArrayList<>(); + ArrayList systemOnlyList = new ArrayList<>(); + ArrayList textOnlyList = new ArrayList<>(); + List> andOrParams = theSearchParameterMap.get(IndexConstants.CODE_SEARCH_PARAM); + for (List nextAnd : andOrParams) { + codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); + codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); + systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); + textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); + } + if (codeSystemHashList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode_system_hash", codeSystemHashList)); + } + if (codeOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode", codeOnlyList)); + } + if (systemOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingsystem", systemOnlyList)); + } + if (textOnlyList.size() > 0) { + BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); + for (String textOnlyParam : textOnlyList) { + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconcepttext", textOnlyParam)); + } + theBoolQueryBuilder.must(myTextBoolQueryBuilder); + } + } - if (nextOr instanceof TokenParam) { - TokenParam ref = (TokenParam) nextOr; - if (ref.isText() && ref.getValue() != null) { - textOnlyList.add(ref.getValue()); - } - } else { - throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); - } - } - return textOnlyList; - } + } - private void addObservationCodeCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { - if (theSearchParameterMap.containsKey("code")) { - ArrayList codeSystemHashList = new ArrayList<>(); - ArrayList codeOnlyList = new ArrayList<>(); - ArrayList systemOnlyList = new ArrayList<>(); - ArrayList textOnlyList = new ArrayList<>(); - List> andOrParams = theSearchParameterMap.get("code"); - for (List nextAnd : andOrParams) { - codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); - codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); - systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); - textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); - } - if (codeSystemHashList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode_system_hash", codeSystemHashList)); - } - if (codeOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode", codeOnlyList)); - } - if (systemOnlyList.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingsystem", systemOnlyList)); - } - if (textOnlyList.size() > 0) { - BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); - for (String textOnlyParam : textOnlyList) { - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconceptcodingdisplay", textOnlyParam)); - myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconcepttext", textOnlyParam)); - } - theBoolQueryBuilder.must(myTextBoolQueryBuilder); - } - } + @VisibleForTesting + void deleteAllDocuments(String theIndexName) throws IOException { + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(theIndexName); + deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); + myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); + } - } - - void deleteAllDocuments(String theIndexName) throws IOException { - DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(theIndexName); - deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); - myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); - } - -/* @Override - public IBundleProvider lastN(HttpServletRequest theServletRequest, RequestDetails theRequestDetails) { - - return null; - } - - @Override - public IBundleProvider uniqueCodes(HttpServletRequest theServletRequest, RequestDetails theRequestDetails) { - return null; - } - - */ } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java index bd6064481f1..091d85b742f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -1,20 +1,10 @@ package ca.uhn.fhir.jpa.search.lastn; -//import ca.uhn.fhir.rest.api.server.IBundleProvider; - -import ca.uhn.fhir.jpa.dao.index.IdHelperService; -import ca.uhn.fhir.jpa.model.cross.ResourcePersistentId; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.api.server.RequestDetails; import java.util.List; public interface IElasticsearchSvc { - -// IBundleProvider lastN(javax.servlet.http.HttpServletRequest theServletRequest, ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails); - -// IBundleProvider uniqueCodes(javax.servlet.http.HttpServletRequest theServletRequest, ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails); - - List executeLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, IdHelperService theIdHelperService); + List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java index 94276924896..9a48d3fc3b5 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java @@ -2,8 +2,14 @@ package ca.uhn.fhir.jpa.search.lastn; public class IndexConstants { - public static final String OBSERVATION_INDEX = "observation_index"; - public static final String CODE_INDEX = "code_index"; - public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; - public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity"; + public static final String OBSERVATION_INDEX = "observation_index"; + public static final String CODE_INDEX = "code_index"; + public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; + public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity"; + + public static final String SUBJECT_SEARCH_PARAM = "subject"; + public static final String PATIENT_SEARCH_PARAM = "patient"; + public static final String CODE_SEARCH_PARAM = "code"; + public static final String CATEGORY_SEARCH_PARAM = "category"; + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java deleted file mode 100644 index 10c0453431e..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java +++ /dev/null @@ -1,122 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.cli; - -import ca.uhn.fhir.jpa.search.lastn.json.IdJson; -import ca.uhn.fhir.jpa.search.lastn.json.IndexJson; -import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import org.hl7.fhir.r4.model.CodeableConcept; -import org.hl7.fhir.r4.model.Coding; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.UUID; - -public class MulticodeObservationsIntoElasticSearch { - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(MulticodeObservationsIntoElasticSearch.class); - - private static final ObjectMapper ourMapperNonPrettyPrint; - - static { - ourMapperNonPrettyPrint = new ObjectMapper(); - ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); - ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); - ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); - } - - public static void main(String[] theArgs) { - - for (int patientCount = 0; patientCount < 10 ; patientCount++) { - - String subject = "Patient/"+UUID.randomUUID().toString(); - - for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { - String nextResourceId = UUID.randomUUID().toString(); - - IdJson id = new IdJson(nextResourceId); - IndexJson documentIndex = new IndexJson(id); - - ObservationJson observationDocument = new ObservationJson(); - observationDocument.setIdentifier(nextResourceId); - observationDocument.setSubject(subject); - // Add three CodeableConcepts for category - List category = new ArrayList<>(); - // Create three codings and first category CodeableConcept - Coding categoryCoding1_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test heart-rate"); - Coding categoryCoding1_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test alternate heart-rate"); - Coding categoryCoding1_3 = new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test second alternate heart-rate"); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept(); - categoryCodeableConcept1.getCoding().add(categoryCoding1_1); - categoryCodeableConcept1.getCoding().add(categoryCoding1_2); - categoryCodeableConcept1.getCoding().add(categoryCoding1_3); - categoryCodeableConcept1.setText("Heart Rate Codeable Concept"); - category.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - Coding categoryCoding2_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test vital signs"); - Coding categoryCoding2_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test alternate vital signs"); - Coding categoryCoding2_3 = new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test second alternate vital signs"); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept(); - categoryCodeableConcept2.getCoding().add(categoryCoding2_1); - categoryCodeableConcept2.getCoding().add(categoryCoding2_2); - categoryCodeableConcept2.getCoding().add(categoryCoding2_3); - categoryCodeableConcept2.setText("Vital Signs Codeable Concept"); - category.add(categoryCodeableConcept2); - // Create three codings and third category CodeableConcept - Coding categoryCoding3_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test vital signs panel"); - Coding categoryCoding3_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test alternate vital signs panel"); - Coding categoryCoding3_3 = new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test second alternate vital signs panel"); - CodeableConcept categoryCodeableConcept3 = new CodeableConcept(); - categoryCodeableConcept3.getCoding().add(categoryCoding3_1); - categoryCodeableConcept3.getCoding().add(categoryCoding3_2); - categoryCodeableConcept3.getCoding().add(categoryCoding3_3); - categoryCodeableConcept3.setText("Vital Signs Panel Codeable Concept"); - category.add(categoryCodeableConcept3); - observationDocument.setCategories(category); - - Coding codeCoding1 = new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test observation code"); - Coding codeCoding2 = new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test observation code"); - Coding codeCoding3 = new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test observation code"); - CodeableConcept code = new CodeableConcept(); - code.getCoding().add(codeCoding1); - code.getCoding().add(codeCoding2); - code.getCoding().add(codeCoding3); - code.setText("Observation code CodeableConcept"); - observationDocument.setCode(code); - observationDocument.setCode_concept_id("multicode_test_normalized_code"); - - Date effectiveDtm = new Date(); - observationDocument.setEffectiveDtm(effectiveDtm); - - StringWriter stringWriter = new StringWriter(); - - File outputFile = new File("Observations_multiplecodes.json"); - try { - FileOutputStream outputStream = new FileOutputStream(outputFile, true); - ourMapperNonPrettyPrint.writeValue(stringWriter, documentIndex); - stringWriter.append('\n'); - ourMapperNonPrettyPrint.writeValue(stringWriter, observationDocument); - ourMapperNonPrettyPrint.writeValue(outputStream, documentIndex); - outputStream.write('\n'); - ourMapperNonPrettyPrint.writeValue(outputStream, observationDocument); - outputStream.write('\n'); - outputStream.flush(); - outputStream.close(); - } catch (IOException theE) { - theE.printStackTrace(); - } - } - - } - - ourLog.info("Upload complete"); - - } - -} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java deleted file mode 100644 index 300fbba9315..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java +++ /dev/null @@ -1,128 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.cli; - -import ca.uhn.fhir.jpa.search.lastn.ElasticsearchBulkIndexSvcImpl; -import ca.uhn.fhir.jpa.search.lastn.IndexConstants; -import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; -import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; -import ca.uhn.fhir.jpa.search.lastn.util.SimpleStopWatch; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import org.apache.commons.lang3.RandomStringUtils; -import org.hl7.fhir.r4.model.CodeableConcept; -import org.hl7.fhir.r4.model.Coding; - -import java.io.IOException; -import java.util.*; - -public class OneMillionPatientsIntoElasticSearch { -// private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(OneMillionPatientsIntoElasticSearch.class); - - private static final ObjectMapper ourMapperNonPrettyPrint; - - static { - ourMapperNonPrettyPrint = new ObjectMapper(); - ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); - ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); - ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); - } - - public static void main(String[] theArgs) throws IOException { - - ElasticsearchBulkIndexSvcImpl elasticsearchSvc = new ElasticsearchBulkIndexSvcImpl("localhost",9301, "elastic", "changeme"); - try { - - SimpleStopWatch stopwatch = new SimpleStopWatch(); - - List observationCodeIds = new ArrayList<>(); - List observationCodes = new ArrayList<>(); - - for (int codeCount = 0; codeCount < 1000; codeCount++) { - String code = RandomStringUtils.random(10,true,true); - Coding codeCoding = new Coding("http://mycodes.org/fhir/observation-code", code, "test observation code " + code); - CodeableConcept codeConcept = new CodeableConcept(); - codeConcept.getCoding().add(codeCoding); - codeConcept.setText("test observation code concept " + code); - String codeableConceptId = UUID.randomUUID().toString(); - observationCodeIds.add(codeableConceptId); - observationCodes.add(codeConcept); - CodeJson codeDocument = new CodeJson(codeConcept, codeableConceptId); - String printedObservationDocument = ourMapperNonPrettyPrint.writeValueAsString(codeDocument); - System.out.println(printedObservationDocument); - System.out.println(codeableConceptId); - elasticsearchSvc.addToBulkIndexRequest("code_index", codeableConceptId, printedObservationDocument, IndexConstants.CODE_DOCUMENT_TYPE); - if ((codeCount+1)%250 == 0) { - elasticsearchSvc.executeBulkIndex(); - long elapsedTime = stopwatch.getElapsedTime(); - stopwatch.restart(); - System.out.println("Elapsed processing time = " + elapsedTime/1000 + "s"); - System.out.println("Average processing time/code = " + elapsedTime/5000 + "ms"); - } - } - - for (int patientCount = 0; patientCount < 1000000 ; patientCount++) { - String subject = "Patient/"+UUID.randomUUID().toString(); - ArrayList observationCodesSubSet = new ArrayList<>(); - ArrayList observationCodeIdsSubSet = new ArrayList<>(); - for (int observationCount = 0; observationCount < 15 ; observationCount++) { - int codeIndex = (int) (1000 * Math.random()); - observationCodesSubSet.add(observationCodes.get(codeIndex)); - observationCodeIdsSubSet.add(observationCodeIds.get(codeIndex)); - } - int repeatedCodeIndex = (int) (1000 * Math.random()); - CodeableConcept repeatedCoding = observationCodes.get(repeatedCodeIndex); - for (int observationCount = 0; observationCount < 10 ; observationCount++ ) { - observationCodesSubSet.add(repeatedCoding); - observationCodeIdsSubSet.add(observationCodeIds.get(repeatedCodeIndex)); - } - int entryCount = 0; - for (int codingCount=0; codingCount category = new ArrayList<>(); - Coding categoryCoding = new Coding("http://mycodes.org/fhir/category-code", "test-category-code", "test category display"); - CodeableConcept categoryCodeableConcept = new CodeableConcept(); - categoryCodeableConcept.getCoding().add(categoryCoding); - categoryCodeableConcept.setText("Test Category CodeableConcept Text"); - category.add(categoryCodeableConcept); - observationDocument.setCategories(category); - observationDocument.setCode(observationCodesSubSet.get(codingCount)); - observationDocument.setCode_concept_id(observationCodeIdsSubSet.get(codingCount)); - Calendar observationDate = new GregorianCalendar(); - observationDate.add(Calendar.HOUR, -10 + entryCount); - entryCount++; - Date effectiveDtm = observationDate.getTime(); - observationDocument.setEffectiveDtm(effectiveDtm); - - String printedObservationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationDocument); - elasticsearchSvc.addToBulkIndexRequest("observation_index", nextResourceId, printedObservationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE ); - } - if ((patientCount+1)%100 == 0) { - System.out.println("Entries created = " + (patientCount+1)*25); - } - if ((patientCount+1)%250 == 0) { - elasticsearchSvc.executeBulkIndex(); - long elapsedTime = stopwatch.getElapsedTime(); - stopwatch.restart(); - System.out.println("Elapsed processing time = " + elapsedTime/1000 + "s"); - System.out.println("Average processing time/observation = " + elapsedTime/5000 + "ms"); - } - - - } - - if (elasticsearchSvc.bulkRequestPending()) { - elasticsearchSvc.executeBulkIndex(); - } - - // ourLog.info("Upload complete"); - } finally { - elasticsearchSvc.closeClient(); - } - - } - -} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java deleted file mode 100644 index 3a6d2218a3a..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java +++ /dev/null @@ -1,116 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.cli; - -import ca.uhn.fhir.jpa.search.lastn.json.IdJson; -import ca.uhn.fhir.jpa.search.lastn.json.IndexJson; -import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import org.hl7.fhir.r4.model.CodeableConcept; -import org.hl7.fhir.r4.model.Coding; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.UUID; - -public class OneThousandObservationsIntoElasticSearch { - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(OneThousandObservationsIntoElasticSearch.class); - - private static final ObjectMapper ourMapperNonPrettyPrint; - - static { - ourMapperNonPrettyPrint = new ObjectMapper(); - ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); - ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); - ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); - } - - public static void main(String[] theArgs) { - - for (int patientCount = 0; patientCount < 3 ; patientCount++) { - - String subject = "Patient/"+UUID.randomUUID().toString(); - - for ( int entryCount = 0; entryCount < 1100 ; entryCount++ ) { - String nextResourceId = UUID.randomUUID().toString(); - - IdJson id = new IdJson(nextResourceId); - IndexJson documentIndex = new IndexJson(id); - - ObservationJson observationDocument = new ObservationJson(); - observationDocument.setIdentifier(nextResourceId); - observationDocument.setSubject(subject); - // Add three CodeableConcepts for category - List category = new ArrayList<>(); - // Create three codings and first category CodeableConcept - Coding categoryCoding1_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test heart-rate"); - Coding categoryCoding1_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test alternate heart-rate"); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept(); - categoryCodeableConcept1.getCoding().add(categoryCoding1_1); - categoryCodeableConcept1.getCoding().add(categoryCoding1_2); - categoryCodeableConcept1.setText("Heart Rate CodeableConcept"); - category.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - Coding categoryCoding2_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test vital signs"); - Coding categoryCoding2_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test alternate vital signs"); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept(); - categoryCodeableConcept2.getCoding().add(categoryCoding2_1); - categoryCodeableConcept2.getCoding().add(categoryCoding2_2); - categoryCodeableConcept2.setText("Vital Signs CodeableConcept"); - category.add(categoryCodeableConcept2); - // Create three codings and third category CodeableConcept - Coding categoryCoding3_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test vital signs panel"); - Coding categoryCoding3_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test alternate vital signs panel"); - CodeableConcept categoryCodeableConcept3 = new CodeableConcept(); - categoryCodeableConcept3.getCoding().add(categoryCoding3_1); - categoryCodeableConcept3.getCoding().add(categoryCoding3_2); - categoryCodeableConcept3.setText("Vital Signs Panel CodeableConcept"); - category.add(categoryCodeableConcept3); - observationDocument.setCategories(category); - - Coding codeCoding1 = new Coding("http://mycodes.org/fhir/observation-code", "test-code_" + entryCount, "test observation code"); - Coding codeCoding2 = new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code_" + entryCount, "test observation code"); - Coding codeCoding3 = new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code" + entryCount, "test observation code"); - CodeableConcept code = new CodeableConcept(); - code.getCoding().add(codeCoding1); - code.getCoding().add(codeCoding2); - code.getCoding().add(codeCoding3); - code.setText("Observation code CodeableConcept " + entryCount); - observationDocument.setCode(code); - observationDocument.setCode_concept_id("multicode_test_normalized_code_" + entryCount); - - Date effectiveDtm = new Date(); - observationDocument.setEffectiveDtm(effectiveDtm); - - StringWriter stringWriter = new StringWriter(); - - File outputFile = new File("one_thousand_observations.json"); - try { - FileOutputStream outputStream = new FileOutputStream(outputFile, true); - ourMapperNonPrettyPrint.writeValue(stringWriter, documentIndex); - stringWriter.append('\n'); - ourMapperNonPrettyPrint.writeValue(stringWriter, observationDocument); - ourMapperNonPrettyPrint.writeValue(outputStream, documentIndex); - outputStream.write('\n'); - ourMapperNonPrettyPrint.writeValue(outputStream, observationDocument); - outputStream.write('\n'); - outputStream.flush(); - outputStream.close(); - } catch (IOException theE) { - theE.printStackTrace(); - } - } - - } - - ourLog.info("Upload complete"); - - } - -} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java deleted file mode 100644 index bfe2e82bd8f..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java +++ /dev/null @@ -1,153 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.cli; - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.parser.IParser; -import ca.uhn.fhir.parser.LenientErrorHandler; -import ca.uhn.fhir.jpa.search.lastn.json.IdJson; -import ca.uhn.fhir.jpa.search.lastn.json.IndexJson; -import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.google.common.base.Charsets; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; -import org.apache.commons.io.IOUtils; -import org.hl7.fhir.r4.model.*; -import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; -import org.springframework.core.io.Resource; -import org.springframework.core.io.support.PathMatchingResourcePatternResolver; - -import java.io.*; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - -public class UploadSampleDatasetIntoElasticSearch { - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(UploadSampleDatasetIntoElasticSearch.class); - - private static final ObjectMapper ourMapperNonPrettyPrint; - - static { - ourMapperNonPrettyPrint = new ObjectMapper(); - ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); - ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); - ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); - } - - public static void main(String[] theArgs) { - - FhirContext myFhirCtx = FhirContext.forR4(); - myFhirCtx.getRestfulClientFactory().setSocketTimeout(120000); - - PathMatchingResourcePatternResolver provider = new PathMatchingResourcePatternResolver(); - final Resource[] bundleResources; - try { - bundleResources = provider.getResources("*.json.bz2"); - } catch (IOException e) { - throw new RuntimeException("Unexpected error during transmission: " + e.toString(), e); - } - - AtomicInteger index = new AtomicInteger(); - - Arrays.stream(bundleResources).forEach( - resource -> { - index.incrementAndGet(); - - InputStream resIs = null; - String nextBundleString; - try { - resIs = resource.getInputStream(); - resIs = new BZip2CompressorInputStream(resIs); - nextBundleString = IOUtils.toString(resIs, Charsets.UTF_8); - } catch (IOException e) { - ourLog.error("Failure reading: {}", resource.getFilename(), e); - return; - } finally { - try { - if (resIs != null) { - resIs.close(); - } - } catch (final IOException ioe) { - // ignore - } - } - - ourLog.info("Uploading {}/{} - {} ({} bytes)", index, bundleResources.length, resource.getFilename(), nextBundleString.length()); - - /* - * SMART demo apps rely on the use of LOINC 3141-9 (Body Weight Measured) - * instead of LOINC 29463-7 (Body Weight) - */ - nextBundleString = nextBundleString.replace("\"29463-7\"", "\"3141-9\""); - - IParser parser = myFhirCtx.newJsonParser(); - parser.setParserErrorHandler(new LenientErrorHandler(false)); - Bundle bundle = parser.parseResource(Bundle.class, nextBundleString); - - for (BundleEntryComponent nextEntry : bundle.getEntry()) { - - /* - * Synthea gives resources UUIDs with urn:uuid: prefix, which is only - * used for placeholders. We're going to use these as the actual resource - * IDs, so we strip the prefix. - */ - String nextResourceId = nextEntry.getFullUrl(); - if (nextResourceId == null) { - nextResourceId = UUID.randomUUID().toString(); - } - - nextResourceId = nextResourceId.replace("urn:uuid:", ""); - nextEntry.getResource().setId(nextResourceId); - nextEntry.setFullUrl(nextResourceId); - - if (nextEntry.getResource().getResourceType().equals(ResourceType.Observation)) { - - IdJson id = new IdJson(nextResourceId); - IndexJson documentIndex = new IndexJson(id); - - org.hl7.fhir.r4.model.Observation observation = (Observation) nextEntry.getResource(); - ObservationJson observationDocument = new ObservationJson(); - observationDocument.setIdentifier(nextResourceId); - String subject = "Patient/"+observation.getSubject().getReference(); - observationDocument.setSubject(subject); - List category = observation.getCategory(); - observationDocument.setCategories(category); - observationDocument.setCode_concept_id(category.get(0).getCodingFirstRep().getSystem() + "/" + category.get(0).getCodingFirstRep().getCode()); - CodeableConcept code = observation.getCode(); - observationDocument.setCode(code); - Date effectiveDtm = observation.getEffectiveDateTimeType().getValue(); - observationDocument.setEffectiveDtm(effectiveDtm); - - StringWriter stringWriter = new StringWriter(); - File outputFile = new File("Observations.json"); - try { - FileOutputStream outputStream = new FileOutputStream(outputFile, true); - ourMapperNonPrettyPrint.writeValue(stringWriter, documentIndex); - stringWriter.append('\n'); - ourMapperNonPrettyPrint.writeValue(stringWriter, observationDocument); - ourMapperNonPrettyPrint.writeValue(outputStream, documentIndex); - outputStream.write('\n'); - ourMapperNonPrettyPrint.writeValue(outputStream, observationDocument); - outputStream.write('\n'); - outputStream.flush(); - outputStream.close(); - } catch (IOException theE) { - theE.printStackTrace(); - } - System.out.println(stringWriter.toString()); - - } - - } - - } - ); - - ourLog.info("Upload complete"); - - } - -} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java index f04edb8baf8..ebf3fc9e094 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java @@ -22,6 +22,10 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config { private static final Logger ourLog = LoggerFactory.getLogger(TestR4ConfigWithElasticSearch.class); private static final String ELASTIC_VERSION = "6.5.4"; + protected final String elasticsearchHost = "localhost"; + protected final String elasticsearchUserId = ""; + protected final String elasticsearchPassword = ""; + @Override @Bean @@ -38,9 +42,9 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config { .setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE) .setIndexManagementWaitTimeoutMillis(10000) .setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW) - .setRestUrl("http://localhost:" + httpPort) - .setUsername("") - .setPassword("") + .setRestUrl("http://"+ elasticsearchHost + ":" + httpPort) + .setUsername(elasticsearchUserId) + .setPassword(elasticsearchPassword) .apply(retVal); return retVal; @@ -65,7 +69,6 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config { return embeddedElastic; } - @PreDestroy public void stop() { embeddedElasticSearch().stop(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java new file mode 100644 index 00000000000..9ff6bc4c491 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java @@ -0,0 +1,16 @@ +package ca.uhn.fhir.jpa.config; + +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class TestR4ConfigWithElasticsearchClient extends TestR4ConfigWithElasticSearch { + + @Bean() + public ElasticsearchSvcImpl myElasticsearchSvc() { + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java index 1bafb4f7e86..593fa2bd9fb 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java @@ -2,20 +2,16 @@ package ca.uhn.fhir.jpa.dao.lastn; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao; -import ca.uhn.fhir.jpa.dao.lastn.config.TestIntegratedObservationIndexSearchConfig; +import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.parser.LenientErrorHandler; -import ca.uhn.fhir.rest.param.ReferenceParam; -import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.param.*; import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; -import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import com.google.common.base.Charsets; import org.apache.commons.io.IOUtils; -import org.shadehapi.elasticsearch.action.search.SearchRequest; -import org.shadehapi.elasticsearch.action.search.SearchResponse; import org.hl7.fhir.r4.model.*; import org.junit.After; import org.junit.Before; @@ -34,7 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestIntegratedObservationIndexSearchConfig.class }) +@ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class }) public class IntegratedObservationIndexedSearchParamLastNTest { @Autowired @@ -128,29 +124,16 @@ public class IntegratedObservationIndexedSearchParamLastNTest { SearchParameterMap searchParameterMap = new SearchParameterMap(); ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); - searchParameterMap.add("subject", subjectParam); + searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam))); TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); - searchParameterMap.add("category", categoryParam); + searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); - searchParameterMap.add("code", codeParam); + searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - // execute Observation ID search - Terms Aggregation - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsTermsSearchRequest(1000, searchParameterMap, 3); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); assertEquals(1, observationIdsOnly.size()); - ObservationJson observationIdOnly = observationIdsOnly.get(0); - assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); - - // execute Observation ID search - Composite Aggregation - searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); - responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); - - assertEquals(1, observationIdsOnly.size()); - observationIdOnly = observationIdsOnly.get(0); - assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + assertEquals(RESOURCEPID, observationIdsOnly.get(0)); } @@ -278,19 +261,13 @@ public class IntegratedObservationIndexedSearchParamLastNTest { SearchParameterMap searchParameterMap = new SearchParameterMap(); // execute Observation ID search - Composite Aggregation - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 1); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1); assertEquals(20, observationIdsOnly.size()); - ObservationJson observationIdOnly = observationIdsOnly.get(0); - searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); - responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); assertEquals(38, observationIdsOnly.size()); - observationIdOnly = observationIdsOnly.get(0); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java index d117a41de72..2d5e6515291 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.lastn; -import ca.uhn.fhir.jpa.dao.lastn.config.TestObservationIndexSearchConfig; +import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticSearch; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; @@ -19,7 +19,7 @@ import java.util.*; import static org.junit.Assert.assertEquals; @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestObservationIndexSearchConfig.class }) +@ContextConfiguration(classes = { TestR4ConfigWithElasticSearch.class }) public class PersistObservationIndexedSearchParamLastNTest { @Autowired diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java deleted file mode 100644 index 5f8be3aac8d..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java +++ /dev/null @@ -1,23 +0,0 @@ -package ca.uhn.fhir.jpa.dao.lastn.config; - -import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -import java.io.IOException; - -@Configuration -@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory", - basePackages = {"ca.uhn.fhir.jpa.dao.data"}) -@EnableTransactionManagement -public class TestIntegratedObservationIndexSearchConfig extends TestObservationIndexSearchConfig { - - @Bean() - public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { - int elasticsearchPort = embeddedElasticSearch().getHttpPort(); - return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); - } - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java deleted file mode 100644 index 64dab0a32b0..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java +++ /dev/null @@ -1,93 +0,0 @@ -package ca.uhn.fhir.jpa.dao.lastn.config; - -import ca.uhn.fhir.context.ConfigurationException; -import ca.uhn.fhir.jpa.config.TestR4Config; -import ca.uhn.fhir.jpa.config.r4.BaseR4Config; -import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory; -import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; -import org.hibernate.cfg.AvailableSettings; -import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus; -import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.transaction.annotation.EnableTransactionManagement; -import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; -import pl.allegro.tech.embeddedelasticsearch.PopularProperties; - -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.Properties; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -@Configuration -@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory", - basePackages = {"ca.uhn.fhir.jpa.dao.data"}) -@EnableTransactionManagement -public class TestObservationIndexSearchConfig extends TestR4Config { - - final String elasticsearchHost = "127.0.0.1"; - final String elasticsearchUserId = ""; - final String elasticsearchPassword = ""; - - private static final String ELASTIC_VERSION = "6.5.4"; - - @Override - public Properties jpaProperties() { - - Properties extraProperties = new Properties(); - extraProperties.put("hibernate.dialect", org.hibernate.dialect.H2Dialect.class); - extraProperties.put("hibernate.format_sql", "false"); - extraProperties.put("hibernate.show_sql", "false"); - extraProperties.put(AvailableSettings.HBM2DDL_AUTO, "update"); - extraProperties.put("hibernate.jdbc.batch_size", "5000"); - extraProperties.put("hibernate.cache.use_query_cache", "false"); - extraProperties.put("hibernate.cache.use_second_level_cache", "false"); - extraProperties.put("hibernate.cache.use_structured_entries", "false"); - extraProperties.put("hibernate.cache.use_minimal_puts", "false"); - extraProperties.put("hibernate.search.model_mapping", LuceneSearchMappingFactory.class.getName()); - extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); - extraProperties.put("hibernate.search.default.worker.execution", "sync"); - - int elasticsearchPort = embeddedElasticSearch().getHttpPort(); - new ElasticsearchHibernatePropertiesBuilder() - .setDebugRefreshAfterWrite(true) - .setDebugPrettyPrintJsonLog(true) - .setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE) - .setIndexManagementWaitTimeoutMillis(10000) - .setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW) - .setRestUrl("http://" + elasticsearchHost + ":" + elasticsearchPort) - .setUsername(elasticsearchUserId) - .setPassword(elasticsearchPassword) - .apply(extraProperties); - - extraProperties.setProperty("hibernate.search.default.elasticsearch.refresh_after_write", "true"); - return extraProperties; - } - - @Bean - public EmbeddedElastic embeddedElasticSearch() { - EmbeddedElastic embeddedElastic; - try { - embeddedElastic = EmbeddedElastic.builder() - .withElasticVersion(ELASTIC_VERSION) - .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) - .withSetting(PopularProperties.HTTP_PORT, 0) - .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) - .withStartTimeout(60, TimeUnit.SECONDS) - .build() - .start(); - } catch (IOException | InterruptedException e) { - throw new ConfigurationException(e); - } - - return embeddedElastic; - } - - @PreDestroy - public void stop() { - embeddedElasticSearch().stop(); - } - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java deleted file mode 100644 index 30797965ebb..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LastNTest.java +++ /dev/null @@ -1,134 +0,0 @@ -package ca.uhn.fhir.jpa.dao.r4; - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.jpa.api.dao.*; -import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.dao.lastn.config.TestIntegratedObservationIndexSearchConfig; -import ca.uhn.fhir.jpa.dao.BaseJpaTest; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; -import ca.uhn.fhir.util.TestUtil; -import org.hl7.fhir.instance.model.api.IIdType; -import org.hl7.fhir.r4.model.*; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.PlatformTransactionManager; - -import javax.servlet.http.HttpServletRequest; -import java.util.*; - -import static org.junit.Assert.*; -import static org.mockito.Mockito.when; - -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestIntegratedObservationIndexSearchConfig.class }) -public class FhirResourceDaoR4LastNTest extends BaseJpaTest { - - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4LastNTest.class); - - @Autowired - @Qualifier("myPatientDaoR4") - protected IFhirResourceDaoPatient myPatientDao; - - @Autowired - @Qualifier("myObservationDaoR4") - protected IFhirResourceDaoObservation myObservationDao; - - @Autowired - protected DaoConfig myDaoConfig; - - @Autowired - protected FhirContext myFhirCtx; - - @Autowired - protected PlatformTransactionManager myPlatformTransactionManager; - - @Override - protected FhirContext getContext() { - return myFhirCtx; - } - - @Override - protected PlatformTransactionManager getTxManager() { - return myPlatformTransactionManager; - } - - @Before - public void beforeDisableResultReuse() { - myDaoConfig.setReuseCachedSearchResultsForMillis(null); - } - - private ServletRequestDetails mockSrd() { - return mySrd; - } - - @Test - public void testLastN() { - Patient pt = new Patient(); - pt.addName().setFamily("Lastn").addGiven("Arthur"); - IIdType ptId = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); - - Map requestParameters = new HashMap<>(); - String[] maxParam = new String[1]; - maxParam[0] = "1"; - requestParameters.put("max", maxParam); - when(mySrd.getParameters()).thenReturn(requestParameters); - - Map observationIds = new HashMap<>(); - for(int observationIdx = 0 ; observationIdx < 20 ; observationIdx++) { - Calendar observationDate = new GregorianCalendar(); - String idxSuffix = String.valueOf(observationIdx); - - Observation obs = new Observation(); - obs.getText().setDivAsString("
OBSTEXT0_" + idxSuffix + "
"); - obs.getSubject().setReferenceElement(ptId); - obs.getCode().addCoding().setCode("CODE_" + idxSuffix).setSystem("http://mycode.com"); - obs.setValue(new StringType("obsvalue0_" + idxSuffix)); - observationDate.add(Calendar.HOUR, -2); - Date effectiveDtm = observationDate.getTime(); - obs.setEffective(new DateTimeType(effectiveDtm)); - observationIds.put(myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless(), obs.getCode()); - - obs = new Observation(); - obs.getText().setDivAsString("
OBSTEXT1_" + idxSuffix + "
"); - obs.getSubject().setReferenceElement(ptId); - obs.getCode().addCoding().setCode("CODE_" + idxSuffix).setSystem("http://mycode.com"); - obs.setValue(new StringType("obsvalue1_" + idxSuffix)); - observationDate.add(Calendar.HOUR, -1); - effectiveDtm = observationDate.getTime(); - obs.setEffective(new DateTimeType(effectiveDtm)); - observationIds.put(myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless(), obs.getCode()); - - obs = new Observation(); - obs.getText().setDivAsString("
OBSTEXT2_" + idxSuffix + "
"); - obs.getSubject().setReferenceElement(ptId); - obs.getCode().addCoding().setCode("CODE_" + idxSuffix).setSystem("http://mycode.com"); - obs.setValue(new StringType("obsvalue2_" + idxSuffix)); - observationDate.add(Calendar.HOUR, -0); - effectiveDtm = observationDate.getTime(); - obs.setEffective(new DateTimeType(effectiveDtm)); - observationIds.put(myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless(), obs.getCode()); - } - - HttpServletRequest request; - List actual; - SearchParameterMap params = new SearchParameterMap(); - params.setLastN(true); - actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); - - assertEquals(20, actual.size()); - - } - - @AfterClass - public static void afterClassClearContext() { - TestUtil.clearAllStaticFieldsForUnitTest(); - } - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNTest.java new file mode 100644 index 00000000000..c63a1a8932e --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNTest.java @@ -0,0 +1,472 @@ +package ca.uhn.fhir.jpa.dao.r4; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.dao.*; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; +import ca.uhn.fhir.jpa.dao.BaseJpaTest; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.*; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.util.TestUtil; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.*; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.transaction.PlatformTransactionManager; + +import java.util.*; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class }) +public class FhirResourceDaoR4SearchLastNTest extends BaseJpaTest { + + @Autowired + @Qualifier("myPatientDaoR4") + protected IFhirResourceDaoPatient myPatientDao; + + @Autowired + @Qualifier("myObservationDaoR4") + protected IFhirResourceDaoObservation myObservationDao; + + @Autowired + protected DaoConfig myDaoConfig; + + @Autowired + protected FhirContext myFhirCtx; + + @Autowired + protected PlatformTransactionManager myPlatformTransactionManager; + + @Override + protected FhirContext getContext() { + return myFhirCtx; + } + + @Override + protected PlatformTransactionManager getTxManager() { + return myPlatformTransactionManager; + } + + private final String observationCd0 = "code0"; + private final String observationCd1 = "code1"; + private final String observationCd2 = "code2"; + + private final String categoryCd0 = "category0"; + private final String categoryCd1 = "category1"; + private final String categoryCd2 = "category2"; + + private final String codeSystem = "http://mycode.com"; + private final String categorySystem = "http://mycategory.com"; + + // Using static variables including the flag below so that we can initalize the database and indexes once + // (all of the tests only read from the DB and indexes and so no need to re-initialze them for each test). + private static boolean dataLoaded = false; + + private static IIdType patient0Id = null; + private static IIdType patient1Id = null; + private static IIdType patient2Id = null; + + private static final Map observationPatientMap = new HashMap<>(); + private static final Map observationCategoryMap = new HashMap<>(); + private static final Map observationCodeMap = new HashMap<>(); + private static final Map observationEffectiveMap = new HashMap<>(); + + @Before + public void beforeCreateTestPatientsAndObservations() { + // Using a static flag here to ensure that load is only done once. Reason for this is that we cannot + // access Autowired objects in @BeforeClass method. + if(!dataLoaded) { + Patient pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Arthur"); + patient0Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + createObservationsForPatient(patient0Id); + pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Johnathan"); + patient1Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + createObservationsForPatient(patient1Id); + pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Michael"); + patient2Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + createObservationsForPatient(patient2Id); + dataLoaded = true; + } + + } + + private void createObservationsForPatient(IIdType thePatientId) { + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd2, 5); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd0, 10); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd1, 5); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd2, categoryCd2, 5); + } + + private void createFiveObservationsForPatientCodeCategory(IIdType thePatientId, String theObservationCode, String theCategoryCode, + Integer theTimeOffset) { + Calendar observationDate = new GregorianCalendar(); + + for (int idx=0; idx<5; idx++ ) { + Observation obs = new Observation(); + obs.getSubject().setReferenceElement(thePatientId); + obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem); + obs.setValue(new StringType(theObservationCode + "_0")); + observationDate.add(Calendar.HOUR, -theTimeOffset+idx); + Date effectiveDtm = observationDate.getTime(); + obs.setEffective(new DateTimeType(effectiveDtm)); + obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem); + String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue(); + observationPatientMap.put(observationId, thePatientId.getValue()); + observationCategoryMap.put(observationId, theCategoryCode); + observationCodeMap.put(observationId, theObservationCode); + observationEffectiveMap.put(observationId, effectiveDtm); + } + } + + private ServletRequestDetails mockSrd() { + return mySrd; + } + + @Test + public void testLastNNoParams() { + + SearchParameterMap params = new SearchParameterMap(); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null,90); + } + + private void executeTestCase(SearchParameterMap params, List sortedPatients, List sortedObservationCodes, List theCategories, int expectedObservationCount) { + List actual; + params.setLastN(true); + + Map requestParameters = new HashMap<>(); + String[] maxParam = new String[1]; + maxParam[0] = "100"; + requestParameters.put("max", maxParam); + when(mySrd.getParameters()).thenReturn(requestParameters); + + actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + + assertEquals(expectedObservationCount, actual.size()); + + validateSorting(actual, sortedPatients, sortedObservationCodes, theCategories); + } + + private void validateSorting(List theObservationIds, List thePatientIds, List theCodes, List theCategores) { + int theNextObservationIdx = 0; + // Validate patient grouping + for (String patientId : thePatientIds) { + assertEquals(patientId, observationPatientMap.get(theObservationIds.get(theNextObservationIdx))); + theNextObservationIdx = validateSortingWithinPatient(theObservationIds,theNextObservationIdx,theCodes, theCategores, patientId); + } + assertEquals(theObservationIds.size(), theNextObservationIdx); + } + + private int validateSortingWithinPatient(List theObservationIds, int theFirstObservationIdxForPatient, List theCodes, + List theCategories, String thePatientId) { + int theNextObservationIdx = theFirstObservationIdxForPatient; + for (String codeValue : theCodes) { + assertEquals(codeValue, observationCodeMap.get(theObservationIds.get(theNextObservationIdx))); + // Validate sorting within code group + theNextObservationIdx = validateSortingWithinCode(theObservationIds,theNextObservationIdx, + observationCodeMap.get(theObservationIds.get(theNextObservationIdx)), theCategories, thePatientId); + } + return theNextObservationIdx; + } + + private int validateSortingWithinCode(List theObservationIds, int theFirstObservationIdxForPatientAndCode, String theObservationCode, + List theCategories, String thePatientId) { + int theNextObservationIdx = theFirstObservationIdxForPatientAndCode; + Date lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); + theNextObservationIdx++; + while(theObservationCode.equals(observationCodeMap.get(theObservationIds.get(theNextObservationIdx))) + && thePatientId.equals(observationPatientMap.get(theObservationIds.get(theNextObservationIdx)))) { + // Check that effective date is before that of the previous observation. + assertTrue(lastEffectiveDt.compareTo(observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx))) > 0); + lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); + + // Check that observation is in one of the specified categories (if applicable) + if (theCategories != null && !theCategories.isEmpty()) { + assertTrue(theCategories.contains(observationCategoryMap.get(theObservationIds.get(theNextObservationIdx)))); + } + theNextObservationIdx++; + if (theNextObservationIdx >= theObservationIds.size()) { + // Have reached the end of the Observation list. + break; + } + } + return theNextObservationIdx; + } + + @Test + public void testLastNSinglePatient() { + + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,30); + + params = new SearchParameterMap(); + ReferenceParam patientParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); + + sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,30); + } + + private ReferenceAndListParam buildReferenceAndListParam(ReferenceParam... theReference) { + ReferenceOrListParam myReferenceOrListParam = new ReferenceOrListParam(); + for (ReferenceParam referenceParam : theReference) { + myReferenceOrListParam.addOr(referenceParam); + } + return new ReferenceAndListParam().addAnd(myReferenceOrListParam); + } + + @Test + public void testLastNMultiplePatients() { + + // Two Subject parameters. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null,60); + + // Two Patient parameters + params = new SearchParameterMap(); + ReferenceParam patientParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam patientParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(patientParam1, patientParam3)); + + sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + executeTestCase(params,sortedPatients, sortedObservationCodes, null,60); + + } + + @Test + public void testLastNSingleCategory() { + + // One category parameter. + SearchParameterMap params = new SearchParameterMap(); + TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd0); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); + + // Another category parameter. + params = new SearchParameterMap(); + categoryParam = new TokenParam(categorySystem, categoryCd2); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + myCategories = new ArrayList<>(); + myCategories.add(categoryCd2); + + sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); + + } + + @Test + public void testLastNMultipleCategories() { + + // Two category parameters. + SearchParameterMap params = new SearchParameterMap(); + TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd0); + TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd1); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd0); + myCategories.add(categoryCd1); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 60); + } + + @Test + public void testLastNSingleCode() { + + // One code parameter. + SearchParameterMap params = new SearchParameterMap(); + TokenParam code = new TokenParam(codeSystem, observationCd0); + params.add(Observation.SP_CODE, buildTokenAndListParam(code)); + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null, 45); + + // Another code parameter. + params = new SearchParameterMap(); + code = new TokenParam(codeSystem, observationCd2); + params.add(Observation.SP_CODE, buildTokenAndListParam(code)); + sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null, 15); + + } + + @Test + public void testLastNMultipleCodes() { + + // Two code parameters. + SearchParameterMap params = new SearchParameterMap(); + TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); + TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1); + params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null, 75); + + } + + @Test + public void testLastNSinglePatientCategoryCode() { + + // One patient, category and code. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + TokenParam code = new TokenParam(codeSystem, observationCd0); + params.add(Observation.SP_CODE, buildTokenAndListParam(code)); + TokenParam category = new TokenParam(categorySystem, categoryCd2); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(category)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 5); + + } + + @Test + public void testLastNMultiplePatientsCategoriesCodes() { + + // Two patients, categories and codes. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + + TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); + TokenParam codeParam2 = new TokenParam(codeSystem, observationCd2); + params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd2); + + TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd1); + TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd2); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd1); + myCategories.add(categoryCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); + + } + + private TokenAndListParam buildTokenAndListParam(TokenParam... theToken) { + TokenOrListParam myTokenOrListParam = new TokenOrListParam(); + for (TokenParam tokenParam : theToken) { + myTokenOrListParam.addOr(tokenParam); + } + return new TokenAndListParam().addAnd(myTokenOrListParam); + } + + @AfterClass + public static void afterClassClearContext() { + TestUtil.clearAllStaticFieldsForUnitTest(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java index 50da9dbe3da..2f495cdea88 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java @@ -1,9 +1,7 @@ package ca.uhn.fhir.jpa.search.lastn; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.param.ReferenceParam; -import ca.uhn.fhir.rest.param.TokenParam; -import ca.uhn.fhir.rest.param.TokenParamModifier; +import ca.uhn.fhir.rest.param.*; import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchConfig; import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; @@ -11,9 +9,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import org.hl7.fhir.r4.model.Observation; import org.junit.*; -import org.shadehapi.elasticsearch.action.search.SearchRequest; -import org.shadehapi.elasticsearch.action.search.SearchResponse; import org.hl7.fhir.r4.model.CodeableConcept; import org.hl7.fhir.r4.model.Coding; import org.junit.runner.RunWith; @@ -28,298 +25,352 @@ import static ca.uhn.fhir.jpa.search.lastn.IndexConstants.*; import static org.junit.Assert.*; @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestElasticsearchConfig.class } ) +@ContextConfiguration(classes = {TestElasticsearchConfig.class}) public class LastNElasticsearchSvcMultipleObservationsTest { - @Autowired - private ElasticsearchSvcImpl elasticsearchSvc; + @Autowired + private ElasticsearchSvcImpl elasticsearchSvc; - private static ObjectMapper ourMapperNonPrettyPrint; + private static ObjectMapper ourMapperNonPrettyPrint; - private Map>> createdPatientObservationMap = new HashMap<>(); + private final Map>> createdPatientObservationMap = new HashMap<>(); - @BeforeClass - public static void beforeClass() { - ourMapperNonPrettyPrint = new ObjectMapper(); - ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); - ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); - ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); - } + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } - @Before - public void before() throws IOException { - createMultiplePatientsAndObservations(); - } + @Before + public void before() throws IOException { + createMultiplePatientsAndObservations(); + } - @After - public void after() throws IOException { - elasticsearchSvc.deleteAllDocuments(OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(CODE_INDEX); - } + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(CODE_INDEX); + } - @Test - public void testLastNNoCriteriaQuery() throws IOException { + @Test + public void testLastNNoCriteriaQuery() { - // execute Observation ID search (Terms Aggregation) last 3 observations for each patient - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, null, 3); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + // execute Observation ID search (Composite Aggregation) last 3 observations for each patient + List observations = elasticsearchSvc.executeLastNWithAllFields(null, 3); - validateQueryResponse(observationIdsOnly); + validateQueryResponse(observations); - // execute Observation ID search (Composite Aggregation) last 3 observations for each patient - searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, null, 3); - responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + } - validateQueryResponse(observationIdsOnly); + private void validateQueryResponse(List observationIdsOnly) { + assertEquals(60, observationIdsOnly.size()); - // Retrieve all Observation codes - SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); - SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - List codes = elasticsearchSvc.buildCodeResult(response); - assertEquals(2, codes.size()); + // Observation documents should be grouped by subject, then by observation code, and then sorted by effective date/time + // within each observation code. Verify the grouping by creating a nested Map. + Map>> queriedPatientObservationMap = new HashMap<>(); + ObservationJson previousObservationJson = null; + for (ObservationJson observationJson : observationIdsOnly) { + assertNotNull(observationJson.getIdentifier()); + assertNotNull(observationJson.getSubject()); + assertNotNull(observationJson.getCode_concept_id()); + assertNotNull(observationJson.getEffectiveDtm()); + if (previousObservationJson == null) { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(), observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } else if (observationJson.getSubject().equals(previousObservationJson.getSubject())) { + if (observationJson.getCode_concept_id().equals(previousObservationJson.getCode_concept_id())) { + queriedPatientObservationMap.get(observationJson.getSubject()).get(observationJson.getCode_concept_id()). + add(observationJson.getEffectiveDtm()); + } else { + Map> codeObservationDateMap = queriedPatientObservationMap.get(observationJson.getSubject()); + // Ensure that code concept was not already retrieved out of order for this subject/patient. + assertFalse(codeObservationDateMap.containsKey(observationJson.getCode_concept_id())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + codeObservationDateMap.put(observationJson.getCode_concept_id(), observationDates); + } + } else { + // Ensure that subject/patient was not already retrieved out of order + assertFalse(queriedPatientObservationMap.containsKey(observationJson.getSubject())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(), observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } + previousObservationJson = observationJson; + } - } + // Finally check that only the most recent effective date/time values were returned and in the correct order. + for (String subjectId : queriedPatientObservationMap.keySet()) { + Map> queriedObservationCodeMap = queriedPatientObservationMap.get(subjectId); + Map> createdObservationCodeMap = createdPatientObservationMap.get(subjectId); + for (String observationCode : queriedObservationCodeMap.keySet()) { + List queriedObservationDates = queriedObservationCodeMap.get(observationCode); + List createdObservationDates = createdObservationCodeMap.get(observationCode); + for (int dateIdx = 0; dateIdx < queriedObservationDates.size(); dateIdx++) { + assertEquals(createdObservationDates.get(dateIdx), queriedObservationDates.get(dateIdx)); + } + } + } - private void validateQueryResponse(List observationIdsOnly) { - assertEquals(60, observationIdsOnly.size()); + } - // Observation documents should be grouped by subject, then by observation code, and then sorted by effective date/time - // within each observation code. Verify the grouping by creating a nested Map. - Map>> queriedPatientObservationMap = new HashMap<>(); - ObservationJson previousObservationJson = null; - for (ObservationJson observationJson : observationIdsOnly) { - assertNotNull(observationJson.getIdentifier()); - assertNotNull(observationJson.getSubject()); - assertNotNull(observationJson.getCode_concept_id()); - assertNotNull(observationJson.getEffectiveDtm()); - if (previousObservationJson == null) { - ArrayList observationDates = new ArrayList<>(); - observationDates.add(observationJson.getEffectiveDtm()); - Map> codeObservationMap = new HashMap<>(); - codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); - queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); - } else if (observationJson.getSubject().equals(previousObservationJson.getSubject())) { - if (observationJson.getCode_concept_id().equals(previousObservationJson.getCode_concept_id())) { - queriedPatientObservationMap.get(observationJson.getSubject()).get(observationJson.getCode_concept_id()). - add(observationJson.getEffectiveDtm()); - } else { - Map> codeObservationDateMap = queriedPatientObservationMap.get(observationJson.getSubject()); - // Ensure that code concept was not already retrieved out of order for this subject/patient. - assertFalse(codeObservationDateMap.containsKey(observationJson.getCode_concept_id())); - ArrayList observationDates = new ArrayList<>(); - observationDates.add(observationJson.getEffectiveDtm()); - codeObservationDateMap.put(observationJson.getCode_concept_id(),observationDates); - } - } else { - // Ensure that subject/patient was not already retrieved out of order - assertFalse(queriedPatientObservationMap.containsKey(observationJson.getSubject())); - ArrayList observationDates = new ArrayList<>(); - observationDates.add(observationJson.getEffectiveDtm()); - Map> codeObservationMap = new HashMap<>(); - codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); - queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); - } - previousObservationJson = observationJson; - } + @Test + public void testLastNMultiPatientMultiCodeHashMultiCategoryHash() { + // Multiple Subject references + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", "3"); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", "5"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); + TokenParam categoryParam1 = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); + TokenParam categoryParam2 = new TokenParam("http://mycodes.org/fhir/observation-category", "test-vital-signs"); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); + TokenParam codeParam1 = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); + TokenParam codeParam2 = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-2"); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - // Finally check that only the most recent effective date/time values were returned and in the correct order. - for(String subjectId : queriedPatientObservationMap.keySet()) { - Map> queriedObservationCodeMap = queriedPatientObservationMap.get(subjectId); - Map> createdObservationCodeMap = createdPatientObservationMap.get(subjectId); - for(String observationCode : queriedObservationCodeMap.keySet()) { - List queriedObservationDates = queriedObservationCodeMap.get(observationCode); - List createdObservationDates = createdObservationCodeMap.get(observationCode); - for (int dateIdx=0; dateIdx observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); - } + assertEquals(20, observations.size()); - @Test - public void testLastNMultiPatientMultiCodeHashMultiCategoryHash() throws IOException { - // Include subject and patient - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); - searchParameterMap.add("subject", subjectParam); - ReferenceParam patientParam = new ReferenceParam("Patient", "", "8"); - searchParameterMap.add("patient", patientParam); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); - searchParameterMap.add("category", categoryParam); - TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); - searchParameterMap.add("code", codeParam); + // Repeat with multiple Patient parameter + searchParameterMap = new SearchParameterMap(); + ReferenceParam patientParam1 = new ReferenceParam("Patient", "", "8"); + ReferenceParam patientParam2 = new ReferenceParam("Patient", "", "6"); + searchParameterMap.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam1, patientParam2)); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); - assertEquals(10, observationIdsOnly.size()); + assertEquals(20, observations.size()); - } + } - @Test - public void testLastNCodeCodeOnlyCategoryCodeOnly() throws IOException { - // Include subject and patient - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam ("test-heart-rate"); - searchParameterMap.add("category", categoryParam); - TokenParam codeParam = new TokenParam("test-code-1"); - searchParameterMap.add("code", codeParam); + private ReferenceAndListParam buildReferenceAndListParam(ReferenceParam... theReference) { + ReferenceOrListParam myReferenceOrListParam = new ReferenceOrListParam(); + for (ReferenceParam referenceParam : theReference) { + myReferenceOrListParam.addOr(referenceParam); + } + return new ReferenceAndListParam().addAnd(myReferenceOrListParam); + } - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + private TokenAndListParam buildTokenAndListParam(TokenParam... theToken) { + TokenOrListParam myTokenOrListParam = new TokenOrListParam(); + for (TokenParam tokenParam : theToken) { + myTokenOrListParam.addOr(tokenParam); + } + return new TokenAndListParam().addAnd(myTokenOrListParam); + } - assertEquals(5, observationIdsOnly.size()); + @Test + public void testLastNCodeCodeOnlyCategoryCodeOnly() { + // Include subject + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + TokenParam categoryParam = new TokenParam("test-heart-rate"); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + TokenParam codeParam = new TokenParam("test-code-1"); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - } + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); - @Test - public void testLastNCodeSystemOnlyCategorySystemOnly() throws IOException { - // Include subject and patient - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", null); - searchParameterMap.add("category", categoryParam); - TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); - searchParameterMap.add("code", codeParam); + assertEquals(5, observations.size()); - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + } - assertEquals(10, observationIdsOnly.size()); - } + @Test + public void testLastNCodeSystemOnlyCategorySystemOnly() { + // Include subject and patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", null); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - @Test - public void testLastNCodeCodeTextCategoryTextOnly() throws IOException { - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam("test-heart-rate display"); - categoryParam.setModifier(TokenParamModifier.TEXT); - searchParameterMap.add("category", categoryParam); - TokenParam codeParam = new TokenParam("test-code-1 display"); - codeParam.setModifier(TokenParamModifier.TEXT); - searchParameterMap.add("code", codeParam); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + assertEquals(10, observations.size()); + } - assertEquals(5, observationIdsOnly.size()); + @Test + public void testLastNCodeCodeTextCategoryTextOnly() { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + TokenParam categoryParam = new TokenParam("test-heart-rate display"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + TokenParam codeParam = new TokenParam("test-code-1 display"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - } + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); - private void createMultiplePatientsAndObservations() throws IOException { - // Create CodeableConcepts for two Codes, each with three codings. - String codeableConceptId1 = UUID.randomUUID().toString(); - CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); - codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); - // TODO: uncomment the following once there is a solution to supporting multiple codings for Observation Code + assertEquals(5, observations.size()); + + } + + @Test + public void testLastNNoMatchQueries() { + // Invalid Patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam patientParam = new ReferenceParam("Patient", "", "10"); + searchParameterMap.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + assertEquals(0, observations.size()); + + // Invalid subject + searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "10"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + assertEquals(0, observations.size()); + + // Invalid observation code + searchParameterMap = new SearchParameterMap(); + subjectParam = new ReferenceParam("Patient", "", "9"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate"); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-999"); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + assertEquals(0, observations.size()); + + // Invalid category code + searchParameterMap = new SearchParameterMap(); + subjectParam = new ReferenceParam("Patient", "", "9"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-not-a-category"); + searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); + searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + assertEquals(0, observations.size()); + + } + + private void createMultiplePatientsAndObservations() throws IOException { + // Create CodeableConcepts for two Codes, each with three codings. + String codeableConceptId1 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); + // TODO: uncomment the following once there is a solution to supporting multiple codings for Observation Code // codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); // codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); - CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1); - String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1); + CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1); + String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1); - String codeableConceptId2 = UUID.randomUUID().toString(); - CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); - codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); - // TODO: uncomment the following once there is a solution to supporting multiple codings for Observation Code + String codeableConceptId2 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); + // TODO: uncomment the following once there is a solution to supporting multiple codings for Observation Code // codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); // codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); - CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2); - String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2); + CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2); + String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2); - // Create CodeableConcepts for two categories, each with three codings. - List category1 = new ArrayList<>(); - // Create three codings and first category CodeableConcept - category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); - category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); - category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); - List categoryConcepts1 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); - categoryCodeableConcept1.setCoding(category1); - categoryConcepts1.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - List category2 = new ArrayList<>(); - category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); - category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); - category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); - List categoryConcepts2 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); - categoryCodeableConcept2.setCoding(category2); - categoryConcepts2.add(categoryCodeableConcept2); + // Create CodeableConcepts for two categories, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); - for (int patientCount = 0; patientCount < 10 ; patientCount++) { + for (int patientCount = 0; patientCount < 10; patientCount++) { - String subject = String.valueOf(patientCount); + String subject = String.valueOf(patientCount); - for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + for (int entryCount = 0; entryCount < 10; entryCount++) { - ObservationJson observationJson = new ObservationJson(); - String identifier = String.valueOf((entryCount + patientCount*10)); - observationJson.setIdentifier(identifier); - observationJson.setSubject(subject); + ObservationJson observationJson = new ObservationJson(); + String identifier = String.valueOf((entryCount + patientCount * 10)); + observationJson.setIdentifier(identifier); + observationJson.setSubject(subject); - if (entryCount%2 == 1) { - observationJson.setCategories(categoryConcepts1); - observationJson.setCode(codeableConceptField1); - observationJson.setCode_concept_id(codeableConceptId1); - assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId1, codeJson1Document, CODE_DOCUMENT_TYPE)); - } else { - observationJson.setCategories(categoryConcepts2); - observationJson.setCode(codeableConceptField2); - observationJson.setCode_concept_id(codeableConceptId2); - assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId2, codeJson2Document, CODE_DOCUMENT_TYPE)); - } + if (entryCount % 2 == 1) { + observationJson.setCategories(categoryConcepts1); + observationJson.setCode(codeableConceptField1); + observationJson.setCode_concept_id(codeableConceptId1); + assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId1, codeJson1Document, CODE_DOCUMENT_TYPE)); + } else { + observationJson.setCategories(categoryConcepts2); + observationJson.setCode(codeableConceptField2); + observationJson.setCode_concept_id(codeableConceptId2); + assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId2, codeJson2Document, CODE_DOCUMENT_TYPE)); + } - Calendar observationDate = new GregorianCalendar(); - observationDate.add(Calendar.HOUR, -10 + entryCount); - Date effectiveDtm = observationDate.getTime(); - observationJson.setEffectiveDtm(effectiveDtm); + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observationJson.setEffectiveDtm(effectiveDtm); - String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationJson); - assertTrue(elasticsearchSvc.performIndex(OBSERVATION_INDEX, identifier,observationDocument, OBSERVATION_DOCUMENT_TYPE)); + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationJson); + assertTrue(elasticsearchSvc.performIndex(OBSERVATION_INDEX, identifier, observationDocument, OBSERVATION_DOCUMENT_TYPE)); - if (createdPatientObservationMap.containsKey(subject)) { - Map> observationCodeMap = createdPatientObservationMap.get(subject); - if (observationCodeMap.containsKey(observationJson.getCode_concept_id())) { - List observationDates = observationCodeMap.get(observationJson.getCode_concept_id()); - // Want dates to be sorted in descending order - observationDates.add(0, effectiveDtm); - // Only keep the three most recent dates for later check. - if(observationDates.size() > 3) { - observationDates.remove(3); - } - } else { - ArrayList observationDates = new ArrayList<>(); - observationDates.add(effectiveDtm); - observationCodeMap.put(observationJson.getCode_concept_id(), observationDates); - } - } else { - ArrayList observationDates = new ArrayList<>(); - observationDates.add(effectiveDtm); - Map> codeObservationMap = new HashMap<>(); - codeObservationMap.put(observationJson.getCode_concept_id(), observationDates); - createdPatientObservationMap.put(subject, codeObservationMap); - } - } - } + if (createdPatientObservationMap.containsKey(subject)) { + Map> observationCodeMap = createdPatientObservationMap.get(subject); + if (observationCodeMap.containsKey(observationJson.getCode_concept_id())) { + List observationDates = observationCodeMap.get(observationJson.getCode_concept_id()); + // Want dates to be sorted in descending order + observationDates.add(0, effectiveDtm); + // Only keep the three most recent dates for later check. + if (observationDates.size() > 3) { + observationDates.remove(3); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + observationCodeMap.put(observationJson.getCode_concept_id(), observationDates); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(), observationDates); + createdPatientObservationMap.put(subject, codeObservationMap); + } + } + } - try { - Thread.sleep(2000L); - } catch (InterruptedException theE) { - theE.printStackTrace(); - } + try { + Thread.sleep(2000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } - } + } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java index 3c210d42453..e2dc8f23bb8 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java @@ -5,14 +5,12 @@ import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.param.ReferenceParam; -import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.model.dstu2.resource.Observation; +import ca.uhn.fhir.rest.param.*; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; -import org.shadehapi.elasticsearch.action.search.SearchRequest; -import org.shadehapi.elasticsearch.action.search.SearchResponse; import org.hl7.fhir.r4.model.CodeableConcept; import org.hl7.fhir.r4.model.Coding; import org.junit.*; @@ -28,215 +26,185 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestElasticsearchConfig.class } ) +@ContextConfiguration(classes = {TestElasticsearchConfig.class}) public class LastNElasticsearchSvcSingleObservationTest { - @Autowired - ElasticsearchSvcImpl elasticsearchSvc; + @Autowired + ElasticsearchSvcImpl elasticsearchSvc; - static ObjectMapper ourMapperNonPrettyPrint; + static ObjectMapper ourMapperNonPrettyPrint; - final String RESOURCEPID = "123"; -// final String SUBJECTID = "4567"; -// final String SUBJECTTYPEANDID = "Patient/4567"; - final String SUBJECTID = "Patient/4567"; - final Date EFFECTIVEDTM = new Date(); - final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category"; - final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; - final String CATEGORYSECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-category"; - final String CATEGORYTHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-category"; - final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; - final String FIRSTCATEGORYFIRSTCODINGDISPLAY = "test-heart-rate display"; - final String FIRSTCATEGORYSECONDCODINGCODE = "test-alt-heart-rate"; - final String FIRSTCATEGORYSECONDCODINGDISPLAY = "test-alt-heart-rate display"; - final String FIRSTCATEGORYTHIRDCODINGCODE = "test-2nd-alt-heart-rate"; - final String FIRSTCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-heart-rate display"; - final String SECONDCATEGORYTEXT = "Test Codeable Concept Field for for second category"; - final String SECONDCATEGORYFIRSTCODINGCODE = "test-vital-signs"; - final String SECONDCATEGORYFIRSTCODINGDISPLAY = "test-vital-signs display"; - final String SECONDCATEGORYSECONDCODINGCODE = "test-alt-vitals"; - final String SECONDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; - final String SECONDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals"; - final String SECONDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals display"; - final String THIRDCATEGORYTEXT = "Test Codeable Concept Field for third category"; - final String THIRDCATEGORYFIRSTCODINGCODE = "test-vital-panel"; - final String THIRDCATEGORYFIRSTCODINGDISPLAY = "test-vitals-panel display"; - final String THIRDCATEGORYSECONDCODINGCODE = "test-alt-vitals-panel"; - final String THIRDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; - final String THIRDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals-panel"; - final String THIRDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals-panel display"; + final String RESOURCEPID = "123"; + final String SUBJECTID = "Patient/4567"; + final Date EFFECTIVEDTM = new Date(); + final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category"; + final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; + final String CATEGORYSECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-category"; + final String CATEGORYTHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-category"; + final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; + final String FIRSTCATEGORYFIRSTCODINGDISPLAY = "test-heart-rate display"; + final String FIRSTCATEGORYSECONDCODINGCODE = "test-alt-heart-rate"; + final String FIRSTCATEGORYSECONDCODINGDISPLAY = "test-alt-heart-rate display"; + final String FIRSTCATEGORYTHIRDCODINGCODE = "test-2nd-alt-heart-rate"; + final String FIRSTCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-heart-rate display"; + final String SECONDCATEGORYTEXT = "Test Codeable Concept Field for for second category"; + final String SECONDCATEGORYFIRSTCODINGCODE = "test-vital-signs"; + final String SECONDCATEGORYFIRSTCODINGDISPLAY = "test-vital-signs display"; + final String SECONDCATEGORYSECONDCODINGCODE = "test-alt-vitals"; + final String SECONDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String SECONDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals"; + final String SECONDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals display"; + final String THIRDCATEGORYTEXT = "Test Codeable Concept Field for third category"; + final String THIRDCATEGORYFIRSTCODINGCODE = "test-vital-panel"; + final String THIRDCATEGORYFIRSTCODINGDISPLAY = "test-vitals-panel display"; + final String THIRDCATEGORYSECONDCODINGCODE = "test-alt-vitals-panel"; + final String THIRDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String THIRDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals-panel"; + final String THIRDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals-panel display"; - final String OBSERVATIONSINGLECODEID = UUID.randomUUID().toString(); - final String OBSERVATIONCODETEXT = "Test Codeable Concept Field for Code"; - final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; - final String CODEFIRSTCODINGCODE = "test-code"; - final String CODEFIRSTCODINGDISPLAY = "test-code display"; - final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; - final String CODESECONDCODINGCODE = "test-alt-code"; - final String CODESECONDCODINGDISPLAY = "test-alt-code display"; - final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; - final String CODETHIRDCODINGCODE = "test-second-alt-code"; - final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; + final String OBSERVATIONSINGLECODEID = UUID.randomUUID().toString(); + final String OBSERVATIONCODETEXT = "Test Codeable Concept Field for Code"; + final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; + final String CODEFIRSTCODINGCODE = "test-code"; + final String CODEFIRSTCODINGDISPLAY = "test-code display"; + final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; + final String CODESECONDCODINGCODE = "test-alt-code"; + final String CODESECONDCODINGDISPLAY = "test-alt-code display"; + final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; + final String CODETHIRDCODINGCODE = "test-second-alt-code"; + final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; - @BeforeClass - public static void beforeClass() { - ourMapperNonPrettyPrint = new ObjectMapper(); - ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); - ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); - ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); - } + } - // @Before - public void before() throws IOException { - elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); - } + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } - @After - public void after() throws IOException { - elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); - } + @Test + public void testSingleObservationQuery() throws IOException { - @Test - public void testSingleObservationQuery() throws IOException { + createSingleObservation(); - createSingleObservation(); + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); + searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam))); + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); + TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); + searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); - searchParameterMap.add("subject", subjectParam); - TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); - searchParameterMap.add("category", categoryParam); - TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); - searchParameterMap.add("code", codeParam); + // execute Observation ID search + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); - // execute Observation ID search - Terms Aggregation - SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 3); - SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + assertEquals(1, observationIdsOnly.size()); + assertEquals(RESOURCEPID, observationIdsOnly.get(0)); - assertEquals(1, observationIdsOnly.size()); - ObservationJson observationIdOnly = observationIdsOnly.get(0); - assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + // execute Observation search for all search fields + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3); - // execute Observation ID search - Composite Aggregation - searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 3); - responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); - observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + validateFullObservationSearch(observations); + } - assertEquals(1, observationIdsOnly.size()); - observationIdOnly = observationIdsOnly.get(0); - assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + private void validateFullObservationSearch(List observations) throws IOException { - // execute full Observation search - Terms Aggregation - SearchRequest searchRequestAllFields = elasticsearchSvc.buildObservationAllFieldsTermsSearchRequest(1000, searchParameterMap, 3); - SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequestAllFields); - List observations = elasticsearchSvc.buildObservationTermsResults(response); + assertEquals(1, observations.size()); + ObservationJson observation = observations.get(0); + assertEquals(RESOURCEPID, observation.getIdentifier()); - validateFullObservationSearch(observations); + assertEquals(SUBJECTID, observation.getSubject()); + assertEquals(RESOURCEPID, observation.getIdentifier()); + assertEquals(EFFECTIVEDTM, observation.getEffectiveDtm()); + assertEquals(OBSERVATIONSINGLECODEID, observation.getCode_concept_id()); - // execute full Observation search - Composite Aggregation - searchRequestAllFields= elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); - response = elasticsearchSvc.executeSearchRequest(searchRequestAllFields); - observations = elasticsearchSvc.buildObservationCompositeResults(response); + List category_concept_text_values = observation.getCategory_concept_text(); + assertEquals(3, category_concept_text_values.size()); + assertEquals(FIRSTCATEGORYTEXT, category_concept_text_values.get(0)); + assertEquals(SECONDCATEGORYTEXT, category_concept_text_values.get(1)); + assertEquals(THIRDCATEGORYTEXT, category_concept_text_values.get(2)); - validateFullObservationSearch(observations); - } + List> category_codings_systems = observation.getCategory_coding_system(); + assertEquals(3, category_codings_systems.size()); + List category_coding_systems = category_codings_systems.get(0); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_coding_systems = category_codings_systems.get(1); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_coding_systems = category_codings_systems.get(2); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); - private void validateFullObservationSearch(List observations) throws IOException { + List> category_codings_codes = observation.getCategory_coding_code(); + assertEquals(3, category_codings_codes.size()); + List category_coding_codes = category_codings_codes.get(0); + assertEquals(3, category_coding_codes.size()); + assertEquals(FIRSTCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(1); + assertEquals(3, category_coding_codes.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(2); + assertEquals(3, category_coding_codes.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); - assertEquals(1, observations.size()); - ObservationJson observation = observations.get(0); - assertEquals(RESOURCEPID, observation.getIdentifier()); + List> category_codings_displays = observation.getCategory_coding_display(); + assertEquals(3, category_codings_displays.size()); + List category_coding_displays = category_codings_displays.get(0); + assertEquals(FIRSTCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(1); + assertEquals(3, category_coding_displays.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(2); + assertEquals(3, category_coding_displays.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); -// assertEquals(SUBJECTTYPEANDID, observation.getSubject()); - assertEquals(SUBJECTID, observation.getSubject()); - assertEquals(RESOURCEPID, observation.getIdentifier()); - assertEquals(EFFECTIVEDTM, observation.getEffectiveDtm()); - assertEquals(OBSERVATIONSINGLECODEID, observation.getCode_concept_id()); + List> category_codings_code_system_hashes = observation.getCategory_coding_code_system_hash(); + assertEquals(3, category_codings_code_system_hashes.size()); + List category_coding_code_system_hashes = category_codings_code_system_hashes.get(0); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(1); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(2); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); - List category_concept_text_values = observation.getCategory_concept_text(); - assertEquals(3,category_concept_text_values.size()); - assertEquals(FIRSTCATEGORYTEXT, category_concept_text_values.get(0)); - assertEquals(SECONDCATEGORYTEXT, category_concept_text_values.get(1)); - assertEquals(THIRDCATEGORYTEXT, category_concept_text_values.get(2)); + String code_concept_text_values = observation.getCode_concept_text(); + assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); - List> category_codings_systems = observation.getCategory_coding_system(); - assertEquals(3,category_codings_systems.size()); - List category_coding_systems = category_codings_systems.get(0); - assertEquals(3, category_coding_systems.size()); - assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); - assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); - assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); - category_coding_systems = category_codings_systems.get(1); - assertEquals(3, category_coding_systems.size()); - assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); - assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); - assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); - category_coding_systems = category_codings_systems.get(2); - assertEquals(3, category_coding_systems.size()); - assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); - assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); - assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); - - List> category_codings_codes = observation.getCategory_coding_code(); - assertEquals(3, category_codings_codes.size()); - List category_coding_codes = category_codings_codes.get(0); - assertEquals(3, category_coding_codes.size()); - assertEquals(FIRSTCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); - assertEquals(FIRSTCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); - assertEquals(FIRSTCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); - category_coding_codes = category_codings_codes.get(1); - assertEquals(3, category_coding_codes.size()); - assertEquals(SECONDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); - assertEquals(SECONDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); - assertEquals(SECONDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); - category_coding_codes = category_codings_codes.get(2); - assertEquals(3, category_coding_codes.size()); - assertEquals(THIRDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); - assertEquals(THIRDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); - assertEquals(THIRDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); - - List> category_codings_displays = observation.getCategory_coding_display(); - assertEquals(3, category_codings_displays.size()); - List category_coding_displays = category_codings_displays.get(0); - assertEquals(FIRSTCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); - assertEquals(FIRSTCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); - assertEquals(FIRSTCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); - category_coding_displays = category_codings_displays.get(1); - assertEquals(3, category_coding_displays.size()); - assertEquals(SECONDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); - assertEquals(SECONDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); - assertEquals(SECONDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); - category_coding_displays = category_codings_displays.get(2); - assertEquals(3, category_coding_displays.size()); - assertEquals(THIRDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); - assertEquals(THIRDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); - assertEquals(THIRDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); - - List> category_codings_code_system_hashes = observation.getCategory_coding_code_system_hash(); - assertEquals(3, category_codings_code_system_hashes.size()); - List category_coding_code_system_hashes = category_codings_code_system_hashes.get(0); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); - category_coding_code_system_hashes = category_codings_code_system_hashes.get(1); - assertEquals(3, category_coding_code_system_hashes.size()); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); - category_coding_code_system_hashes = category_codings_code_system_hashes.get(2); - assertEquals(3, category_coding_code_system_hashes.size()); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); - - String code_concept_text_values = observation.getCode_concept_text(); - assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); - - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. /* List code_coding_systems = observation.getCode_coding_system(); assertEquals(3,code_coding_systems.size()); @@ -262,122 +230,119 @@ public class LastNElasticsearchSvcSingleObservationTest { assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), code_coding_code_system_hash.get(1)); assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), code_coding_code_system_hash.get(2)); */ - String code_coding_systems = observation.getCode_coding_system(); - assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems); + String code_coding_systems = observation.getCode_coding_system(); + assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems); - String code_coding_codes = observation.getCode_coding_code(); - assertEquals(CODEFIRSTCODINGCODE, code_coding_codes); + String code_coding_codes = observation.getCode_coding_code(); + assertEquals(CODEFIRSTCODINGCODE, code_coding_codes); - String code_coding_display = observation.getCode_coding_display(); - assertEquals(CODEFIRSTCODINGDISPLAY, code_coding_display); + String code_coding_display = observation.getCode_coding_display(); + assertEquals(CODEFIRSTCODINGDISPLAY, code_coding_display); - String code_coding_code_system_hash = observation.getCode_coding_code_system_hash(); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash); + String code_coding_code_system_hash = observation.getCode_coding_code_system_hash(); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash); - // Retrieve all Observation codes - SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); - SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); - List codes = elasticsearchSvc.buildCodeResult(response); - assertEquals(1, codes.size()); - CodeJson persistedObservationCode = codes.get(0); + // Retrieve all Observation codes + List codes = elasticsearchSvc.queryAllIndexedObservationCodes(1000); + assertEquals(1, codes.size()); + CodeJson persistedObservationCode = codes.get(0); - String persistedCodeConceptID = persistedObservationCode.getCodeableConceptId(); - assertEquals(OBSERVATIONSINGLECODEID, persistedCodeConceptID); - String persistedCodeConceptText = persistedObservationCode.getCodeableConceptText(); - assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText); + String persistedCodeConceptID = persistedObservationCode.getCodeableConceptId(); + assertEquals(OBSERVATIONSINGLECODEID, persistedCodeConceptID); + String persistedCodeConceptText = persistedObservationCode.getCodeableConceptText(); + assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText); - List persistedCodeCodingSystems = persistedObservationCode.getCoding_system(); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. + List persistedCodeCodingSystems = persistedObservationCode.getCoding_system(); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. // assertEquals(3,persistedCodeCodingSystems.size()); - assertEquals(1,persistedCodeCodingSystems.size()); - assertEquals(CODEFIRSTCODINGSYSTEM, persistedCodeCodingSystems.get(0)); + assertEquals(1, persistedCodeCodingSystems.size()); + assertEquals(CODEFIRSTCODINGSYSTEM, persistedCodeCodingSystems.get(0)); // assertEquals(CODESECONDCODINGSYSTEM, persistedCodeCodingSystems.get(1)); // assertEquals(CODETHIRDCODINGSYSTEM, persistedCodeCodingSystems.get(2)); - List persistedCodeCodingCodes = persistedObservationCode.getCoding_code(); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. + List persistedCodeCodingCodes = persistedObservationCode.getCoding_code(); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. // assertEquals(3, persistedCodeCodingCodes.size()); - assertEquals(1, persistedCodeCodingCodes.size()); - assertEquals(CODEFIRSTCODINGCODE, persistedCodeCodingCodes.get(0)); + assertEquals(1, persistedCodeCodingCodes.size()); + assertEquals(CODEFIRSTCODINGCODE, persistedCodeCodingCodes.get(0)); // assertEquals(CODESECONDCODINGCODE, persistedCodeCodingCodes.get(1)); // assertEquals(CODETHIRDCODINGCODE, persistedCodeCodingCodes.get(2)); - List persistedCodeCodingDisplays = persistedObservationCode.getCoding_display(); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. + List persistedCodeCodingDisplays = persistedObservationCode.getCoding_display(); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. // assertEquals(3, persistedCodeCodingDisplays.size()); - assertEquals(1, persistedCodeCodingDisplays.size()); - assertEquals(CODEFIRSTCODINGDISPLAY, persistedCodeCodingDisplays.get(0)); + assertEquals(1, persistedCodeCodingDisplays.size()); + assertEquals(CODEFIRSTCODINGDISPLAY, persistedCodeCodingDisplays.get(0)); // assertEquals(CODESECONDCODINGDISPLAY, persistedCodeCodingDisplays.get(1)); // assertEquals(CODETHIRDCODINGDISPLAY, persistedCodeCodingDisplays.get(2)); - List persistedCodeCodingCodeSystemHashes = persistedObservationCode.getCoding_code_system_hash(); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. + List persistedCodeCodingCodeSystemHashes = persistedObservationCode.getCoding_code_system_hash(); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. // assertEquals(3, persistedCodeCodingCodeSystemHashes.size()); - assertEquals(1, persistedCodeCodingCodeSystemHashes.size()); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(0)); + assertEquals(1, persistedCodeCodingCodeSystemHashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(0)); // assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(1)); // assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(2)); - } + } - private void createSingleObservation() throws IOException { - ObservationJson indexedObservation = new ObservationJson(); - indexedObservation.setIdentifier(RESOURCEPID); -// indexedObservation.setSubject(SUBJECTTYPEANDID); - indexedObservation.setSubject(SUBJECTID); - indexedObservation.setEffectiveDtm(EFFECTIVEDTM); + private void createSingleObservation() throws IOException { + ObservationJson indexedObservation = new ObservationJson(); + indexedObservation.setIdentifier(RESOURCEPID); + indexedObservation.setSubject(SUBJECTID); + indexedObservation.setEffectiveDtm(EFFECTIVEDTM); - // Add three CodeableConcepts for category - List categoryConcepts = new ArrayList<>(); - // Create three codings and first category CodeableConcept - List category1 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText(FIRSTCATEGORYTEXT); - category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY)); - category1.add(new Coding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY)); - category1.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY)); - categoryCodeableConcept1.setCoding(category1); - categoryConcepts.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - List category2 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText(SECONDCATEGORYTEXT); - category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY)); - category2.add(new Coding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY)); - category2.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY)); - categoryCodeableConcept2.setCoding(category2); - categoryConcepts.add(categoryCodeableConcept2); - // Create three codings and third category CodeableConcept - List category3 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText(THIRDCATEGORYTEXT); - category3.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY)); - category3.add(new Coding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY)); - category3.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY)); - categoryCodeableConcept3.setCoding(category3); - categoryConcepts.add(categoryCodeableConcept3); - indexedObservation.setCategories(categoryConcepts); + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText(FIRSTCATEGORYTEXT); + category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY)); + category1.add(new Coding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY)); + category1.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText(SECONDCATEGORYTEXT); + category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY)); + category2.add(new Coding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY)); + category2.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText(THIRDCATEGORYTEXT); + category3.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY)); + category3.add(new Coding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY)); + category3.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + indexedObservation.setCategories(categoryConcepts); - // Create CodeableConcept for Code with three codings. - indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID); - CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT); - codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY)); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. + // Create CodeableConcept for Code with three codings. + indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID); + CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT); + codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY)); + // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. // codeableConceptField.addCoding(new Coding(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE, CODESECONDCODINGDISPLAY)); // codeableConceptField.addCoding(new Coding(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE, CODETHIRDCODINGDISPLAY)); - indexedObservation.setCode(codeableConceptField); + indexedObservation.setCode(codeableConceptField); - String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); - assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); - CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); - String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); - assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); + CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); + String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); - try { - Thread.sleep(1000L); - } catch (InterruptedException theE) { - theE.printStackTrace(); - } + try { + Thread.sleep(1000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } - } + } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java deleted file mode 100644 index 973b19b00ba..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java +++ /dev/null @@ -1,53 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.config; - -import ca.uhn.fhir.context.ConfigurationException; -//import ca.uhn.fhir.jpa.search.lastn.ElasticsearchV5SvcImpl; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.transaction.annotation.EnableTransactionManagement; -import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; -import pl.allegro.tech.embeddedelasticsearch.PopularProperties; - -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -//@Configuration -//@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") -//@EnableTransactionManagement -public class TestElasticsearchV5Config { - - private final String elasticsearchHost = "127.0.0.1"; - private final String elasticsearchUserId = ""; - private final String elasticsearchPassword = ""; - - private static final String ELASTIC_VERSION = "5.6.16"; - -/* - @Bean() - public ElasticsearchV5SvcImpl myElasticsearchSvc() throws IOException { - int elasticsearchPort = embeddedElasticSearch().getHttpPort(); - return new ElasticsearchV5SvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); - } - - @Bean - public EmbeddedElastic embeddedElasticSearch() { - EmbeddedElastic embeddedElastic = null; - try { - embeddedElastic = EmbeddedElastic.builder() - .withElasticVersion(ELASTIC_VERSION) - .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) - .withSetting(PopularProperties.HTTP_PORT, 0) - .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) - .withStartTimeout(60, TimeUnit.SECONDS) - .build() - .start(); - } catch (IOException | InterruptedException e) { - throw new ConfigurationException(e); - } - - return embeddedElastic; - } -*/ -} From 197bf2b6a851d4918fb7c44b6b49832cc018d2e0 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Thu, 23 Apr 2020 23:21:45 -0400 Subject: [PATCH 10/31] Further refinements and test fixes. --- .../fhir/jpa/config/BaseConfigDstu3Plus.java | 8 + .../jpa/config/dstu3/BaseDstu3Config.java | 5 - .../uhn/fhir/jpa/config/r4/BaseR4Config.java | 7 +- .../uhn/fhir/jpa/config/r5/BaseR5Config.java | 5 - .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 54 ++- ...ObservationIndexedSearchParamLastNDao.java | 8 + .../FhirResourceDaoObservationDstu3.java | 64 +-- .../ObservationLastNIndexPersistDstu3Svc.java | 85 +++- .../ObservationLastNIndexPersistR4Svc.java | 97 ++++- .../ObservationLastNIndexPersistR5Svc.java | 95 +++- .../ObservationLastNIndexPersistSvc.java | 198 +++++++++ ...nIndexedCategoryCodeableConceptEntity.java | 9 - ...ervationIndexedSearchParamLastNEntity.java | 7 +- .../dao/r4/FhirResourceDaoObservationR4.java | 32 +- .../dao/r5/FhirResourceDaoObservationR5.java | 65 +-- .../search/lastn/ElasticsearchSvcImpl.java | 10 +- ...bservationIndexedSearchParamLastNTest.java | 274 ------------ ...bservationIndexedSearchParamLastNTest.java | 190 -------- ...ervationIndexedSearchParamLastNR4Test.java | 406 ++++++++++++++++++ .../lastn/config/TestElasticsearchConfig.java | 4 - .../extractor/BaseSearchParamExtractor.java | 284 ++++++++---- .../extractor/ISearchParamExtractor.java | 23 +- 22 files changed, 1175 insertions(+), 755 deletions(-) create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4Test.java diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java index 9dd3ebe2c29..4bc9abbb563 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java @@ -23,6 +23,7 @@ package ca.uhn.fhir.jpa.config; import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; import ca.uhn.fhir.context.support.IValidationSupport; import ca.uhn.fhir.jpa.dao.JpaPersistedResourceValidationSupport; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.term.TermCodeSystemStorageSvcImpl; import ca.uhn.fhir.jpa.term.TermDeferredStorageSvcImpl; import ca.uhn.fhir.jpa.term.TermReindexingSvcImpl; @@ -96,4 +97,11 @@ public abstract class BaseConfigDstu3Plus extends BaseConfig { @Bean public abstract ITermReadSvc terminologyService(); + + @Bean + public ObservationLastNIndexPersistSvc baseObservationLastNIndexpersistSvc() { + return new ObservationLastNIndexPersistSvc(); + } + + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java index 2b8cffed6b1..9e0d0207beb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java @@ -126,9 +126,4 @@ public class BaseDstu3Config extends BaseConfigDstu3Plus { return new TermReadSvcDstu3(); } - @Bean - public ObservationLastNIndexPersistDstu3Svc observationLastNIndexpersistSvc() { - return new ObservationLastNIndexPersistDstu3Svc(); - } - } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java index 36b309eb979..11730fbf192 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java @@ -7,10 +7,10 @@ import ca.uhn.fhir.jpa.config.BaseConfigDstu3Plus; import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.dao.TransactionProcessor; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR4Svc; import ca.uhn.fhir.jpa.dao.r4.TransactionProcessorVersionAdapterR4; import ca.uhn.fhir.jpa.provider.GraphQLProvider; -import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorR4; import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl; import ca.uhn.fhir.jpa.term.TermReadSvcR4; import ca.uhn.fhir.jpa.term.TermVersionAdapterSvcR4; @@ -129,9 +129,4 @@ public class BaseR4Config extends BaseConfigDstu3Plus { return new TermReadSvcR4(); } - @Bean - public ObservationLastNIndexPersistR4Svc observationLastNIndexpersistSvc() { - return new ObservationLastNIndexPersistR4Svc(); - } - } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java index 0d3f16c2c62..88e73510b39 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java @@ -129,9 +129,4 @@ public class BaseR5Config extends BaseConfigDstu3Plus { return new TermReadSvcR5(); } - @Bean - public ObservationLastNIndexPersistR5Svc observationLastNIndexpersistSvc() { - return new ObservationLastNIndexPersistR5Svc(); - } - } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index a3a5a5de9a7..68add2eb3de 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -320,7 +320,59 @@ public class SearchBuilder implements ISearchBuilder { } /* - * Fulltext search and lastn + * Fulltext search + */ + /* + if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { + if (myFulltextSearchSvc == null) { + if (myParams.containsKey(Constants.PARAM_TEXT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + } + } + + List pids; + if (myParams.getEverythingMode() != null) { + pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); + } else { + pids = myFulltextSearchSvc.search(myResourceName, myParams); + } + if (pids.isEmpty()) { + // Will never match + pids = Collections.singletonList(new ResourcePersistentId(-1L)); + } + + myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids))); + } else if (myParams.isLastN()) { + if (myIElasticsearchSvc == null) { + if (myParams.isLastN()) { + throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); + } + } + + if (myParams.isLastN()) { + Integer myMaxObservationsPerCode = null; + String[] maxCountParams = theRequest.getParameters().get("max"); + if (maxCountParams != null && maxCountParams.length > 0) { + myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]); + } else { + throw new InvalidRequestException("Max parameter is required for $lastn operation"); + } + List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); + for (String lastnResourceId : lastnResourceIds) { + lastnPids.add(myIdHelperService.resolveResourcePersistentIds(myResourceName, lastnResourceId)); + } + if (lastnPids.isEmpty()) { + // Will never match + pids = Collections.singletonList(new ResourcePersistentId(-1L)); + } + myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids))); + } + } + */ + /* + * lastn search */ if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { List lastnPids = new ArrayList<>(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java index 7a33dec68cd..b4a6209795f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java @@ -2,8 +2,16 @@ package ca.uhn.fhir.jpa.dao.data; import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @Repository public interface IObservationIndexedSearchParamLastNDao extends JpaRepository{ + @Query("" + + "SELECT t FROM ObservationIndexedSearchParamLastNEntity t " + + "WHERE t.myIdentifier = :identifier" + + "") + ObservationIndexedSearchParamLastNEntity findForIdentifier(@Param("identifier") String theIdentifier); + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java index 496eac6240d..dc4e7278b12 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java @@ -20,66 +20,32 @@ package ca.uhn.fhir.jpa.dao.dstu3; * #L% */ -import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; -import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistDstu3Svc; +import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; -import ca.uhn.fhir.model.api.IResource; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; -import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.param.DateRangeParam; -import ca.uhn.fhir.rest.param.StringAndListParam; -import ca.uhn.fhir.rest.param.StringParam; import org.hl7.fhir.instance.model.api.IBaseResource; -import org.hl7.fhir.instance.model.api.IIdType; -import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.hl7.fhir.dstu3.model.Observation; -import org.hl7.fhir.dstu3.model.Reference; import org.springframework.beans.factory.annotation.Autowired; import javax.servlet.http.HttpServletResponse; -import java.util.Collections; import java.util.Date; -public class FhirResourceDaoObservationDstu3 extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { +public class FhirResourceDaoObservationDstu3 extends BaseHapiFhirResourceDaoObservation { @Autowired - ObservationLastNIndexPersistDstu3Svc myObservationLastNIndexPersistDstu3Svc; - - private IBundleProvider doLastNOperation(IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequest) { - SearchParameterMap paramMap = new SearchParameterMap(); - if (theCount != null) { - paramMap.setCount(theCount.getValue()); - } - if (theContent != null) { - paramMap.add(Constants.PARAM_CONTENT, theContent); - } - if (theNarrative != null) { - paramMap.add(Constants.PARAM_TEXT, theNarrative); - } - paramMap.setIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive())); - paramMap.setEverythingMode(theId != null ? EverythingModeEnum.PATIENT_INSTANCE : EverythingModeEnum.PATIENT_TYPE); - paramMap.setSort(theSort); - paramMap.setLastUpdated(theLastUpdated); - if (theId != null) { - paramMap.add("_id", new StringParam(theId.getIdPart())); - } - - if (!isPagingProviderDatabaseBacked(theRequest)) { - paramMap.setLoadSynchronous(true); - } - - return mySearchCoordinatorSvc.registerSearch(this, paramMap, getResourceName(), new CacheControlDirective().parse(theRequest.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequest); - } + ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; @Override - public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse) { + public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse) { + + updateSearchParamsForLastn(theSearchParameterMap, theRequestDetails); + return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); } @@ -88,12 +54,14 @@ public class FhirResourceDaoObservationDstu3 extends BaseHapiFhirResourceDao categoryConcepts = new HashSet<>(); @@ -52,12 +84,17 @@ public class ObservationLastNIndexPersistDstu3Svc { String observationCodeNormalizedId = null; // Determine if a Normalized ID was created previously for Observation Code + boolean observationCodeUpdate = false; for (Coding codeCoding : codeCodeableConcept.getCoding()) { if (codeCoding.hasCode() && codeCoding.hasSystem()) { observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); } else { observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); } + if(observationCodeNormalizedId != null) { + observationCodeUpdate = true; + break; + } } // Generate a new a normalized ID if necessary if (observationCodeNormalizedId == null) { @@ -69,13 +106,27 @@ public class ObservationLastNIndexPersistDstu3Svc { for (Coding codeCoding : codeCodeableConcept.getCoding()) { codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); } - myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); - codeableConceptField = myObservationIndexedCodeableConceptSearchParamDao.findByCodeableConceptId(observationCodeNormalizedId); + if (observationCodeUpdate) { + myEntityManager.merge(codeableConceptField); + } else { + myEntityManager.persist(codeableConceptField); + } indexedObservation.setObservationCode(codeableConceptField); indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); - myResourceIndexedObservationLastNDao.save(indexedObservation); + if (observationIndexUpdate) { + myEntityManager.merge(indexedObservation); + } else { + myEntityManager.persist(indexedObservation); + } } + public void deleteObservationIndex(IBasePersistedResource theEntity) { + ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findForIdentifier(theEntity.getIdDt().getIdPart()); + if (deletedObservationLastNEntity != null) { + myEntityManager.remove(deletedObservationLastNEntity); + } + } + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java index ea77850dac0..d1d1f45e439 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java @@ -4,34 +4,76 @@ import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchPara import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; import ca.uhn.fhir.jpa.dao.lastn.entity.*; -import org.hl7.fhir.r4.model.CodeableConcept; -import org.hl7.fhir.r4.model.Coding; -import org.hl7.fhir.r4.model.Observation; +import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor; +import ca.uhn.fhir.jpa.searchparam.extractor.PathAndRef; +import org.hl7.fhir.r4.model.*; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.PersistenceContextType; +import java.util.*; +@Transactional(propagation = Propagation.REQUIRED) public class ObservationLastNIndexPersistR4Svc { + @PersistenceContext(type = PersistenceContextType.TRANSACTION) + protected EntityManager myEntityManager; + @Autowired IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; - @Autowired - IObservationIndexedCodeCodeableConceptSearchParamDao myObservationIndexedCodeableConceptSearchParamDao; - @Autowired IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; - public void indexObservation(Observation theObservation, String theSubjectId) { - ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + public void indexObservation(Observation theObservation) { + // Only index for lastn if Observation has a subject and effective date/time + if(theObservation.getSubject() == null || !theObservation.hasEffective()) { + return; + } + + // Determine most recent effective date/time + Date effectiveDtm = null; + if (theObservation.hasEffectiveDateTimeType()) { + effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); + } else if (theObservation.hasEffectiveInstantType()) { + effectiveDtm = theObservation.getEffectiveInstantType().getValue(); + } else if (theObservation.hasEffectivePeriod()) { + effectiveDtm = theObservation.getEffectivePeriod().getEnd(); + } else if (theObservation.hasEffectiveTiming()) { + List events = theObservation.getEffectiveTiming().getEvent(); + for (DateTimeType event : events) { + Date eventDtm = event.getValue(); + if (effectiveDtm == null || eventDtm.after(effectiveDtm)) { + effectiveDtm = eventDtm; + } + } + } + if (effectiveDtm == null) { + return; + } + + // Determine if an index already exists for Observation: + boolean observationIndexUpdate = false; + ObservationIndexedSearchParamLastNEntity indexedObservation = null; + if (theObservation.hasId()) { + indexedObservation = myResourceIndexedObservationLastNDao.findForIdentifier(theObservation.getIdElement().getIdPart()); + } + if (indexedObservation == null) { + indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + } else { + observationIndexUpdate = true; + } + + indexedObservation.setEffectiveDtm(effectiveDtm); + Reference subjectReference = theObservation.getSubject(); + String subjectId = subjectReference.getReference(); String resourcePID = theObservation.getIdElement().getIdPart(); indexedObservation.setIdentifier(resourcePID); - indexedObservation.setSubject(theSubjectId); - Date effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); - indexedObservation.setEffectiveDtm(effectiveDtm); + indexedObservation.setSubject(subjectId); // Build CodeableConcept entities for Observation.Category Set categoryConcepts = new HashSet<>(); @@ -52,12 +94,17 @@ public class ObservationLastNIndexPersistR4Svc { String observationCodeNormalizedId = null; // Determine if a Normalized ID was created previously for Observation Code + boolean observationCodeUpdate = false; for (Coding codeCoding : codeCodeableConcept.getCoding()) { if (codeCoding.hasCode() && codeCoding.hasSystem()) { observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); } else { observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); } + if(observationCodeNormalizedId != null) { + observationCodeUpdate = true; + break; + } } // Generate a new a normalized ID if necessary if (observationCodeNormalizedId == null) { @@ -69,13 +116,27 @@ public class ObservationLastNIndexPersistR4Svc { for (Coding codeCoding : codeCodeableConcept.getCoding()) { codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); } - myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); - codeableConceptField = myObservationIndexedCodeableConceptSearchParamDao.findByCodeableConceptId(observationCodeNormalizedId); + if (observationCodeUpdate) { + myEntityManager.merge(codeableConceptField); + } else { + myEntityManager.persist(codeableConceptField); + } indexedObservation.setObservationCode(codeableConceptField); indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); - myResourceIndexedObservationLastNDao.save(indexedObservation); + if (observationIndexUpdate) { + myEntityManager.merge(indexedObservation); + } else { + myEntityManager.persist(indexedObservation); + } } + public void deleteObservationIndex(IBasePersistedResource theEntity) { + ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findForIdentifier(theEntity.getIdDt().getIdPart()); + if(deletedObservationLastNEntity != null) { + myEntityManager.remove(deletedObservationLastNEntity); + } + } + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java index be0b00b58a3..176d35ce7d5 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java @@ -1,37 +1,79 @@ package ca.uhn.fhir.jpa.dao.lastn; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import org.hl7.fhir.r5.model.DateTimeType; import org.hl7.fhir.r5.model.CodeableConcept; import org.hl7.fhir.r5.model.Coding; import org.hl7.fhir.r5.model.Observation; +import org.hl7.fhir.r5.model.Reference; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.PersistenceContextType; +import java.util.*; +@Transactional(propagation = Propagation.REQUIRED) public class ObservationLastNIndexPersistR5Svc { + @PersistenceContext(type = PersistenceContextType.TRANSACTION) + protected EntityManager myEntityManager; + @Autowired IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; - @Autowired - IObservationIndexedCodeCodeableConceptSearchParamDao myObservationIndexedCodeableConceptSearchParamDao; - - @Autowired + @Autowired IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; - public void indexObservation(Observation theObservation, String theSubjectId) { - ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + public void indexObservation(Observation theObservation) { + // Only index for lastn if Observation has a subject and effective date/time + if(theObservation.getSubject() == null || !theObservation.hasEffective()) { + return; + } + + // Determine most recent effective date/time + Date effectiveDtm = null; + if (theObservation.hasEffectiveDateTimeType()) { + effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); + } else if (theObservation.hasEffectiveInstantType()) { + effectiveDtm = theObservation.getEffectiveInstantType().getValue(); + } else if (theObservation.hasEffectivePeriod()) { + effectiveDtm = theObservation.getEffectivePeriod().getEnd(); + } else if (theObservation.hasEffectiveTiming()) { + List events = theObservation.getEffectiveTiming().getEvent(); + for (DateTimeType event : events) { + Date eventDtm = event.getValue(); + if (effectiveDtm == null || eventDtm.after(effectiveDtm)) { + effectiveDtm = eventDtm; + } + } + } + if (effectiveDtm == null) { + return; + } + + // Determine if an index already exists for Observation: + boolean observationIndexUpdate = false; + ObservationIndexedSearchParamLastNEntity indexedObservation = null; + if (theObservation.hasId()) { + indexedObservation = myResourceIndexedObservationLastNDao.findForIdentifier(theObservation.getIdElement().getIdPart()); + } + if (indexedObservation == null) { + indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + } else { + observationIndexUpdate = true; + } + indexedObservation.setEffectiveDtm(effectiveDtm); + Reference subjectReference = theObservation.getSubject(); + String subjectId = subjectReference.getReference(); String resourcePID = theObservation.getIdElement().getIdPart(); indexedObservation.setIdentifier(resourcePID); - indexedObservation.setSubject(theSubjectId); - Date effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); - indexedObservation.setEffectiveDtm(effectiveDtm); + indexedObservation.setSubject(subjectId); // Build CodeableConcept entities for Observation.Category Set categoryConcepts = new HashSet<>(); @@ -52,12 +94,17 @@ public class ObservationLastNIndexPersistR5Svc { String observationCodeNormalizedId = null; // Determine if a Normalized ID was created previously for Observation Code + boolean observationCodeUpdate = false; for (Coding codeCoding : codeCodeableConcept.getCoding()) { if (codeCoding.hasCode() && codeCoding.hasSystem()) { observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); } else { observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); } + if(observationCodeNormalizedId != null) { + observationCodeUpdate = true; + break; + } } // Generate a new a normalized ID if necessary if (observationCodeNormalizedId == null) { @@ -69,13 +116,27 @@ public class ObservationLastNIndexPersistR5Svc { for (Coding codeCoding : codeCodeableConcept.getCoding()) { codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); } - myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); - codeableConceptField = myObservationIndexedCodeableConceptSearchParamDao.findByCodeableConceptId(observationCodeNormalizedId); + if (observationCodeUpdate) { + myEntityManager.merge(codeableConceptField); + } else { + myEntityManager.persist(codeableConceptField); + } indexedObservation.setObservationCode(codeableConceptField); indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); - myResourceIndexedObservationLastNDao.save(indexedObservation); + if (observationIndexUpdate) { + myEntityManager.merge(indexedObservation); + } else { + myEntityManager.persist(indexedObservation); + } } + public void deleteObservationIndex(IBasePersistedResource theEntity) { + ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findForIdentifier(theEntity.getIdDt().getIdPart()); + if(deletedObservationLastNEntity != null) { + myEntityManager.remove(deletedObservationLastNEntity); + } + } + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java new file mode 100644 index 00000000000..a906f852002 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java @@ -0,0 +1,198 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.context.*; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken; +import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor; +import ca.uhn.fhir.jpa.searchparam.extractor.PathAndRef; +import org.hl7.fhir.instance.model.api.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.PersistenceContextType; +import java.util.*; + +@Transactional(propagation = Propagation.REQUIRED) +public class ObservationLastNIndexPersistSvc { + + @PersistenceContext(type = PersistenceContextType.TRANSACTION) + protected EntityManager myEntityManager; + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; + + @Autowired + public ISearchParamExtractor mySearchParameterExtractor; + + public void indexObservation(IBaseResource theResource) { + + String subjectId = null; + List subjectReferenceElement = mySearchParameterExtractor.extractValues("Observation.subject", theResource); + if (subjectReferenceElement.size() == 1) { + PathAndRef subjectPathAndRef = mySearchParameterExtractor.extractReferenceLinkFromResource(subjectReferenceElement.get(0), "Observation.subject"); + if (subjectPathAndRef != null) { + IBaseReference subjectReference = subjectPathAndRef.getRef(); + if (subjectReference != null) { + subjectId = subjectReference.getReferenceElement().getValue(); + } + } + } + + Date effectiveDtm = null; + List effectiveDateElement = mySearchParameterExtractor.extractValues("Observation.effective", theResource); + if (effectiveDateElement.size() == 1) { + effectiveDtm = mySearchParameterExtractor.extractDateFromResource(effectiveDateElement.get(0), "Observation.effective"); + } + + // Only index for lastn if Observation has a subject and effective date/time + if (subjectId == null || effectiveDtm == null) { + return; + } + + String resourcePID = theResource.getIdElement().getIdPart(); + + // Determine if an index already exists for Observation: + boolean observationIndexUpdate = false; + ObservationIndexedSearchParamLastNEntity indexedObservation = null; + if (resourcePID != null) { + indexedObservation = myResourceIndexedObservationLastNDao.findForIdentifier(resourcePID); + } + if (indexedObservation == null) { + indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + } else { + observationIndexUpdate = true; + } + + indexedObservation.setEffectiveDtm(effectiveDtm); + indexedObservation.setIdentifier(resourcePID); + indexedObservation.setSubject(subjectId); + + // Build CodeableConcept entities for Observation.Category + List observationCategoryCodeableConcepts = mySearchParameterExtractor.extractValues("Observation.category", theResource); + Set categoryCodeableConceptEntities = new HashSet<>(); + for (IBase categoryCodeableConcept : observationCategoryCodeableConcepts) { + // Build CodeableConcept entities for each category CodeableConcept + categoryCodeableConceptEntities.add(getCategoryCodeableConceptEntities(categoryCodeableConcept)); + } + indexedObservation.setCategoryCodeableConcepts(categoryCodeableConceptEntities); + + // Build CodeableConcept entity for Observation.Code. + List observationCodeCodeableConcepts = mySearchParameterExtractor.extractValues("Observation.code", theResource); + + // Determine if a Normalized ID was created previously for Observation Code + boolean observationCodeUpdate = false; + String observationCodeNormalizedId = getCodeCodeableConceptIdIfExists(observationCodeCodeableConcepts.get(0)); + if (observationCodeNormalizedId != null) { + observationCodeUpdate = true; + } + // Generate a new a normalized ID if necessary + if (observationCodeNormalizedId == null) { + observationCodeNormalizedId = UUID.randomUUID().toString(); + } + + // Create/update normalized Observation Code index record + ObservationIndexedCodeCodeableConceptEntity codeableConceptField = getCodeCodeableConcept(observationCodeCodeableConcepts.get(0), observationCodeNormalizedId); + + if (observationCodeUpdate) { + myEntityManager.merge(codeableConceptField); + } else { + myEntityManager.persist(codeableConceptField); + } + + indexedObservation.setObservationCode(codeableConceptField); + indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); + if (observationIndexUpdate) { + myEntityManager.merge(indexedObservation); + } else { + myEntityManager.persist(indexedObservation); + } + + } + + private ObservationIndexedCategoryCodeableConceptEntity getCategoryCodeableConceptEntities(IBase theValue) { + String text = mySearchParameterExtractor.getDisplayTextFromCodeableConcept(theValue); + ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConcept = new ObservationIndexedCategoryCodeableConceptEntity(text); + + List codings = mySearchParameterExtractor.getCodingsFromCodeableConcept(theValue); + Set categoryCodingEntities = new HashSet<>(); + for (IBase nextCoding : codings) { + categoryCodingEntities.add(getCategoryCoding(nextCoding)); + } + + categoryCodeableConcept.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); + + return categoryCodeableConcept; + } + + private ObservationIndexedCodeCodeableConceptEntity getCodeCodeableConcept(IBase theValue, String observationCodeNormalizedId) { + String text = mySearchParameterExtractor.getDisplayTextFromCodeableConcept(theValue); + ObservationIndexedCodeCodeableConceptEntity codeCodeableConcept = new ObservationIndexedCodeCodeableConceptEntity(text, observationCodeNormalizedId); + + List codings = mySearchParameterExtractor.getCodingsFromCodeableConcept(theValue); + for (IBase nextCoding : codings) { + codeCodeableConcept.addCoding(getCodeCoding(nextCoding, observationCodeNormalizedId)); + } + + return codeCodeableConcept; + } + + private String getCodeCodeableConceptIdIfExists(IBase theValue) { + List codings = mySearchParameterExtractor.getCodingsFromCodeableConcept(theValue); + String codeCodeableConceptId = null; + for (IBase nextCoding : codings) { + ResourceIndexedSearchParamToken param = mySearchParameterExtractor.createSearchParamForCoding("Observation", + new RuntimeSearchParam(null, null, "code", null, null, null, null, null, null, null), + nextCoding); + String system = param.getSystem(); + String code = param.getValue(); + String text = mySearchParameterExtractor.getDisplayTextForCoding(nextCoding); + if (code != null && system != null) { + codeCodeableConceptId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(code, system); + } else { + codeCodeableConceptId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(text); + } + if (codeCodeableConceptId != null) { + break; + } + } + + return codeCodeableConceptId; + } + + private ObservationIndexedCategoryCodingEntity getCategoryCoding(IBase theValue) { + ResourceIndexedSearchParamToken param = mySearchParameterExtractor.createSearchParamForCoding("Observation", + new RuntimeSearchParam(null, null, "category", null, null, null, null, null, null, null), + theValue); + String system = param.getSystem(); + String code = param.getValue(); + String text = mySearchParameterExtractor.getDisplayTextForCoding(theValue); + return new ObservationIndexedCategoryCodingEntity(system, code, text); + } + + private ObservationIndexedCodeCodingEntity getCodeCoding(IBase theValue, String observationCodeNormalizedId) { + ResourceIndexedSearchParamToken param = mySearchParameterExtractor.createSearchParamForCoding("Observation", + new RuntimeSearchParam(null, null, "code", null, null, null, null, null, null, null), + theValue); + String system = param.getSystem(); + String code = param.getValue(); + String text = mySearchParameterExtractor.getDisplayTextForCoding(theValue); + return new ObservationIndexedCodeCodingEntity(system, code, text, observationCodeNormalizedId); + } + + public void deleteObservationIndex(IBasePersistedResource theEntity) { + ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findForIdentifier(theEntity.getIdDt().getIdPart()); + if (deletedObservationLastNEntity != null) { + myEntityManager.remove(deletedObservationLastNEntity); + } + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java index 7d5d98525af..281b5e59b91 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java @@ -1,6 +1,5 @@ package ca.uhn.fhir.jpa.dao.lastn.entity; -import org.hibernate.search.annotations.DocumentId; import org.hibernate.search.annotations.Field; import org.hibernate.search.annotations.IndexedEmbedded; @@ -10,15 +9,7 @@ import java.util.Set; @Embeddable public class ObservationIndexedCategoryCodeableConceptEntity { - @Id - @DocumentId(name = "category_concept_id") - @SequenceGenerator(name = "SEQ_CATEGORY_CONCEPT", sequenceName = "SEQ_CATEGORY_CONCEPT") - @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CATEGORY_CONCEPT") - @Column(name = "CATEGORY_CONCEPT_ID") - private Long myId; - @Field(name = "text") - @Column(name = "CODEABLE_CONCEPT_TEXT", nullable = true) private String myCodeableConceptText; @IndexedEmbedded(depth=2, prefix = "coding") diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java index c56753063ee..f3e26cdb847 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java @@ -3,10 +3,13 @@ package ca.uhn.fhir.jpa.dao.lastn.entity; import org.hibernate.search.annotations.*; import javax.persistence.*; +import javax.persistence.Index; import java.util.*; @Entity -@Table(name = "HFJ_LASTN_OBSERVATION") +@Table(name = "HFJ_LASTN_OBSERVATION", indexes = { + @Index(name = "IDX_LASTN_OBSERVATION_RESID", columnList = "RESOURCE_IDENTIFIER", unique = true) +}) @Indexed(index = "observation_index") public class ObservationIndexedSearchParamLastNEntity { @@ -16,7 +19,7 @@ public class ObservationIndexedSearchParamLastNEntity { @Column(name = "LASTN_ID") private Long myId; - @Field(name = "subject", analyze = Analyze.NO) + @Field(name = "subject", analyze = Analyze.NO) @Column(name = "LASTN_SUBJECT_ID", nullable = true) private String mySubject; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index 7ef7972b0b1..14828dd2727 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -20,29 +20,37 @@ package ca.uhn.fhir.jpa.dao.r4; * #L% */ -import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; -import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR4Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.Observation; -import org.hl7.fhir.r4.model.Reference; import org.springframework.beans.factory.annotation.Autowired; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.PersistenceContextType; import javax.servlet.http.HttpServletResponse; import java.util.Date; public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDaoObservation { + + @PersistenceContext(type = PersistenceContextType.TRANSACTION) + protected EntityManager myEntityManager; + @Autowired - ObservationLastNIndexPersistR4Svc myObservationLastNIndexPersistR4Svc; + ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; @Override public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse) { @@ -52,19 +60,19 @@ public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDaoObserva return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); } - - @Override public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, boolean theUpdateVersion, Date theUpdateTime, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theUpdateTime, theForceUpdate, theCreateNewHistoryEntry); - if(thePerformIndexing) { - // Update indexes here for LastN operation. - Observation observation = (Observation)theResource; - Reference subjectReference = observation.getSubject(); - String subjectID = subjectReference.getReference(); - myObservationLastNIndexPersistR4Svc.indexObservation(observation, subjectID); + if (!retVal.isUnchangedInCurrentOperation()) { + if (retVal.getDeleted() == null) { + // Update indexes here for LastN operation. + Observation observation = (Observation) theResource; + myObservationLastNIndexPersistSvc.indexObservation(observation); + } else { + myObservationLastNIndexPersistSvc.deleteObservationIndex(retVal); + } } return retVal; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java index c1168660e63..9d85c24f881 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java @@ -20,66 +20,32 @@ package ca.uhn.fhir.jpa.dao.r5; * #L% */ -import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; -import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR5Svc; +import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; -import ca.uhn.fhir.model.api.IResource; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; -import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.param.DateRangeParam; -import ca.uhn.fhir.rest.param.StringAndListParam; -import ca.uhn.fhir.rest.param.StringParam; import org.hl7.fhir.instance.model.api.IBaseResource; -import org.hl7.fhir.instance.model.api.IIdType; -import org.hl7.fhir.instance.model.api.IPrimitiveType; -import org.hl7.fhir.r5.model.Reference; import org.hl7.fhir.r5.model.Observation; import org.springframework.beans.factory.annotation.Autowired; import javax.servlet.http.HttpServletResponse; -import java.util.Collections; import java.util.Date; -public class FhirResourceDaoObservationR5 extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { +public class FhirResourceDaoObservationR5 extends BaseHapiFhirResourceDaoObservation { @Autowired - ObservationLastNIndexPersistR5Svc myObservationLastNIndexPersistR5Svc; - - private IBundleProvider doLastNOperation(IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequest) { - SearchParameterMap paramMap = new SearchParameterMap(); - if (theCount != null) { - paramMap.setCount(theCount.getValue()); - } - if (theContent != null) { - paramMap.add(Constants.PARAM_CONTENT, theContent); - } - if (theNarrative != null) { - paramMap.add(Constants.PARAM_TEXT, theNarrative); - } - paramMap.setIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive())); - paramMap.setEverythingMode(theId != null ? EverythingModeEnum.PATIENT_INSTANCE : EverythingModeEnum.PATIENT_TYPE); - paramMap.setSort(theSort); - paramMap.setLastUpdated(theLastUpdated); - if (theId != null) { - paramMap.add("_id", new StringParam(theId.getIdPart())); - } - - if (!isPagingProviderDatabaseBacked(theRequest)) { - paramMap.setLoadSynchronous(true); - } - - return mySearchCoordinatorSvc.registerSearch(this, paramMap, getResourceName(), new CacheControlDirective().parse(theRequest.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequest); - } + ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; @Override - public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse) { + public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse) { + + updateSearchParamsForLastn(theSearchParameterMap, theRequestDetails); + return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); } @@ -88,15 +54,16 @@ public class FhirResourceDaoObservationR5 extends BaseHapiFhirResourceDao executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { - String[] topHitsInclude = {"identifier"}; + String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(theMaxObservationsPerCode, topHitsInclude)); try { SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); @@ -518,4 +519,11 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); } + public void deleteObservationIndex(String theObservationIdentifier) throws IOException { + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(IndexConstants.OBSERVATION_DOCUMENT_TYPE); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_IDENTIFIER_FIELD_NAME, theObservationIdentifier)); + deleteByQueryRequest.setQuery(boolQueryBuilder); + myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); + } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java deleted file mode 100644 index 593fa2bd9fb..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java +++ /dev/null @@ -1,274 +0,0 @@ -package ca.uhn.fhir.jpa.dao.lastn; - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao; -import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.parser.IParser; -import ca.uhn.fhir.parser.LenientErrorHandler; -import ca.uhn.fhir.rest.param.*; -import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; -import com.google.common.base.Charsets; -import org.apache.commons.io.IOUtils; -import org.hl7.fhir.r4.model.*; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.Resource; -import org.springframework.core.io.support.PathMatchingResourcePatternResolver; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import java.io.*; -import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; - -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class }) -public class IntegratedObservationIndexedSearchParamLastNTest { - - @Autowired - IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; - - @Autowired - IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; - - @Autowired - ObservationLastNIndexPersistR4Svc myObservationLastNIndexPersistR4Svc; - - @Autowired - private ElasticsearchSvcImpl elasticsearchSvc; - - @Autowired - private IFhirSystemDao myDao; - - final String RESOURCEPID = "123"; - final String SUBJECTID = "4567"; - final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; - final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; - - final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; - final String CODEFIRSTCODINGCODE = "test-code"; - - @Before - public void before() throws IOException { - - myResourceIndexedObservationLastNDao.deleteAll(); - myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); - - } - - @After - public void after() { - - myResourceIndexedObservationLastNDao.deleteAll(); - myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); - - } - - @Test - public void testIndexObservationSingle() throws IOException { - - Observation myObservation = new Observation(); - String resourcePID = "123"; - myObservation.setId(resourcePID); - Reference subjectId = new Reference("4567"); - myObservation.setSubject(subjectId); - Date effectiveDtm = new Date(); - myObservation.setEffective(new DateTimeType(effectiveDtm)); - - // Add three CodeableConcepts for category - List categoryConcepts = new ArrayList<>(); - // Create three codings and first category CodeableConcept - List category1 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); - category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); - category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); - category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); - categoryCodeableConcept1.setCoding(category1); - categoryConcepts.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - List category2 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for for second category"); - category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); - category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); - category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); - categoryCodeableConcept2.setCoding(category2); - categoryConcepts.add(categoryCodeableConcept2); - // Create three codings and third category CodeableConcept - List category3 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText("Test Codeable Concept Field for third category"); - category3.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test-vitals-panel display")); - category3.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test-alt-vitals-panel display")); - category3.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test-2nd-alt-vitals-panel display")); - categoryCodeableConcept3.setCoding(category3); - categoryConcepts.add(categoryCodeableConcept3); - myObservation.setCategory(categoryConcepts); - - // Create CodeableConcept for Code with three codings. - // TODO: Temporarily limit this to two codings until we sort out how to manage multiple codings - String observationCodeText = "Test Codeable Concept Field for Code"; - CodeableConcept codeableConceptField = new CodeableConcept().setText(observationCodeText); - codeableConceptField.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test-code display")); -// codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); -// codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); - myObservation.setCode(codeableConceptField); - - myObservationLastNIndexPersistR4Svc.indexObservation(myObservation, "4567"); - - SearchParameterMap searchParameterMap = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); - searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam))); - TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); - searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); - TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); - searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); - - assertEquals(1, observationIdsOnly.size()); - assertEquals(RESOURCEPID, observationIdsOnly.get(0)); - - } - - - @Test - public void testIndexObservationMultiple() { - - // Create two CodeableConcept values each for a Code with three codings. - CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); - codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); - codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); - codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); - - CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); - codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); - codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); - codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); - - // Create two CodeableConcept entities for category, each with three codings. - List category1 = new ArrayList<>(); - // Create three codings and first category CodeableConcept - category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); - category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); - category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); - List categoryConcepts1 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); - categoryCodeableConcept1.setCoding(category1); - categoryConcepts1.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - List category2 = new ArrayList<>(); - category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); - category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); - category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); - List categoryConcepts2 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); - categoryCodeableConcept2.setCoding(category2); - categoryConcepts2.add(categoryCodeableConcept2); - - for (int patientCount = 0; patientCount < 10 ; patientCount++) { - - String subjectId = String.valueOf(patientCount); - - for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { - - Observation observation = new Observation(); - observation.setId(String.valueOf(entryCount + patientCount*10)); - Reference subject = new Reference(subjectId); - observation.setSubject(subject); - - if (entryCount%2 == 1) { - observation.setCategory(categoryConcepts1); - observation.setCode(codeableConceptField1); - } else { - observation.setCategory(categoryConcepts2); - observation.setCode(codeableConceptField2); - } - - Calendar observationDate = new GregorianCalendar(); - observationDate.add(Calendar.HOUR, -10 + entryCount); - Date effectiveDtm = observationDate.getTime(); - observation.setEffective(new DateTimeType(effectiveDtm)); - - myObservationLastNIndexPersistR4Svc.indexObservation(observation, String.valueOf(patientCount)); - - } - - } - - assertEquals(100, myResourceIndexedObservationLastNDao.count()); - assertEquals(2, myCodeableConceptIndexedSearchParamNormalizedDao.count()); - - } - - @Test - public void testSampleBundleInTransaction() throws IOException { - FhirContext myFhirCtx = FhirContext.forR4(); - - PathMatchingResourcePatternResolver provider = new PathMatchingResourcePatternResolver(); - final Resource[] bundleResources; - try { - bundleResources = provider.getResources("lastntestbundle.json"); - } catch (IOException e) { - throw new RuntimeException("Unexpected error during transmission: " + e.toString(), e); - } - - AtomicInteger index = new AtomicInteger(); - - Arrays.stream(bundleResources).forEach( - resource -> { - index.incrementAndGet(); - - InputStream resIs = null; - String nextBundleString; - try { - resIs = resource.getInputStream(); - nextBundleString = IOUtils.toString(resIs, Charsets.UTF_8); - } catch (IOException e) { - return; - } finally { - try { - if (resIs != null) { - resIs.close(); - } - } catch (final IOException ioe) { - // ignore - } - } - - /* - * SMART demo apps rely on the use of LOINC 3141-9 (Body Weight Measured) - * instead of LOINC 29463-7 (Body Weight) - */ - nextBundleString = nextBundleString.replace("\"29463-7\"", "\"3141-9\""); - - IParser parser = myFhirCtx.newJsonParser(); - parser.setParserErrorHandler(new LenientErrorHandler(false)); - Bundle bundle = parser.parseResource(Bundle.class, nextBundleString); - - myDao.transaction(null, bundle); - - - } - ); - - SearchParameterMap searchParameterMap = new SearchParameterMap(); - - // execute Observation ID search - Composite Aggregation - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1); - - assertEquals(20, observationIdsOnly.size()); - - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); - - assertEquals(38, observationIdsOnly.size()); - - } - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java deleted file mode 100644 index 2d5e6515291..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java +++ /dev/null @@ -1,190 +0,0 @@ -package ca.uhn.fhir.jpa.dao.lastn; - -import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticSearch; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import org.hl7.fhir.r4.model.*; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import java.util.*; - -import static org.junit.Assert.assertEquals; - -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestR4ConfigWithElasticSearch.class }) -public class PersistObservationIndexedSearchParamLastNTest { - - @Autowired - IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; - - @Autowired - IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; - - @Autowired - ObservationLastNIndexPersistR4Svc myObservationLastNIndexPersistR4Svc; - - @Before - public void before() { - - myResourceIndexedObservationLastNDao.deleteAll(); - myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); - - } - - @After - public void after() { - - myResourceIndexedObservationLastNDao.deleteAll(); - myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); - - } - - @Test - public void testIndexObservationSingle() { - - Observation myObservation = new Observation(); - String resourcePID = "123"; - myObservation.setId(resourcePID); - Reference subjectId = new Reference("4567"); - myObservation.setSubject(subjectId); - Date effectiveDtm = new Date(); - myObservation.setEffective(new DateTimeType(effectiveDtm)); - - // Add three CodeableConcepts for category - List categoryConcepts = new ArrayList<>(); - // Create three codings and first category CodeableConcept - List category1 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); - category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); - category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); - category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); - categoryCodeableConcept1.setCoding(category1); - categoryConcepts.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - List category2 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for for second category"); - category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); - category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); - category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); - categoryCodeableConcept2.setCoding(category2); - categoryConcepts.add(categoryCodeableConcept2); - // Create three codings and third category CodeableConcept - List category3 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText("Test Codeable Concept Field for third category"); - category3.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test-vitals-panel display")); - category3.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test-alt-vitals-panel display")); - category3.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test-2nd-alt-vitals-panel display")); - categoryCodeableConcept3.setCoding(category3); - categoryConcepts.add(categoryCodeableConcept3); - myObservation.setCategory(categoryConcepts); - - // Create CodeableConcept for Code with three codings. - String observationCodeText = "Test Codeable Concept Field for Code"; - CodeableConcept codeableConceptField = new CodeableConcept().setText(observationCodeText); - codeableConceptField.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test-code display")); - codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); - codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); - myObservation.setCode(codeableConceptField); - - myObservationLastNIndexPersistR4Svc.indexObservation(myObservation, "4567"); - - List persistedObservationEntities = myResourceIndexedObservationLastNDao.findAll(); - assertEquals(1, persistedObservationEntities.size()); - ObservationIndexedSearchParamLastNEntity persistedObservationEntity = persistedObservationEntities.get(0); - assertEquals(subjectId.getReference(), persistedObservationEntity.getSubject()); - assertEquals(resourcePID, persistedObservationEntity.getIdentifier()); - assertEquals(effectiveDtm, persistedObservationEntity.getEffectiveDtm()); - - String observationCodeNormalizedId = persistedObservationEntity.getCodeNormalizedId(); - - List persistedObservationCodes = myCodeableConceptIndexedSearchParamNormalizedDao.findAll(); - assertEquals(1, persistedObservationCodes.size()); - ObservationIndexedCodeCodeableConceptEntity persistedObservationCode = persistedObservationCodes.get(0); - assertEquals(observationCodeNormalizedId, persistedObservationCode.getCodeableConceptId()); - assertEquals(observationCodeText, persistedObservationCode.getCodeableConceptText()); - - } - - @Test - public void testIndexObservationMultiple() { - - // Create two CodeableConcept values each for a Code with three codings. - CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); - codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); - codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); - codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); - - CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); - codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); - codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); - codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); - - // Create two CodeableConcept entities for category, each with three codings. - List category1 = new ArrayList<>(); - // Create three codings and first category CodeableConcept - category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); - category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); - category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); - List categoryConcepts1 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); - categoryCodeableConcept1.setCoding(category1); - categoryConcepts1.add(categoryCodeableConcept1); - // Create three codings and second category CodeableConcept - List category2 = new ArrayList<>(); - category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); - category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); - category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); - List categoryConcepts2 = new ArrayList<>(); - CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); - categoryCodeableConcept2.setCoding(category2); - categoryConcepts2.add(categoryCodeableConcept2); - - for (int patientCount = 0; patientCount < 10 ; patientCount++) { - - String subjectId = String.valueOf(patientCount); - - for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { - - Observation observation = new Observation(); - observation.setId(String.valueOf(entryCount + patientCount*10)); - Reference subject = new Reference(subjectId); - observation.setSubject(subject); - - if (entryCount%2 == 1) { - observation.setCategory(categoryConcepts1); - observation.setCode(codeableConceptField1); - } else { - observation.setCategory(categoryConcepts2); - observation.setCode(codeableConceptField2); - } - - Calendar observationDate = new GregorianCalendar(); - observationDate.add(Calendar.HOUR, -10 + entryCount); - Date effectiveDtm = observationDate.getTime(); - observation.setEffective(new DateTimeType(effectiveDtm)); - - myObservationLastNIndexPersistR4Svc.indexObservation(observation, subjectId); - - } - - } - - assertEquals(100, myResourceIndexedObservationLastNDao.count()); - assertEquals(2, myCodeableConceptIndexedSearchParamNormalizedDao.count()); - - } - - @Test - public void testSampleObservationResource() { - - } - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4Test.java new file mode 100644 index 00000000000..79f08459135 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4Test.java @@ -0,0 +1,406 @@ +package ca.uhn.fhir.jpa.dao.r4; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao; +import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.parser.IParser; +import ca.uhn.fhir.rest.param.*; +import com.google.common.base.Charsets; +import org.apache.commons.io.IOUtils; +import org.hl7.fhir.r4.model.*; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.PersistenceContextType; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = {TestR4ConfigWithElasticsearchClient.class}) +public class PersistObservationIndexedSearchParamLastNR4Test { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; + +// @Autowired +// ObservationLastNIndexPersistR4Svc myObservationLastNIndexPersistR4Svc; + + @Autowired + private ElasticsearchSvcImpl elasticsearchSvc; + + @Autowired + private IFhirSystemDao myDao; + + @PersistenceContext(type = PersistenceContextType.TRANSACTION) + protected EntityManager myEntityManager; + + @Autowired + ObservationLastNIndexPersistSvc testObservationPersist; + + @Before + public void before() { + + myResourceIndexedObservationLastNDao.deleteAll(); + myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); +// testObservationPersist = new BaseObservationLastNIndexPersistSvc(myEntityManager, myResourceIndexedObservationLastNDao, +// myObservationIndexedCodeCodingSearchParamDao, mySearchParamExtractor, myContext); + + + } + + private final String SINGLE_SUBJECT_ID = "4567"; + private final String SINGLE_OBSERVATION_PID = "123"; + private final Date SINGLE_EFFECTIVEDTM = new Date(); + private final String SINGLE_OBSERVATION_CODE_TEXT = "Test Codeable Concept Field for Code"; + + private final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; + private final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; + + private final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; + private final String CODEFIRSTCODINGCODE = "test-code"; + + @Test + public void testIndexObservationSingle() { + indexSingleObservation(); + List persistedObservationEntities = myResourceIndexedObservationLastNDao.findAll(); + assertEquals(1, persistedObservationEntities.size()); + ObservationIndexedSearchParamLastNEntity persistedObservationEntity = persistedObservationEntities.get(0); + assertEquals(SINGLE_SUBJECT_ID, persistedObservationEntity.getSubject()); + assertEquals(SINGLE_OBSERVATION_PID, persistedObservationEntity.getIdentifier()); + assertEquals(SINGLE_EFFECTIVEDTM, persistedObservationEntity.getEffectiveDtm()); + + String observationCodeNormalizedId = persistedObservationEntity.getCodeNormalizedId(); + + List persistedObservationCodes = myCodeableConceptIndexedSearchParamNormalizedDao.findAll(); + assertEquals(1, persistedObservationCodes.size()); + ObservationIndexedCodeCodeableConceptEntity persistedObservationCode = persistedObservationCodes.get(0); + assertEquals(observationCodeNormalizedId, persistedObservationCode.getCodeableConceptId()); + assertEquals(SINGLE_OBSERVATION_CODE_TEXT, persistedObservationCode.getCodeableConceptText()); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", SINGLE_SUBJECT_ID); + searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam))); + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); + TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); + searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); + + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); + + assertEquals(1, observationIdsOnly.size()); + assertEquals(SINGLE_OBSERVATION_PID, observationIdsOnly.get(0)); + } + + private void indexSingleObservation() { + + Observation myObservation = new Observation(); + IdType observationID = new IdType("Observation", SINGLE_OBSERVATION_PID, "1"); + myObservation.setId(observationID); + Reference subjectId = new Reference(SINGLE_SUBJECT_ID); + myObservation.setSubject(subjectId); + myObservation.setEffective(new DateTimeType(SINGLE_EFFECTIVEDTM)); + + myObservation.setCategory(getCategoryCode()); + + myObservation.setCode(getObservationCode()); + + //myObservationLastNIndexPersistR4Svc.indexObservation(myObservation); + testObservationPersist.indexObservation(myObservation); + + } + + private List getCategoryCode() { + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for for second category"); + category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText("Test Codeable Concept Field for third category"); + category3.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, "test-vitals-panel", "test-vitals-panel display")); + category3.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test-alt-vitals-panel display")); + category3.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test-2nd-alt-vitals-panel display")); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + return categoryConcepts; + } + + private CodeableConcept getObservationCode() { + // Create CodeableConcept for Code with three codings. + CodeableConcept codeableConceptField = new CodeableConcept().setText(SINGLE_OBSERVATION_CODE_TEXT); + codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, "test-code display")); + // TODO: Temporarily limit this to a single Observation Code coding until we sort out how to manage multiple codings +// codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); +// codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); + return codeableConceptField; + } + + @Test + public void testIndexObservationMultiple() { + indexMultipleObservations(); + assertEquals(100, myResourceIndexedObservationLastNDao.count()); + assertEquals(2, myCodeableConceptIndexedSearchParamNormalizedDao.count()); + + // Check that all observations were indexed. + SearchParameterMap searchParameterMap = new SearchParameterMap(); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + assertEquals(100, observationIdsOnly.size()); + + // Filter the results by category code. + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); + + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + + assertEquals(50, observationIdsOnly.size()); + + } + + private void indexMultipleObservations() { + + // Create two CodeableConcept values each for a Code with three codings. + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, "test-code-1", "test-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, "test-code-2", "test-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + + // Create two CodeableConcept entities for category, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); + + for (int patientCount = 0; patientCount < 10; patientCount++) { + + String subjectId = String.valueOf(patientCount); + + for (int entryCount = 0; entryCount < 10; entryCount++) { + + Observation observation = new Observation(); + IdType observationId = new IdType("Observation", String.valueOf(entryCount + patientCount * 10), "1"); + observation.setId(observationId); + Reference subject = new Reference(subjectId); + observation.setSubject(subject); + + if (entryCount % 2 == 1) { + observation.setCategory(categoryConcepts1); + observation.setCode(codeableConceptField1); + } else { + observation.setCategory(categoryConcepts2); + observation.setCode(codeableConceptField2); + } + + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observation.setEffective(new DateTimeType(effectiveDtm)); + +// myObservationLastNIndexPersistR4Svc.indexObservation(observation); + + testObservationPersist.indexObservation(observation); + } + + } + } + + @Test + public void testDeleteObservation() { + indexMultipleObservations(); + assertEquals(100, myResourceIndexedObservationLastNDao.count()); + // Check that fifth observation for fifth patient has been indexed. + ObservationIndexedSearchParamLastNEntity observation = myResourceIndexedObservationLastNDao.findForIdentifier("55"); + assertNotNull(observation); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + assertEquals(100, observationIdsOnly.size()); + assertTrue(observationIdsOnly.contains("55")); + + // Delete fifth observation for fifth patient. + ResourceTable entity = new ResourceTable(); + entity.setId(55L); + entity.setResourceType("Observation"); + entity.setVersion(0L); + +// myObservationLastNIndexPersistR4Svc.deleteObservationIndex(entity); + testObservationPersist.deleteObservationIndex(entity); + + // Confirm that observation was deleted. + assertEquals(99, myResourceIndexedObservationLastNDao.count()); + observation = myResourceIndexedObservationLastNDao.findForIdentifier("55"); + assertNull(observation); + + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + assertEquals(99, observationIdsOnly.size()); + assertTrue(!observationIdsOnly.contains("55")); + + } + + @Test + public void testUpdateObservation() { + indexSingleObservation(); + ObservationIndexedSearchParamLastNEntity observationIndexEntity = myResourceIndexedObservationLastNDao.findAll().get(0); + assertEquals(SINGLE_OBSERVATION_PID, observationIndexEntity.getIdentifier()); + assertEquals(SINGLE_SUBJECT_ID, observationIndexEntity.getSubject()); + assertEquals(SINGLE_EFFECTIVEDTM, observationIndexEntity.getEffectiveDtm()); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", SINGLE_SUBJECT_ID); + searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam))); + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); + TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); + searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + assertEquals(1, observationIdsOnly.size()); + assertTrue(observationIdsOnly.contains(SINGLE_OBSERVATION_PID)); + + // Update the Observation with a new Subject and effective date: + Observation updatedObservation = new Observation(); + IdType observationId = new IdType("Observation", observationIndexEntity.getIdentifier(), "2"); + updatedObservation.setId(observationId); + Reference subjectId = new Reference("1234"); + updatedObservation.setSubject(subjectId); + DateTimeType newEffectiveDtm = new DateTimeType(new Date()); + updatedObservation.setEffective(newEffectiveDtm); + updatedObservation.setCategory(getCategoryCode()); + updatedObservation.setCode(getObservationCode()); + +// myObservationLastNIndexPersistR4Svc.indexObservation(updatedObservation); + testObservationPersist.indexObservation(updatedObservation); + + ObservationIndexedSearchParamLastNEntity updatedObservationEntity = myResourceIndexedObservationLastNDao.findForIdentifier(SINGLE_OBSERVATION_PID); + assertEquals("1234", updatedObservationEntity.getSubject()); + assertEquals(newEffectiveDtm.getValue(), updatedObservationEntity.getEffectiveDtm()); + + // Repeat earlier Elasticsearch query. This time, should return no matches. + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + assertEquals(0, observationIdsOnly.size()); + + // Try again with the new patient ID. + searchParameterMap = new SearchParameterMap(); + subjectParam = new ReferenceParam("Patient", "", "1234"); + searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam))); + searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); + searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + + // Should see the observation returned now. + assertEquals(1, observationIdsOnly.size()); + assertTrue(observationIdsOnly.contains(SINGLE_OBSERVATION_PID)); + + } + + @Test + public void testSampleBundleInTransaction() throws IOException { + FhirContext myFhirCtx = FhirContext.forR4(); + + PathMatchingResourcePatternResolver provider = new PathMatchingResourcePatternResolver(); + final Resource[] bundleResources; + bundleResources = provider.getResources("lastntestbundle.json"); + + AtomicInteger index = new AtomicInteger(); + + Arrays.stream(bundleResources).forEach( + resource -> { + index.incrementAndGet(); + + InputStream resIs = null; + String nextBundleString; + try { + resIs = resource.getInputStream(); + nextBundleString = IOUtils.toString(resIs, Charsets.UTF_8); + } catch (IOException e) { + return; + } finally { + try { + if (resIs != null) { + resIs.close(); + } + } catch (final IOException ioe) { + // ignore + } + } + + IParser parser = myFhirCtx.newJsonParser(); + Bundle bundle = parser.parseResource(Bundle.class, nextBundleString); + + myDao.transaction(null, bundle); + + } + ); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + + // execute Observation ID search - Composite Aggregation + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1); + + assertEquals(20, observationIdsOnly.size()); + + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); + + assertEquals(38, observationIdsOnly.size()); + + } + + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java index 0ffb6a61118..1e5116e3f7d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java @@ -4,8 +4,6 @@ import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.transaction.annotation.EnableTransactionManagement; import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; import pl.allegro.tech.embeddedelasticsearch.PopularProperties; @@ -15,8 +13,6 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; @Configuration -@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") -@EnableTransactionManagement public class TestElasticsearchConfig { private final String elasticsearchHost = "127.0.0.1"; diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java index 593394d58fc..8a73cdf1349 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java @@ -115,17 +115,31 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor @Override public SearchParamSet extractResourceLinks(IBaseResource theResource) { - IExtractor extractor = (params, searchParam, value, path) -> { - if (value instanceof IBaseResource) { + return extractSearchParams(theResource, new ResourceLinkExtractor(), RestSearchParameterTypeEnum.REFERENCE); + } + + @Override + public PathAndRef extractReferenceLinkFromResource(IBase theValue, String thePath) { + ResourceLinkExtractor extractor = new ResourceLinkExtractor(); + return extractor.get(theValue, thePath); + } + + private class ResourceLinkExtractor implements IExtractor { + + private PathAndRef myPathAndRef = null; + + @Override + public void extract(SearchParamSet theParams, RuntimeSearchParam theSearchParam, IBase theValue, String thePath) { + if (theValue instanceof IBaseResource) { return; } - String nextType = toRootTypeName(value); + String nextType = toRootTypeName(theValue); switch (nextType) { case "uri": case "canonical": - String typeName = toTypeName(value); - IPrimitiveType valuePrimitive = (IPrimitiveType) value; + String typeName = toTypeName(theValue); + IPrimitiveType valuePrimitive = (IPrimitiveType) theValue; IBaseReference fakeReference = (IBaseReference) myContext.getElementDefinition("Reference").newInstance(); fakeReference.setReference(valuePrimitive.getValueAsString()); @@ -141,23 +155,23 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor */ IIdType parsed = fakeReference.getReferenceElement(); if (parsed.hasIdPart() && parsed.hasResourceType() && !parsed.isAbsolute()) { - PathAndRef ref = new PathAndRef(searchParam.getName(), path, fakeReference, false); - params.add(ref); + myPathAndRef = new PathAndRef(theSearchParam.getName(), thePath, fakeReference, false); + theParams.add(myPathAndRef); break; } if (parsed.isAbsolute()) { - PathAndRef ref = new PathAndRef(searchParam.getName(), path, fakeReference, true); - params.add(ref); + myPathAndRef = new PathAndRef(theSearchParam.getName(), thePath, fakeReference, true); + theParams.add(myPathAndRef); break; } } - params.addWarning("Ignoring canonical reference (indexing canonical is not yet supported)"); + theParams.addWarning("Ignoring canonical reference (indexing canonical is not yet supported)"); break; case "reference": case "Reference": - IBaseReference valueRef = (IBaseReference) value; + IBaseReference valueRef = (IBaseReference) theValue; IIdType nextId = valueRef.getReferenceElement(); if (nextId.isEmpty() && valueRef.getResource() != null) { @@ -171,19 +185,23 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor return; } - PathAndRef ref = new PathAndRef(searchParam.getName(), path, valueRef, false); - params.add(ref); + myPathAndRef = new PathAndRef(theSearchParam.getName(), thePath, valueRef, false); + theParams.add(myPathAndRef); break; default: - addUnexpectedDatatypeWarning(params, searchParam, value); + addUnexpectedDatatypeWarning(theParams, theSearchParam, theValue); break; } - }; + } - return extractSearchParams(theResource, extractor, RestSearchParameterTypeEnum.REFERENCE); + public PathAndRef get(IBase theValue, String thePath) { + extract(new SearchParamSet<>(), + new RuntimeSearchParam(null, null, "Reference", null, null, null, null, null, null, null), + theValue, thePath); + return myPathAndRef; + } } - @Override public SearchParamSet extractSearchParamTokens(IBaseResource theResource) { IExtractor extractor = createTokenExtractor(theResource); @@ -262,31 +280,115 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor @Override public SearchParamSet extractSearchParamDates(IBaseResource theResource) { - IExtractor extractor = (params, searchParam, value, path) -> { - String nextType = toRootTypeName(value); - String resourceType = toRootTypeName(theResource); + IExtractor extractor = new DateExtractor(theResource); + + return extractSearchParams(theResource, extractor, RestSearchParameterTypeEnum.DATE); + } + + @Override + public Date extractDateFromResource(IBase theValue, String thePath) { + DateExtractor extractor = new DateExtractor("DateType"); + return extractor.get(theValue, thePath).getValueHigh(); + } + + private class DateExtractor implements IExtractor { + + String myResourceType; + ResourceIndexedSearchParamDate myIndexedSearchParamDate = null; + + public DateExtractor(IBaseResource theResource) { + myResourceType = toRootTypeName(theResource); + } + + public DateExtractor(String theResourceType) { + myResourceType = theResourceType; + } + + @Override + public void extract(SearchParamSet theParams, RuntimeSearchParam theSearchParam, IBase theValue, String thePath) { + String nextType = toRootTypeName(theValue); switch (nextType) { case "date": case "dateTime": case "instant": - addDateTimeTypes(resourceType, params, searchParam, value); + addDateTimeTypes(myResourceType, theParams, theSearchParam, theValue); break; case "Period": - addDate_Period(resourceType, params, searchParam, value); + addDate_Period(myResourceType, theParams, theSearchParam, theValue); break; case "Timing": - addDate_Timing(resourceType, params, searchParam, value); + addDate_Timing(myResourceType, theParams, theSearchParam, theValue); break; case "string": // CarePlan.activitydate can be a string - ignored for now break; default: - addUnexpectedDatatypeWarning(params, searchParam, value); + addUnexpectedDatatypeWarning(theParams, theSearchParam, theValue); break; - } - }; - return extractSearchParams(theResource, extractor, RestSearchParameterTypeEnum.DATE); + } + } + + private void addDate_Period(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { + Date start = extractValueAsDate(myPeriodStartValueChild, theValue); + String startAsString = extractValueAsString(myPeriodStartValueChild, theValue); + Date end = extractValueAsDate(myPeriodEndValueChild, theValue); + + if (start != null || end != null) { + myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(theResourceType, theSearchParam.getName(), start, end, startAsString); + theParams.add(myIndexedSearchParamDate); + } + } + + private void addDate_Timing(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { + List> values = extractValuesAsFhirDates(myTimingEventValueChild, theValue); + + TreeSet dates = new TreeSet<>(); + String firstValue = null; + for (IPrimitiveType nextEvent : values) { + if (nextEvent.getValue() != null) { + dates.add(nextEvent.getValue()); + if (firstValue == null) { + firstValue = nextEvent.getValueAsString(); + } + } + } + + Optional repeat = myTimingRepeatValueChild.getAccessor().getFirstValueOrNull(theValue); + if (repeat.isPresent()) { + Optional bounds = myTimingRepeatBoundsValueChild.getAccessor().getFirstValueOrNull(repeat.get()); + if (bounds.isPresent()) { + String boundsType = toRootTypeName(bounds.get()); + if ("Period".equals(boundsType)) { + Date start = extractValueAsDate(myPeriodStartValueChild, bounds.get()); + Date end = extractValueAsDate(myPeriodEndValueChild, bounds.get()); + dates.add(start); + dates.add(end); + } + } + } + + if (!dates.isEmpty()) { + myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(theResourceType, theSearchParam.getName(), dates.first(), dates.last(), firstValue); + theParams.add(myIndexedSearchParamDate); + } + } + + @SuppressWarnings("unchecked") + private void addDateTimeTypes(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { + IPrimitiveType nextBaseDateTime = (IPrimitiveType) theValue; + if (nextBaseDateTime.getValue() != null) { + myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(theResourceType, theSearchParam.getName(), nextBaseDateTime.getValue(), nextBaseDateTime.getValue(), nextBaseDateTime.getValueAsString()); + theParams.add(myIndexedSearchParamDate); + } + } + + public ResourceIndexedSearchParamDate get(IBase theValue, String thePath) { + extract(new SearchParamSet<>(), + new RuntimeSearchParam(null, null, "date", null, null, null, null, null, null, null), + theValue, thePath); + return myIndexedSearchParamDate; + } } @Override @@ -389,7 +491,8 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor /** * Override parent because we're using FHIRPath here */ - private List extractValues(String thePaths, IBaseResource theResource) { + @Override + public List extractValues(String thePaths, IBaseResource theResource) { List values = new ArrayList<>(); if (isNotBlank(thePaths)) { String[] nextPathsSplit = split(thePaths); @@ -500,27 +603,67 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } + @Override + public List getCodingsFromCodeableConcept(IBase theValue) { + String nextType = BaseSearchParamExtractor.this.toRootTypeName(theValue); + if ("CodeableConcept".equals(nextType)) { + return myCodeableConceptCodingValueChild.getAccessor().getValues(theValue); + } else { + return null; + } + } + + @Override + public String getDisplayTextFromCodeableConcept(IBase theValue) { + String nextType = BaseSearchParamExtractor.this.toRootTypeName(theValue); + if ("CodeableConcept".equals(nextType)) { + return extractValueAsString(myCodeableConceptTextValueChild, theValue); + } else { + return null; + } + } + private void addToken_CodeableConcept(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - List codings = myCodeableConceptCodingValueChild.getAccessor().getValues(theValue); + List codings = getCodingsFromCodeableConcept(theValue); for (IBase nextCoding : codings) { addToken_Coding(theResourceType, theParams, theSearchParam, nextCoding); } - String text = extractValueAsString(myCodeableConceptTextValueChild, theValue); + String text = getDisplayTextFromCodeableConcept(theValue); if (isNotBlank(text)) { createStringIndexIfNotBlank(theResourceType, theParams, theSearchParam, text); } } private void addToken_Coding(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - String system = extractValueAsString(myCodingSystemValueChild, theValue); - String code = extractValueAsString(myCodingCodeValueChild, theValue); - createTokenIndexIfNotBlank(theResourceType, theParams, theSearchParam, system, code); + theParams.add(createSearchParamForCoding(theResourceType, theSearchParam, theValue)); - String text = extractValueAsString(myCodingDisplayValueChild, theValue); + String text = getDisplayTextForCoding(theValue); createStringIndexIfNotBlank(theResourceType, theParams, theSearchParam, text); } + @Override + public ResourceIndexedSearchParamToken createSearchParamForCoding(String theResourceType, RuntimeSearchParam theSearchParam, IBase theValue) { + String nextType = BaseSearchParamExtractor.this.toRootTypeName(theValue); + if ("Coding".equals(nextType)) { + String system = extractValueAsString(myCodingSystemValueChild, theValue); + String code = extractValueAsString(myCodingCodeValueChild, theValue); + return createTokenIndexIfNotBlank(theResourceType, theSearchParam, system, code); + } else { + return null; + } + } + + @Override + public String getDisplayTextForCoding(IBase theValue) { + String nextType = BaseSearchParamExtractor.this.toRootTypeName(theValue); + if ("Coding".equals(nextType)) { + return extractValueAsString(myCodingDisplayValueChild, theValue); + } else { + return null; + } + } + private void addToken_ContactPoint(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { String system = extractValueAsString(myContactPointSystemValueChild, theValue); String value = extractValueAsString(myContactPointValueValueChild, theValue); @@ -541,51 +684,6 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } } - private void addDate_Period(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - Date start = extractValueAsDate(myPeriodStartValueChild, theValue); - String startAsString = extractValueAsString(myPeriodStartValueChild, theValue); - Date end = extractValueAsDate(myPeriodEndValueChild, theValue); - - if (start != null || end != null) { - ResourceIndexedSearchParamDate nextEntity = new ResourceIndexedSearchParamDate(theResourceType, theSearchParam.getName(), start, end, startAsString); - theParams.add(nextEntity); - } - } - - private void addDate_Timing(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - List> values = extractValuesAsFhirDates(myTimingEventValueChild, theValue); - - TreeSet dates = new TreeSet<>(); - String firstValue = null; - for (IPrimitiveType nextEvent : values) { - if (nextEvent.getValue() != null) { - dates.add(nextEvent.getValue()); - if (firstValue == null) { - firstValue = nextEvent.getValueAsString(); - } - } - } - - Optional repeat = myTimingRepeatValueChild.getAccessor().getFirstValueOrNull(theValue); - if (repeat.isPresent()) { - Optional bounds = myTimingRepeatBoundsValueChild.getAccessor().getFirstValueOrNull(repeat.get()); - if (bounds.isPresent()) { - String boundsType = toRootTypeName(bounds.get()); - if ("Period".equals(boundsType)) { - Date start = extractValueAsDate(myPeriodStartValueChild, bounds.get()); - Date end = extractValueAsDate(myPeriodEndValueChild, bounds.get()); - dates.add(start); - dates.add(end); - } - } - } - - if (!dates.isEmpty()) { - ResourceIndexedSearchParamDate nextEntity = new ResourceIndexedSearchParamDate(theResourceType, theSearchParam.getName(), dates.first(), dates.last(), firstValue); - theParams.add(nextEntity); - } - } - private void addNumber_Duration(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { String system = extractValueAsString(myDurationSystemValueChild, theValue); String code = extractValueAsString(myDurationCodeValueChild, theValue); @@ -757,27 +855,19 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } } - private String toRootTypeName(IBase nextObject) { + @Override + public String toRootTypeName(IBase nextObject) { BaseRuntimeElementDefinition elementDefinition = getContext().getElementDefinition(nextObject.getClass()); BaseRuntimeElementDefinition rootParentDefinition = elementDefinition.getRootParentDefinition(); return rootParentDefinition.getName(); } - private String toTypeName(IBase nextObject) { + @Override + public String toTypeName(IBase nextObject) { BaseRuntimeElementDefinition elementDefinition = getContext().getElementDefinition(nextObject.getClass()); return elementDefinition.getName(); } - @SuppressWarnings("unchecked") - private void addDateTimeTypes(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - IPrimitiveType nextBaseDateTime = (IPrimitiveType) theValue; - if (nextBaseDateTime.getValue() != null) { - ResourceIndexedSearchParamDate param = new ResourceIndexedSearchParamDate(theResourceType, theSearchParam.getName(), nextBaseDateTime.getValue(), nextBaseDateTime.getValue(), nextBaseDateTime.getValueAsString()); - theParams.add(param); - } - } - - private void addUri_Uri(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { IPrimitiveType value = (IPrimitiveType) theValue; String valueAsString = value.getValueAsString(); @@ -809,8 +899,16 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } private void createTokenIndexIfNotBlank(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, String theSystem, String theValue) { + ResourceIndexedSearchParamToken nextEntity = createTokenIndexIfNotBlank(theResourceType, theSearchParam, theSystem, theValue); + if (nextEntity != null) { + theParams.add(nextEntity); + } + } + + private ResourceIndexedSearchParamToken createTokenIndexIfNotBlank(String theResourceType, RuntimeSearchParam theSearchParam, String theSystem, String theValue) { String system = theSystem; String value = theValue; + ResourceIndexedSearchParamToken nextEntity = null; if (isNotBlank(system) || isNotBlank(value)) { if (system != null && system.length() > ResourceIndexedSearchParamToken.MAX_LENGTH) { system = system.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH); @@ -819,10 +917,10 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor value = value.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH); } - ResourceIndexedSearchParamToken nextEntity; nextEntity = new ResourceIndexedSearchParamToken(theResourceType, theSearchParam.getName(), system, value); - theParams.add(nextEntity); } + + return nextEntity; } @Override diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/ISearchParamExtractor.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/ISearchParamExtractor.java index 15e4479ab8c..5182a63cd2d 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/ISearchParamExtractor.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/ISearchParamExtractor.java @@ -2,12 +2,10 @@ package ca.uhn.fhir.jpa.searchparam.extractor; import ca.uhn.fhir.context.RuntimeSearchParam; import ca.uhn.fhir.jpa.model.entity.*; +import org.hl7.fhir.instance.model.api.IBase; import org.hl7.fhir.instance.model.api.IBaseResource; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; +import java.util.*; /* * #%L @@ -53,6 +51,23 @@ public interface ISearchParamExtractor { String[] split(String theExpression); + List extractValues(String thePaths, IBaseResource theResource); + + String toRootTypeName(IBase nextObject); + + String toTypeName(IBase nextObject); + + PathAndRef extractReferenceLinkFromResource(IBase theValue, String thePath); + + Date extractDateFromResource(IBase theValue, String thePath); + + ResourceIndexedSearchParamToken createSearchParamForCoding(String theResourceType, RuntimeSearchParam theSearchParam, IBase theValue); + + String getDisplayTextForCoding(IBase theValue); + + List getCodingsFromCodeableConcept(IBase theValue); + + String getDisplayTextFromCodeableConcept(IBase theValue); class SearchParamSet extends HashSet { From 0e45db0ba92e3d693ddca05f893d38dcac2973e3 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Fri, 24 Apr 2020 16:23:59 -0400 Subject: [PATCH 11/31] More cleanup and test fixes. --- .../ca/uhn/fhir/cli/RunServerCommand.java | 13 -- hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml | 4 - .../ca/uhn/fhir/jpa/demo/ContextHolder.java | 19 --- .../ca/uhn/fhir/jpa/demo/JpaServerDemo.java | 2 +- hapi-fhir-jpaserver-base/pom.xml | 22 +++ .../jpa/config/dstu3/BaseDstu3Config.java | 2 - .../uhn/fhir/jpa/config/r4/BaseR4Config.java | 2 - .../uhn/fhir/jpa/config/r5/BaseR5Config.java | 2 - .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 113 +++---------- .../ObservationLastNIndexPersistDstu3Svc.java | 132 --------------- .../ObservationLastNIndexPersistR4Svc.java | 142 ---------------- .../ObservationLastNIndexPersistR5Svc.java | 142 ---------------- .../ObservationLastNIndexPersistSvc.java | 74 +++++---- ...ationIndexedCodeCodeableConceptEntity.java | 1 + .../dao/r4/FhirResourceDaoObservationR4.java | 4 - ...seJpaResourceProviderObservationDstu2.java | 153 ------------------ .../fhir/jpa/search/lastn/json/IdJson.java | 19 --- .../fhir/jpa/search/lastn/json/IndexJson.java | 20 --- .../search/lastn/util/SimpleStopWatch.java | 18 --- ...va => FhirResourceDaoR4SearchLastNIT.java} | 2 +- ...sourceDaoR4SearchWithElasticSearchIT.java} | 4 +- ...servationIndexedSearchParamLastNR4IT.java} | 8 +- ...asticsearchSvcMultipleObservationsIT.java} | 2 +- ...NElasticsearchSvcSingleObservationIT.java} | 2 +- .../extractor/BaseSearchParamExtractor.java | 5 +- .../autoconfigure/FhirAutoConfiguration.java | 2 +- .../resources/vm/jpa_resource_provider.vm | 2 +- 27 files changed, 100 insertions(+), 811 deletions(-) delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java rename hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/{FhirResourceDaoR4SearchLastNTest.java => FhirResourceDaoR4SearchLastNIT.java} (99%) rename hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/{FhirResourceDaoR4SearchWithElasticSearchTest.java => FhirResourceDaoR4SearchWithElasticSearchIT.java} (98%) rename hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/{PersistObservationIndexedSearchParamLastNR4Test.java => PersistObservationIndexedSearchParamLastNR4IT.java} (98%) rename hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/{LastNElasticsearchSvcMultipleObservationsTest.java => LastNElasticsearchSvcMultipleObservationsIT.java} (99%) rename hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/{LastNElasticsearchSvcSingleObservationTest.java => LastNElasticsearchSvcSingleObservationIT.java} (99%) diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java index c79ce66547a..30d9439b417 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java +++ b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/RunServerCommand.java @@ -72,8 +72,6 @@ public class RunServerCommand extends BaseCommand { options.addOption(null, OPTION_DISABLE_REFERENTIAL_INTEGRITY, false, "If this flag is set, the server will not enforce referential integrity"); addOptionalOption(options, "u", "url", "Url", "If this option is set, specifies the JDBC URL to use for the database connection"); - addOptionalOption(options, "d", "default-size", "PageSize", "If this option is set, specifies the default page size for number of query results"); - addOptionalOption(options, "m", "max-size", "MaxSize", "If this option is set, specifies the maximum result set size for queries"); Long defaultReuseSearchResults = DaoConfig.DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS; String defaultReuseSearchResultsStr = defaultReuseSearchResults == null ? "off" : String.valueOf(defaultReuseSearchResults); @@ -112,17 +110,6 @@ public class RunServerCommand extends BaseCommand { ContextHolder.setDatabaseUrl(theCommandLine.getOptionValue("u")); - String defaultPageSize = theCommandLine.getOptionValue("d"); - String maxPageSize = theCommandLine.getOptionValue("m"); - if (defaultPageSize != null) { - ContextHolder.setDefaultPageSize(Integer.valueOf(defaultPageSize)); - if (maxPageSize != null) { - ContextHolder.setMaxPageSize(Integer.valueOf(maxPageSize)); - } else { - ContextHolder.setMaxPageSize(Integer.valueOf(defaultPageSize)); - } - } - String reuseSearchResults = theCommandLine.getOptionValue(OPTION_REUSE_SEARCH_RESULTS_MILLIS); if (reuseSearchResults != null) { if (reuseSearchResults.equals("off")) { diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml index 2509802ef16..9988c9b8aba 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml @@ -136,10 +136,6 @@ - - org.postgresql - postgresql - diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java index 5968d9ef739..7e3eb9fecf1 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java @@ -33,8 +33,6 @@ public class ContextHolder { private static String ourPath; private static Long ourReuseSearchResultsMillis; private static String ourDatabaseUrl; - private static Integer myDefaultPageSize = 10; - private static Integer myMaxPageSize = 50; static { ourReuseSearchResultsMillis = DaoConfig.DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS; @@ -102,21 +100,4 @@ public class ContextHolder { public static void setDatabaseUrl(String theDatabaseUrl) { ourDatabaseUrl = theDatabaseUrl; } - - public static void setDefaultPageSize(Integer theDefaultPageSize) { - myDefaultPageSize = theDefaultPageSize; - } - - public static Integer getDefaultPageSize() { - return myDefaultPageSize; - } - - public static void setMaxPageSize(Integer theMaxPageSize) { - myMaxPageSize = theMaxPageSize; - } - - public static Integer getMaxPageSize() { - return myMaxPageSize; - } - } diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java index 40b8e8f1e57..2211b0ee469 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java @@ -160,7 +160,7 @@ public class JpaServerDemo extends RestfulServer { /* * This is a simple paging strategy that keeps the last 10 searches in memory */ - setPagingProvider(new FifoMemoryPagingProvider(10).setDefaultPageSize(ContextHolder.getDefaultPageSize()).setMaximumPageSize(ContextHolder.getMaxPageSize())); + setPagingProvider(new FifoMemoryPagingProvider(10)); // Register a CORS filter CorsInterceptor corsInterceptor = new CorsInterceptor(); diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index af0cee2d749..ba1e3d61c74 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -640,6 +640,28 @@ + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.2 + + true + 1 + false + alphabetical + + **/*IT.java + + + + + + integration-test + verify + + + + de.jpdigital hibernate54-ddl-maven-plugin diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java index 9e0d0207beb..bd6b9dc0a84 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/dstu3/BaseDstu3Config.java @@ -8,9 +8,7 @@ import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.dao.TransactionProcessor; import ca.uhn.fhir.jpa.dao.dstu3.TransactionProcessorVersionAdapterDstu3; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistDstu3Svc; import ca.uhn.fhir.jpa.provider.GraphQLProvider; -import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorDstu3; import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl; import ca.uhn.fhir.jpa.term.TermReadSvcDstu3; import ca.uhn.fhir.jpa.term.TermVersionAdapterSvcDstu3; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java index 11730fbf192..4034e720a63 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java @@ -7,8 +7,6 @@ import ca.uhn.fhir.jpa.config.BaseConfigDstu3Plus; import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.dao.TransactionProcessor; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR4Svc; import ca.uhn.fhir.jpa.dao.r4.TransactionProcessorVersionAdapterR4; import ca.uhn.fhir.jpa.provider.GraphQLProvider; import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java index 88e73510b39..615272a4e9c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r5/BaseR5Config.java @@ -7,10 +7,8 @@ import ca.uhn.fhir.jpa.config.BaseConfigDstu3Plus; import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.dao.TransactionProcessor; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR5Svc; import ca.uhn.fhir.jpa.dao.r5.TransactionProcessorVersionAdapterR5; import ca.uhn.fhir.jpa.provider.GraphQLProvider; -import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorR5; import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl; import ca.uhn.fhir.jpa.term.TermReadSvcR5; import ca.uhn.fhir.jpa.term.TermVersionAdapterSvcR5; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 68add2eb3de..35d03ae2c2e 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -320,86 +320,30 @@ public class SearchBuilder implements ISearchBuilder { } /* - * Fulltext search - */ - /* - if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { - if (myFulltextSearchSvc == null) { - if (myParams.containsKey(Constants.PARAM_TEXT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); - } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); - } - } - - List pids; - if (myParams.getEverythingMode() != null) { - pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); - } else { - pids = myFulltextSearchSvc.search(myResourceName, myParams); - } - if (pids.isEmpty()) { - // Will never match - pids = Collections.singletonList(new ResourcePersistentId(-1L)); - } - - myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids))); - } else if (myParams.isLastN()) { - if (myIElasticsearchSvc == null) { - if (myParams.isLastN()) { - throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); - } - } - - if (myParams.isLastN()) { - Integer myMaxObservationsPerCode = null; - String[] maxCountParams = theRequest.getParameters().get("max"); - if (maxCountParams != null && maxCountParams.length > 0) { - myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]); - } else { - throw new InvalidRequestException("Max parameter is required for $lastn operation"); - } - List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); - for (String lastnResourceId : lastnResourceIds) { - lastnPids.add(myIdHelperService.resolveResourcePersistentIds(myResourceName, lastnResourceId)); - } - if (lastnPids.isEmpty()) { - // Will never match - pids = Collections.singletonList(new ResourcePersistentId(-1L)); - } - myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids))); - } - } - */ - /* - * lastn search + * Fulltext or lastn search */ if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { - List lastnPids = new ArrayList<>(); - List fullTextSearchPids = new ArrayList<>(); - - if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { - if (myFulltextSearchSvc == null) { - if (myParams.containsKey(Constants.PARAM_TEXT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); - } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + List pids = new ArrayList<>(); + if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { + if (myFulltextSearchSvc == null) { + if (myParams.containsKey(Constants.PARAM_TEXT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + } } - } - if (myParams.getEverythingMode() != null) { - fullTextSearchPids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); - } else { - fullTextSearchPids = myFulltextSearchSvc.search(myResourceName, myParams); - } - } else { - if (myIElasticsearchSvc == null) { - if (myParams.isLastN()) { - throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); + if (myParams.getEverythingMode() != null) { + pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); + } else { + pids = myFulltextSearchSvc.search(myResourceName, myParams); + } + } else if (myParams.isLastN()) { + if (myIElasticsearchSvc == null) { + if (myParams.isLastN()) { + throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); + } } - } - - if (myParams.isLastN()) { Integer myMaxObservationsPerCode = null; String[] maxCountParams = theRequest.getParameters().get("max"); if (maxCountParams != null && maxCountParams.length > 0) { @@ -409,31 +353,16 @@ public class SearchBuilder implements ISearchBuilder { } List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); for (String lastnResourceId : lastnResourceIds) { - lastnPids.add(myIdHelperService.resolveResourcePersistentIds(myResourceName, lastnResourceId)); + pids.add(myIdHelperService.resolveResourcePersistentIds(myResourceName, lastnResourceId)); } } - } - - // - List pids; - if (fullTextSearchPids.isEmpty()) { - pids = lastnPids; - } else if (lastnPids.isEmpty()) { - pids = fullTextSearchPids; - } else { - // Intersection of the fullTextSearchPids and lastnPids - Set pidIntersection = fullTextSearchPids.stream() - .distinct() - .filter(lastnPids::contains) - .collect(Collectors.toSet()); - pids = new ArrayList<>(pidIntersection); - } if (pids.isEmpty()) { // Will never match pids = Collections.singletonList(new ResourcePersistentId(-1L)); } myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids))); + } /* diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java deleted file mode 100644 index 5834d1bb5d7..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistDstu3Svc.java +++ /dev/null @@ -1,132 +0,0 @@ -package ca.uhn.fhir.jpa.dao.lastn; - -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.*; -import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; -import org.hl7.fhir.dstu3.model.CodeableConcept; -import org.hl7.fhir.dstu3.model.Coding; -import org.hl7.fhir.dstu3.model.Observation; -import org.hl7.fhir.dstu3.model.Reference; -import org.hl7.fhir.r4.model.DateTimeType; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; - -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.PersistenceContextType; -import java.util.*; - -@Transactional(propagation = Propagation.REQUIRED) -public class ObservationLastNIndexPersistDstu3Svc { - - @PersistenceContext(type = PersistenceContextType.TRANSACTION) - protected EntityManager myEntityManager; - - @Autowired - IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; - - @Autowired - IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; - - public void indexObservation(Observation theObservation) { - // Only index for lastn if Observation has a subject and effective date/time - if(theObservation.getSubject() == null || !theObservation.hasEffective()) { - return; - } - - // Determine most recent effective date/time - Date effectiveDtm = null; - if (theObservation.hasEffectiveDateTimeType()) { - effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); - } else if (theObservation.hasEffectivePeriod()) { - effectiveDtm = theObservation.getEffectivePeriod().getEnd(); - } - if (effectiveDtm == null) { - return; - } - - // Determine if an index already exists for Observation: - boolean observationIndexUpdate = false; - ObservationIndexedSearchParamLastNEntity indexedObservation = null; - if (theObservation.hasId()) { - indexedObservation = myResourceIndexedObservationLastNDao.findForIdentifier(theObservation.getIdElement().getIdPart()); - } - if (indexedObservation == null) { - indexedObservation = new ObservationIndexedSearchParamLastNEntity(); - } else { - observationIndexUpdate = true; - } - indexedObservation.setEffectiveDtm(effectiveDtm); - Reference subjectReference = theObservation.getSubject(); - String subjectId = subjectReference.getReference(); - String resourcePID = theObservation.getIdElement().getIdPart(); - indexedObservation.setIdentifier(resourcePID); - indexedObservation.setSubject(subjectId); - - // Build CodeableConcept entities for Observation.Category - Set categoryConcepts = new HashSet<>(); - for(CodeableConcept categoryCodeableConcept : theObservation.getCategory()) { - // Build Coding entities for each category CodeableConcept - Set categoryCodingEntities = new HashSet<>(); - ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConceptEntity = new ObservationIndexedCategoryCodeableConceptEntity(categoryCodeableConcept.getText()); - for(Coding categoryCoding : categoryCodeableConcept.getCoding()){ - categoryCodingEntities.add(new ObservationIndexedCategoryCodingEntity(categoryCoding.getSystem(), categoryCoding.getCode(), categoryCoding.getDisplay())); - } - categoryCodeableConceptEntity.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); - categoryConcepts.add(categoryCodeableConceptEntity); - } - indexedObservation.setCategoryCodeableConcepts(categoryConcepts); - - // Build CodeableConcept entity for Observation.Code. - CodeableConcept codeCodeableConcept = theObservation.getCode(); - String observationCodeNormalizedId = null; - - // Determine if a Normalized ID was created previously for Observation Code - boolean observationCodeUpdate = false; - for (Coding codeCoding : codeCodeableConcept.getCoding()) { - if (codeCoding.hasCode() && codeCoding.hasSystem()) { - observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); - } else { - observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); - } - if(observationCodeNormalizedId != null) { - observationCodeUpdate = true; - break; - } - } - // Generate a new a normalized ID if necessary - if (observationCodeNormalizedId == null) { - observationCodeNormalizedId = UUID.randomUUID().toString(); - } - - // Create/update normalized Observation Code index record - ObservationIndexedCodeCodeableConceptEntity codeableConceptField = new ObservationIndexedCodeCodeableConceptEntity(codeCodeableConcept.getText(), observationCodeNormalizedId); - for (Coding codeCoding : codeCodeableConcept.getCoding()) { - codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); - } - if (observationCodeUpdate) { - myEntityManager.merge(codeableConceptField); - } else { - myEntityManager.persist(codeableConceptField); - } - - indexedObservation.setObservationCode(codeableConceptField); - indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); - if (observationIndexUpdate) { - myEntityManager.merge(indexedObservation); - } else { - myEntityManager.persist(indexedObservation); - } - - } - - public void deleteObservationIndex(IBasePersistedResource theEntity) { - ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findForIdentifier(theEntity.getIdDt().getIdPart()); - if (deletedObservationLastNEntity != null) { - myEntityManager.remove(deletedObservationLastNEntity); - } - } - -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java deleted file mode 100644 index d1d1f45e439..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR4Svc.java +++ /dev/null @@ -1,142 +0,0 @@ -package ca.uhn.fhir.jpa.dao.lastn; - -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.*; -import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; -import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor; -import ca.uhn.fhir.jpa.searchparam.extractor.PathAndRef; -import org.hl7.fhir.r4.model.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; - -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.PersistenceContextType; -import java.util.*; - -@Transactional(propagation = Propagation.REQUIRED) -public class ObservationLastNIndexPersistR4Svc { - - @PersistenceContext(type = PersistenceContextType.TRANSACTION) - protected EntityManager myEntityManager; - - @Autowired - IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; - - @Autowired - IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; - - public void indexObservation(Observation theObservation) { - // Only index for lastn if Observation has a subject and effective date/time - if(theObservation.getSubject() == null || !theObservation.hasEffective()) { - return; - } - - // Determine most recent effective date/time - Date effectiveDtm = null; - if (theObservation.hasEffectiveDateTimeType()) { - effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); - } else if (theObservation.hasEffectiveInstantType()) { - effectiveDtm = theObservation.getEffectiveInstantType().getValue(); - } else if (theObservation.hasEffectivePeriod()) { - effectiveDtm = theObservation.getEffectivePeriod().getEnd(); - } else if (theObservation.hasEffectiveTiming()) { - List events = theObservation.getEffectiveTiming().getEvent(); - for (DateTimeType event : events) { - Date eventDtm = event.getValue(); - if (effectiveDtm == null || eventDtm.after(effectiveDtm)) { - effectiveDtm = eventDtm; - } - } - } - if (effectiveDtm == null) { - return; - } - - // Determine if an index already exists for Observation: - boolean observationIndexUpdate = false; - ObservationIndexedSearchParamLastNEntity indexedObservation = null; - if (theObservation.hasId()) { - indexedObservation = myResourceIndexedObservationLastNDao.findForIdentifier(theObservation.getIdElement().getIdPart()); - } - if (indexedObservation == null) { - indexedObservation = new ObservationIndexedSearchParamLastNEntity(); - } else { - observationIndexUpdate = true; - } - - indexedObservation.setEffectiveDtm(effectiveDtm); - Reference subjectReference = theObservation.getSubject(); - String subjectId = subjectReference.getReference(); - String resourcePID = theObservation.getIdElement().getIdPart(); - indexedObservation.setIdentifier(resourcePID); - indexedObservation.setSubject(subjectId); - - // Build CodeableConcept entities for Observation.Category - Set categoryConcepts = new HashSet<>(); - for(CodeableConcept categoryCodeableConcept : theObservation.getCategory()) { - // Build Coding entities for each category CodeableConcept - Set categoryCodingEntities = new HashSet<>(); - ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConceptEntity = new ObservationIndexedCategoryCodeableConceptEntity(categoryCodeableConcept.getText()); - for(Coding categoryCoding : categoryCodeableConcept.getCoding()){ - categoryCodingEntities.add(new ObservationIndexedCategoryCodingEntity(categoryCoding.getSystem(), categoryCoding.getCode(), categoryCoding.getDisplay())); - } - categoryCodeableConceptEntity.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); - categoryConcepts.add(categoryCodeableConceptEntity); - } - indexedObservation.setCategoryCodeableConcepts(categoryConcepts); - - // Build CodeableConcept entity for Observation.Code. - CodeableConcept codeCodeableConcept = theObservation.getCode(); - String observationCodeNormalizedId = null; - - // Determine if a Normalized ID was created previously for Observation Code - boolean observationCodeUpdate = false; - for (Coding codeCoding : codeCodeableConcept.getCoding()) { - if (codeCoding.hasCode() && codeCoding.hasSystem()) { - observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); - } else { - observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); - } - if(observationCodeNormalizedId != null) { - observationCodeUpdate = true; - break; - } - } - // Generate a new a normalized ID if necessary - if (observationCodeNormalizedId == null) { - observationCodeNormalizedId = UUID.randomUUID().toString(); - } - - // Create/update normalized Observation Code index record - ObservationIndexedCodeCodeableConceptEntity codeableConceptField = new ObservationIndexedCodeCodeableConceptEntity(codeCodeableConcept.getText(), observationCodeNormalizedId); - for (Coding codeCoding : codeCodeableConcept.getCoding()) { - codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); - } - if (observationCodeUpdate) { - myEntityManager.merge(codeableConceptField); - } else { - myEntityManager.persist(codeableConceptField); - } - - indexedObservation.setObservationCode(codeableConceptField); - indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); - if (observationIndexUpdate) { - myEntityManager.merge(indexedObservation); - } else { - myEntityManager.persist(indexedObservation); - } - - } - - public void deleteObservationIndex(IBasePersistedResource theEntity) { - ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findForIdentifier(theEntity.getIdDt().getIdPart()); - if(deletedObservationLastNEntity != null) { - myEntityManager.remove(deletedObservationLastNEntity); - } - } - -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java deleted file mode 100644 index 176d35ce7d5..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistR5Svc.java +++ /dev/null @@ -1,142 +0,0 @@ -package ca.uhn.fhir.jpa.dao.lastn; - -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.*; -import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; -import org.hl7.fhir.r5.model.DateTimeType; -import org.hl7.fhir.r5.model.CodeableConcept; -import org.hl7.fhir.r5.model.Coding; -import org.hl7.fhir.r5.model.Observation; -import org.hl7.fhir.r5.model.Reference; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; - -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.PersistenceContextType; -import java.util.*; - -@Transactional(propagation = Propagation.REQUIRED) -public class ObservationLastNIndexPersistR5Svc { - - @PersistenceContext(type = PersistenceContextType.TRANSACTION) - protected EntityManager myEntityManager; - - @Autowired - IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; - - @Autowired - IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; - - public void indexObservation(Observation theObservation) { - // Only index for lastn if Observation has a subject and effective date/time - if(theObservation.getSubject() == null || !theObservation.hasEffective()) { - return; - } - - // Determine most recent effective date/time - Date effectiveDtm = null; - if (theObservation.hasEffectiveDateTimeType()) { - effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); - } else if (theObservation.hasEffectiveInstantType()) { - effectiveDtm = theObservation.getEffectiveInstantType().getValue(); - } else if (theObservation.hasEffectivePeriod()) { - effectiveDtm = theObservation.getEffectivePeriod().getEnd(); - } else if (theObservation.hasEffectiveTiming()) { - List events = theObservation.getEffectiveTiming().getEvent(); - for (DateTimeType event : events) { - Date eventDtm = event.getValue(); - if (effectiveDtm == null || eventDtm.after(effectiveDtm)) { - effectiveDtm = eventDtm; - } - } - } - if (effectiveDtm == null) { - return; - } - - // Determine if an index already exists for Observation: - boolean observationIndexUpdate = false; - ObservationIndexedSearchParamLastNEntity indexedObservation = null; - if (theObservation.hasId()) { - indexedObservation = myResourceIndexedObservationLastNDao.findForIdentifier(theObservation.getIdElement().getIdPart()); - } - if (indexedObservation == null) { - indexedObservation = new ObservationIndexedSearchParamLastNEntity(); - } else { - observationIndexUpdate = true; - } - indexedObservation.setEffectiveDtm(effectiveDtm); - Reference subjectReference = theObservation.getSubject(); - String subjectId = subjectReference.getReference(); - String resourcePID = theObservation.getIdElement().getIdPart(); - indexedObservation.setIdentifier(resourcePID); - indexedObservation.setSubject(subjectId); - - // Build CodeableConcept entities for Observation.Category - Set categoryConcepts = new HashSet<>(); - for(CodeableConcept categoryCodeableConcept : theObservation.getCategory()) { - // Build Coding entities for each category CodeableConcept - Set categoryCodingEntities = new HashSet<>(); - ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConceptEntity = new ObservationIndexedCategoryCodeableConceptEntity(categoryCodeableConcept.getText()); - for(Coding categoryCoding : categoryCodeableConcept.getCoding()){ - categoryCodingEntities.add(new ObservationIndexedCategoryCodingEntity(categoryCoding.getSystem(), categoryCoding.getCode(), categoryCoding.getDisplay())); - } - categoryCodeableConceptEntity.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); - categoryConcepts.add(categoryCodeableConceptEntity); - } - indexedObservation.setCategoryCodeableConcepts(categoryConcepts); - - // Build CodeableConcept entity for Observation.Code. - CodeableConcept codeCodeableConcept = theObservation.getCode(); - String observationCodeNormalizedId = null; - - // Determine if a Normalized ID was created previously for Observation Code - boolean observationCodeUpdate = false; - for (Coding codeCoding : codeCodeableConcept.getCoding()) { - if (codeCoding.hasCode() && codeCoding.hasSystem()) { - observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); - } else { - observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); - } - if(observationCodeNormalizedId != null) { - observationCodeUpdate = true; - break; - } - } - // Generate a new a normalized ID if necessary - if (observationCodeNormalizedId == null) { - observationCodeNormalizedId = UUID.randomUUID().toString(); - } - - // Create/update normalized Observation Code index record - ObservationIndexedCodeCodeableConceptEntity codeableConceptField = new ObservationIndexedCodeCodeableConceptEntity(codeCodeableConcept.getText(), observationCodeNormalizedId); - for (Coding codeCoding : codeCodeableConcept.getCoding()) { - codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); - } - if (observationCodeUpdate) { - myEntityManager.merge(codeableConceptField); - } else { - myEntityManager.persist(codeableConceptField); - } - - indexedObservation.setObservationCode(codeableConceptField); - indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); - if (observationIndexUpdate) { - myEntityManager.merge(indexedObservation); - } else { - myEntityManager.persist(indexedObservation); - } - - } - - public void deleteObservationIndex(IBasePersistedResource theEntity) { - ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findForIdentifier(theEntity.getIdDt().getIdPart()); - if(deletedObservationLastNEntity != null) { - myEntityManager.remove(deletedObservationLastNEntity); - } - } - -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java index a906f852002..105fc81efda 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java @@ -53,8 +53,12 @@ public class ObservationLastNIndexPersistSvc { effectiveDtm = mySearchParameterExtractor.extractDateFromResource(effectiveDateElement.get(0), "Observation.effective"); } - // Only index for lastn if Observation has a subject and effective date/time - if (subjectId == null || effectiveDtm == null) { + // Build CodeableConcept entity for Observation.Code. + List observationCodeCodeableConcepts = mySearchParameterExtractor.extractValues("Observation.code", theResource); + + + // Only index for lastn if Observation has a subject, effective date/time and code + if (subjectId == null || effectiveDtm == null || observationCodeCodeableConcepts.size() == 0) { return; } @@ -76,17 +80,6 @@ public class ObservationLastNIndexPersistSvc { indexedObservation.setIdentifier(resourcePID); indexedObservation.setSubject(subjectId); - // Build CodeableConcept entities for Observation.Category - List observationCategoryCodeableConcepts = mySearchParameterExtractor.extractValues("Observation.category", theResource); - Set categoryCodeableConceptEntities = new HashSet<>(); - for (IBase categoryCodeableConcept : observationCategoryCodeableConcepts) { - // Build CodeableConcept entities for each category CodeableConcept - categoryCodeableConceptEntities.add(getCategoryCodeableConceptEntities(categoryCodeableConcept)); - } - indexedObservation.setCategoryCodeableConcepts(categoryCodeableConceptEntities); - - // Build CodeableConcept entity for Observation.Code. - List observationCodeCodeableConcepts = mySearchParameterExtractor.extractValues("Observation.code", theResource); // Determine if a Normalized ID was created previously for Observation Code boolean observationCodeUpdate = false; @@ -102,6 +95,15 @@ public class ObservationLastNIndexPersistSvc { // Create/update normalized Observation Code index record ObservationIndexedCodeCodeableConceptEntity codeableConceptField = getCodeCodeableConcept(observationCodeCodeableConcepts.get(0), observationCodeNormalizedId); + // Build CodeableConcept entities for Observation.Category + List observationCategoryCodeableConcepts = mySearchParameterExtractor.extractValues("Observation.category", theResource); + Set categoryCodeableConceptEntities = new HashSet<>(); + for (IBase categoryCodeableConcept : observationCategoryCodeableConcepts) { + // Build CodeableConcept entities for each category CodeableConcept + categoryCodeableConceptEntities.add(getCategoryCodeableConceptEntities(categoryCodeableConcept)); + } + indexedObservation.setCategoryCodeableConcepts(categoryCodeableConceptEntities); + if (observationCodeUpdate) { myEntityManager.merge(codeableConceptField); } else { @@ -152,16 +154,18 @@ public class ObservationLastNIndexPersistSvc { ResourceIndexedSearchParamToken param = mySearchParameterExtractor.createSearchParamForCoding("Observation", new RuntimeSearchParam(null, null, "code", null, null, null, null, null, null, null), nextCoding); - String system = param.getSystem(); - String code = param.getValue(); - String text = mySearchParameterExtractor.getDisplayTextForCoding(nextCoding); - if (code != null && system != null) { - codeCodeableConceptId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(code, system); - } else { - codeCodeableConceptId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(text); - } - if (codeCodeableConceptId != null) { - break; + if (param != null) { + String system = param.getSystem(); + String code = param.getValue(); + String text = mySearchParameterExtractor.getDisplayTextForCoding(nextCoding); + if (code != null && system != null) { + codeCodeableConceptId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(code, system); + } else { + codeCodeableConceptId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(text); + } + if (codeCodeableConceptId != null) { + break; + } } } @@ -172,20 +176,28 @@ public class ObservationLastNIndexPersistSvc { ResourceIndexedSearchParamToken param = mySearchParameterExtractor.createSearchParamForCoding("Observation", new RuntimeSearchParam(null, null, "category", null, null, null, null, null, null, null), theValue); - String system = param.getSystem(); - String code = param.getValue(); - String text = mySearchParameterExtractor.getDisplayTextForCoding(theValue); - return new ObservationIndexedCategoryCodingEntity(system, code, text); + ObservationIndexedCategoryCodingEntity observationIndexedCategoryCodingEntity = null; + if (param != null) { + String system = param.getSystem(); + String code = param.getValue(); + String text = mySearchParameterExtractor.getDisplayTextForCoding(theValue); + observationIndexedCategoryCodingEntity = new ObservationIndexedCategoryCodingEntity(system, code, text); + } + return observationIndexedCategoryCodingEntity; } private ObservationIndexedCodeCodingEntity getCodeCoding(IBase theValue, String observationCodeNormalizedId) { ResourceIndexedSearchParamToken param = mySearchParameterExtractor.createSearchParamForCoding("Observation", new RuntimeSearchParam(null, null, "code", null, null, null, null, null, null, null), theValue); - String system = param.getSystem(); - String code = param.getValue(); - String text = mySearchParameterExtractor.getDisplayTextForCoding(theValue); - return new ObservationIndexedCodeCodingEntity(system, code, text, observationCodeNormalizedId); + ObservationIndexedCodeCodingEntity observationIndexedCodeCodingEntity = null; + if (param != null) { + String system = param.getSystem(); + String code = param.getValue(); + String text = mySearchParameterExtractor.getDisplayTextForCoding(theValue); + observationIndexedCodeCodingEntity = new ObservationIndexedCodeCodingEntity(system, code, text, observationCodeNormalizedId); + } + return observationIndexedCodeCodingEntity; } public void deleteObservationIndex(IBasePersistedResource theEntity) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java index bf1d994d31d..2757173505d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java @@ -22,6 +22,7 @@ public class ObservationIndexedCodeCodeableConceptEntity { @Column(name = "CODEABLE_CONCEPT_TEXT", nullable = true) private String myCodeableConceptText; + // TODO: Make coding a Collection. Need to first figure out how to maintain this over time. @IndexedEmbedded(depth=2, prefix = "coding") // @OneToMany(mappedBy = "myCodeableConceptId", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) @JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_CONCEPT_CODE")) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index 14828dd2727..55f47758b3f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -21,14 +21,10 @@ package ca.uhn.fhir.jpa.dao.r4; */ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; -import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistR4Svc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java deleted file mode 100644 index 9a61f635368..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaResourceProviderObservationDstu2.java +++ /dev/null @@ -1,153 +0,0 @@ -package ca.uhn.fhir.jpa.provider; - -import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; -import ca.uhn.fhir.jpa.model.util.JpaConstants; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.model.api.Include; -import ca.uhn.fhir.model.api.annotation.Description; -import ca.uhn.fhir.model.dstu2.resource.Observation; -import ca.uhn.fhir.model.valueset.BundleTypeEnum; -import ca.uhn.fhir.rest.annotation.*; -import ca.uhn.fhir.rest.api.SearchTotalModeEnum; -import ca.uhn.fhir.rest.api.SortOrderEnum; -import ca.uhn.fhir.rest.api.SortSpec; -import ca.uhn.fhir.rest.api.SummaryEnum; -import ca.uhn.fhir.rest.api.server.IBundleProvider; -import ca.uhn.fhir.rest.param.*; - -import java.util.Set; - -/* - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2020 University Health Network - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -public class BaseJpaResourceProviderObservationDstu2 extends JpaResourceProviderDstu2 { - - /** - * Observation/$lastn - */ - @Operation(name = JpaConstants.OPERATION_LASTN, idempotent = true, bundleType = BundleTypeEnum.SEARCHSET) - public IBundleProvider observationLastN( - - javax.servlet.http.HttpServletRequest theServletRequest, - javax.servlet.http.HttpServletResponse theServletResponse, - - ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails, - - @Description(shortDefinition="Search the contents of the resource's data using a filter") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_FILTER) - StringAndListParam theFtFilter, - - @Description(shortDefinition="Search the contents of the resource's data using a fulltext search") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT) - StringAndListParam theFtContent, - - @Description(shortDefinition="Search the contents of the resource's narrative using a fulltext search") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_TEXT) - StringAndListParam theFtText, - - @Description(shortDefinition="The classification of the type of observation") - @OperationParam(name="category") - TokenAndListParam theCategory, - - @Description(shortDefinition="The code of the observation type") - @OperationParam(name="code") - TokenAndListParam theCode, - - @Description(shortDefinition="Obtained date/time. If the obtained element is a period, a date that falls in the period") - @OperationParam(name="date") - DateRangeParam theDate, - - @Description(shortDefinition="The subject that the observation is about (if patient)") - @OperationParam(name="patient") - ReferenceAndListParam thePatient, - - @Description(shortDefinition="The subject that the observation is about") - @OperationParam(name="subject" ) - ReferenceAndListParam theSubject, - - @IncludeParam(reverse=true) - Set theRevIncludes, - @Description(shortDefinition="Only return resources which were last updated as specified by the given range") - @OperationParam(name="_lastUpdated") - DateRangeParam theLastUpdated, - - @IncludeParam(allow= { - "Observation:based-on", - "Observation:derived-from", - "Observation:device", - "Observation:encounter", - "Observation:focus", - "Observation:has-member", - "Observation:part-of", - "Observation:patient", - "Observation:performer", - "Observation:specimen", - "Observation:subject", - "*" - }) - Set theIncludes, - - @Sort - SortSpec theSort, - - @ca.uhn.fhir.rest.annotation.Count - Integer theCount, - - SummaryEnum theSummaryMode, - - SearchTotalModeEnum theSearchTotalMode - - ) { - startRequest(theServletRequest); - try { - SearchParameterMap paramMap = new SearchParameterMap(); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_FILTER, theFtFilter); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, theFtContent); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_TEXT, theFtText); - paramMap.add("category", theCategory); - paramMap.add("code", theCode); - paramMap.add("date", theDate); - paramMap.add("patient", thePatient); - paramMap.add("subject", theSubject); - paramMap.setRevIncludes(theRevIncludes); - paramMap.setLastUpdated(theLastUpdated); - paramMap.setIncludes(theIncludes); - paramMap.setLastN(true); - if (theSort == null) { - SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); - SortSpec observationCode = new SortSpec("code").setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); - if (thePatient != null && theSubject == null) { - theSort = new SortSpec("patient").setChain(observationCode); - } else { - theSort = new SortSpec("subject").setChain(observationCode); - } - } - paramMap.setSort(theSort); - paramMap.setCount(theCount); - paramMap.setSummaryMode(theSummaryMode); - paramMap.setSearchTotalMode(theSearchTotalMode); - - return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); - } finally { - endRequest(theServletRequest); - } - } - -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java deleted file mode 100644 index 1df4410f054..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java +++ /dev/null @@ -1,19 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.json; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) -public class IdJson { - - @JsonProperty(value = "_id", required = true) - private String myId; - - public IdJson(String theId) { - myId = theId; - } - - public String getId() { return myId; } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java deleted file mode 100644 index dc1880124b4..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java +++ /dev/null @@ -1,20 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.json; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) -public class IndexJson { - - @JsonProperty(value = "index", required = true) - private IdJson myIndex; - - public IndexJson(IdJson theIndex) { - myIndex = theIndex; - } - - public IdJson getId() { return myIndex; } - -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java deleted file mode 100644 index 20381a5e2fe..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java +++ /dev/null @@ -1,18 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.util; - -public class SimpleStopWatch { - private long myStarted = System.currentTimeMillis(); - - public SimpleStopWatch() { - - } - - public long getElapsedTime() { - return System.currentTimeMillis() - myStarted; - } - - public void restart() { - myStarted = System.currentTimeMillis(); - } - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java similarity index 99% rename from hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNTest.java rename to hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java index c63a1a8932e..58f16a68383 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java @@ -28,7 +28,7 @@ import static org.mockito.Mockito.when; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class }) -public class FhirResourceDaoR4SearchLastNTest extends BaseJpaTest { +public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { @Autowired @Qualifier("myPatientDaoR4") diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java similarity index 98% rename from hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchTest.java rename to hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java index 7c6bcca1295..85e53af4ad6 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java @@ -60,10 +60,10 @@ import static org.junit.Assert.assertThat; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = {TestR4ConfigWithElasticSearch.class}) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) -public class FhirResourceDaoR4SearchWithElasticSearchTest extends BaseJpaTest { +public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest { public static final String URL_MY_CODE_SYSTEM = "http://example.com/my_code_system"; public static final String URL_MY_VALUE_SET = "http://example.com/my_value_set"; - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4SearchWithElasticSearchTest.class); + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4SearchWithElasticSearchIT.class); @Autowired protected DaoConfig myDaoConfig; @Autowired diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java similarity index 98% rename from hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4Test.java rename to hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java index 79f08459135..9a552c09a0d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java @@ -40,7 +40,7 @@ import static org.junit.Assert.assertTrue; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = {TestR4ConfigWithElasticsearchClient.class}) -public class PersistObservationIndexedSearchParamLastNR4Test { +public class PersistObservationIndexedSearchParamLastNR4IT { @Autowired IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; @@ -48,9 +48,6 @@ public class PersistObservationIndexedSearchParamLastNR4Test { @Autowired IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; -// @Autowired -// ObservationLastNIndexPersistR4Svc myObservationLastNIndexPersistR4Svc; - @Autowired private ElasticsearchSvcImpl elasticsearchSvc; @@ -68,9 +65,6 @@ public class PersistObservationIndexedSearchParamLastNR4Test { myResourceIndexedObservationLastNDao.deleteAll(); myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); -// testObservationPersist = new BaseObservationLastNIndexPersistSvc(myEntityManager, myResourceIndexedObservationLastNDao, -// myObservationIndexedCodeCodingSearchParamDao, mySearchParamExtractor, myContext); - } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java similarity index 99% rename from hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java rename to hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index 2f495cdea88..7eb78281ec3 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -26,7 +26,7 @@ import static org.junit.Assert.*; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = {TestElasticsearchConfig.class}) -public class LastNElasticsearchSvcMultipleObservationsTest { +public class LastNElasticsearchSvcMultipleObservationsIT { @Autowired private ElasticsearchSvcImpl elasticsearchSvc; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java similarity index 99% rename from hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java rename to hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java index e2dc8f23bb8..b78e14821b3 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java @@ -27,7 +27,7 @@ import static org.junit.Assert.assertTrue; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = {TestElasticsearchConfig.class}) -public class LastNElasticsearchSvcSingleObservationTest { +public class LastNElasticsearchSvcSingleObservationIT { @Autowired ElasticsearchSvcImpl elasticsearchSvc; diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java index 8a73cdf1349..78d54ead1eb 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java @@ -636,7 +636,10 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } private void addToken_Coding(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - theParams.add(createSearchParamForCoding(theResourceType, theSearchParam, theValue)); + ResourceIndexedSearchParamToken resourceIndexedSearchParamToken = createSearchParamForCoding(theResourceType, theSearchParam, theValue); + if (resourceIndexedSearchParamToken != null) { + theParams.add(resourceIndexedSearchParamToken); + } String text = getDisplayTextForCoding(theValue); createStringIndexIfNotBlank(theResourceType, theParams, theSearchParam, text); diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/src/main/java/ca/uhn/fhir/spring/boot/autoconfigure/FhirAutoConfiguration.java b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/src/main/java/ca/uhn/fhir/spring/boot/autoconfigure/FhirAutoConfiguration.java index c84e17cad24..91566fc9527 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/src/main/java/ca/uhn/fhir/spring/boot/autoconfigure/FhirAutoConfiguration.java +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/src/main/java/ca/uhn/fhir/spring/boot/autoconfigure/FhirAutoConfiguration.java @@ -164,7 +164,7 @@ public class FhirAutoConfiguration { private ScheduledExecutorService myScheduledExecutorService; @Configuration - @EntityScan(basePackages = {"ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.model.entity"}) + @EntityScan(basePackages = {"ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.dao.lastn.entity"}) @Import({ SubscriptionChannelConfig.class, SubscriptionProcessorConfig.class, diff --git a/hapi-tinder-plugin/src/main/resources/vm/jpa_resource_provider.vm b/hapi-tinder-plugin/src/main/resources/vm/jpa_resource_provider.vm index 665f598fd98..737dfb1fb04 100644 --- a/hapi-tinder-plugin/src/main/resources/vm/jpa_resource_provider.vm +++ b/hapi-tinder-plugin/src/main/resources/vm/jpa_resource_provider.vm @@ -24,7 +24,7 @@ import ca.uhn.fhir.rest.api.SearchTotalModeEnum; public class ${className}ResourceProvider extends ## We have specialized base classes for RPs that handle certain resource types. These ## RPs implement type specific operations -#if ( $version != 'dstu' && (${className} == 'Encounter' || ${className} == 'Patient' || ${className} == 'ValueSet' || ${className} == 'QuestionnaireAnswers' || ${className} == 'CodeSystem' || ($version != 'dstu2' && ${className} == 'ConceptMap') || ${className} == 'MessageHeader' || ${className} == 'Composition' || ${className} == 'StructureDefinition' || ${className} == 'Observation' )) +#if ( $version != 'dstu' && (${className} == 'Encounter' || ${className} == 'Patient' || ${className} == 'ValueSet' || ${className} == 'QuestionnaireAnswers' || ${className} == 'CodeSystem' || ($version != 'dstu2' && ${className} == 'ConceptMap') || ${className} == 'MessageHeader' || ${className} == 'Composition' || ${className} == 'StructureDefinition' || ($version != 'dstu2' && ${className} == 'Observation') )) BaseJpaResourceProvider${className}${versionCapitalized} #else JpaResourceProvider${versionCapitalized}<${className}> From 3e49e5615fe5772e10075988e786a63b0aee5906 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Sat, 25 Apr 2020 22:14:55 -0400 Subject: [PATCH 12/31] Fixing merge conflicts after merge from Master. --- .../src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java | 2 +- .../main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java | 4 ---- .../jpa/searchparam/extractor/BaseSearchParamExtractor.java | 4 ++-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 1d7e5b68cd0..8f45cb977dc 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -359,7 +359,7 @@ public class SearchBuilder implements ISearchBuilder { } List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); for (String lastnResourceId : lastnResourceIds) { - pids.add(myIdHelperService.resolveResourcePersistentIds(myResourceName, lastnResourceId)); + pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); } } if (pids.isEmpty()) { diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java index 9f4dcc03cba..0acddbc609b 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java @@ -180,10 +180,6 @@ public class ResourceLink extends BaseResourceIndex { myTargetResourceUrl = theTargetResourceUrl.getValue(); } - public String getTargetResourceUrl() { - return myTargetResourceUrl; - } - public void setTargetResourceUrlCanonical(String theTargetResourceUrl) { Validate.notBlank(theTargetResourceUrl); diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java index 34b5804e666..a35195e6c94 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java @@ -374,7 +374,7 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor Date end = extractValueAsDate(myPeriodEndValueChild, theValue); if (start != null || end != null) { - myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(theResourceType, theSearchParam.getName(), start, end, startAsString); + myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), start, end, startAsString); theParams.add(myIndexedSearchParamDate); } } @@ -408,7 +408,7 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } if (!dates.isEmpty()) { - myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(theResourceType, theSearchParam.getName(), dates.first(), dates.last(), firstValue); + myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), dates.first(), dates.last(), firstValue); theParams.add(myIndexedSearchParamDate); } } From 4db5beeabf0094ae3ad242dfd0c2ae8369713b23 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 5 May 2020 16:53:47 -0400 Subject: [PATCH 13/31] Additional changes to enable processing of lastn operations with large numbers of parameters. --- .../BaseHapiFhirResourceDaoObservation.java | 2 +- .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 61 +++++++- .../FhirResourceDaoObservationDstu3.java | 2 + .../dao/r5/FhirResourceDaoObservationR5.java | 2 + ...seJpaResourceProviderObservationDstu3.java | 96 +++---------- .../BaseJpaResourceProviderObservationR4.java | 87 +++-------- .../BaseJpaResourceProviderObservationR5.java | 88 +++--------- .../search/lastn/ElasticsearchSvcImpl.java | 135 +++++++++++++++--- .../r4/FhirResourceDaoR4SearchLastNIT.java | 64 ++++++++- ...bservationIndexedSearchParamLastNR4IT.java | 12 ++ ...lasticsearchSvcMultipleObservationsIT.java | 48 +++++-- .../lastn/config/TestElasticsearchConfig.java | 8 +- .../jpa/searchparam/SearchParameterMap.java | 19 +++ 13 files changed, 367 insertions(+), 257 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java index f0755c825b2..5f9239d208c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java @@ -38,7 +38,7 @@ public abstract class BaseHapiFhirResourceDaoObservation 0) { - myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]); +// String[] maxCountParams = theRequest.getParameters().get("max"); +// if (maxCountParams != null && maxCountParams.length > 0) { +// myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]); + if(myParams.getLastNMax() != null) { + myMaxObservationsPerCode = myParams.getLastNMax(); } else { throw new InvalidRequestException("Max parameter is required for $lastn operation"); } List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); - for (String lastnResourceId : lastnResourceIds) { - pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); - } +// for (String lastnResourceId : lastnResourceIds) { +// pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); +// } + pids = normalizeIdListForLastNInClause(lastnResourceIds); } if (pids.isEmpty()) { // Will never match @@ -411,6 +414,50 @@ public class SearchBuilder implements ISearchBuilder { return query; } + private List normalizeIdListForLastNInClause(List lastnResourceIds) { + List retVal = new ArrayList<>(); + for (String lastnResourceId : lastnResourceIds) { + retVal.add(new ResourcePersistentId(Long.parseLong(lastnResourceId))); + } + + /* + The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying + numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info: + https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage. + + Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of + arguments never exceeds the maximum specified below. + */ + int listSize = retVal.size(); + if(listSize > 1 && listSize < 10) { + padIdListWithPlaceholders(retVal, 10); + } else if (listSize > 10 && listSize < 100) { + padIdListWithPlaceholders(retVal, 100); + } else if (listSize > 100 && listSize < 200) { + padIdListWithPlaceholders(retVal, 200); + } else if (listSize > 200 && listSize < 500) { + padIdListWithPlaceholders(retVal, 500); + } else if (listSize > 500 && listSize < 1000) { + padIdListWithPlaceholders(retVal, 1000); + } else if (listSize > 1000 && listSize < 500) { + padIdListWithPlaceholders(retVal, 5000); + } else if (listSize > 5000 && listSize < 10000) { + padIdListWithPlaceholders(retVal, 10000); + } else if (listSize > 10000 && listSize < 20000) { + padIdListWithPlaceholders(retVal, 20000); + } else if (listSize > 20000 && listSize < 30000) { + padIdListWithPlaceholders(retVal, 30000); + } + + return retVal; + } + + private void padIdListWithPlaceholders(List theIdList, int preferredListSize) { + while(theIdList.size() < preferredListSize) { + theIdList.add(new ResourcePersistentId(-1L)); + } + } + /** * @return Returns {@literal true} if any search parameter sorts were found, or false if * no sorts were found, or only non-search parameters ones (e.g. _id, _lastUpdated) @@ -1190,7 +1237,7 @@ public class SearchBuilder implements ISearchBuilder { mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); - Query hibernateQuery = (Query) query; + Query hibernateQuery = (Query) query; hibernateQuery.setFetchSize(myFetchSize); ScrollableResults scroll = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY); myResultsIterator = new ScrollableResultsIterator<>(scroll); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java index dc4e7278b12..db7a0cd47bf 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java @@ -29,6 +29,8 @@ import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.param.ReferenceAndListParam; +import ca.uhn.fhir.rest.param.TokenAndListParam; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.dstu3.model.Observation; import org.springframework.beans.factory.annotation.Autowired; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java index 9d85c24f881..63d8333e763 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java @@ -29,6 +29,8 @@ import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.param.ReferenceAndListParam; +import ca.uhn.fhir.rest.param.TokenAndListParam; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r5.model.Observation; import org.springframework.beans.factory.annotation.Autowired; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java index 039ee2059c3..a70b7a61d89 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java @@ -3,19 +3,17 @@ package ca.uhn.fhir.jpa.provider.dstu3; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.model.api.annotation.Description; import ca.uhn.fhir.model.valueset.BundleTypeEnum; import ca.uhn.fhir.rest.annotation.*; -import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; -import ca.uhn.fhir.rest.api.SummaryEnum; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.param.*; import org.hl7.fhir.dstu3.model.Observation; - -import java.util.Set; +import org.hl7.fhir.dstu3.model.UnsignedIntType; +import org.hl7.fhir.instance.model.api.IPrimitiveType; /* * #%L @@ -50,17 +48,9 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails, - @Description(shortDefinition="Search the contents of the resource's data using a filter") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_FILTER) - StringAndListParam theFtFilter, - - @Description(shortDefinition="Search the contents of the resource's data using a fulltext search") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT) - StringAndListParam theFtContent, - - @Description(shortDefinition="Search the contents of the resource's narrative using a fulltext search") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_TEXT) - StringAndListParam theFtText, + @Description(formalDefinition = "Results from this method are returned across multiple pages. This parameter controls the size of those pages.") + @OperationParam(name = Constants.PARAM_COUNT) + UnsignedIntType theCount, @Description(shortDefinition="The classification of the type of observation") @OperationParam(name="category") @@ -70,10 +60,6 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider @OperationParam(name="code") TokenAndListParam theCode, - @Description(shortDefinition="Obtained date/time. If the obtained element is a period, a date that falls in the period") - @OperationParam(name="date") - DateRangeParam theDate, - @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -82,69 +68,31 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider @OperationParam(name="subject" ) ReferenceAndListParam theSubject, - @IncludeParam(reverse=true) - Set theRevIncludes, - @Description(shortDefinition="Only return resources which were last updated as specified by the given range") - @OperationParam(name="_lastUpdated") - DateRangeParam theLastUpdated, - - @IncludeParam(allow= { - "Observation:based-on", - "Observation:derived-from", - "Observation:device", - "Observation:encounter", - "Observation:focus", - "Observation:has-member", - "Observation:part-of", - "Observation:patient", - "Observation:performer", - "Observation:specimen", - "Observation:subject", - "*" - }) - Set theIncludes, + @Description(shortDefinition="The maximum number of observations to return for each observation code") + @OperationParam(name = "max", typeName = "integer", min = 0, max = 1) + IPrimitiveType theMax, @Sort - SortSpec theSort, - - @ca.uhn.fhir.rest.annotation.Count - Integer theCount, - - SummaryEnum theSummaryMode, - - SearchTotalModeEnum theSearchTotalMode + SortSpec theSort ) { startRequest(theServletRequest); try { SearchParameterMap paramMap = new SearchParameterMap(); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_FILTER, theFtFilter); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, theFtContent); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_TEXT, theFtText); - paramMap.add("category", theCategory); - paramMap.add("code", theCode); - paramMap.add("date", theDate); - paramMap.add("patient", thePatient); - paramMap.add("subject", theSubject); - paramMap.setRevIncludes(theRevIncludes); - paramMap.setLastUpdated(theLastUpdated); - paramMap.setIncludes(theIncludes); - paramMap.setLastN(true); - if (theSort == null) { - SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); - SortSpec observationCode = new SortSpec("code").setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); - if (thePatient != null && theSubject == null) { - theSort = new SortSpec("patient").setChain(observationCode); - } else { - theSort = new SortSpec("subject").setChain(observationCode); - } + paramMap.add(Observation.SP_CATEGORY, theCategory); + paramMap.add(Observation.SP_CODE, theCode); + paramMap.add(Observation.SP_PATIENT, thePatient); + paramMap.add(Observation.SP_SUBJECT, theSubject); + paramMap.setLastNMax(theMax.getValue()); + if (theCount != null) { + paramMap.setCount(theCount.getValue()); } - paramMap.setSort(theSort); - paramMap.setCount(theCount); - paramMap.setSummaryMode(theSummaryMode); - paramMap.setSearchTotalMode(theSearchTotalMode); - return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); + if (theSort != null) { + paramMap.setSort(theSort); + } + + return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); } finally { endRequest(theServletRequest); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java index a2bdac6f7b4..a4b81125af9 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java @@ -3,20 +3,17 @@ package ca.uhn.fhir.jpa.provider.r4; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.model.api.annotation.Description; import ca.uhn.fhir.model.valueset.BundleTypeEnum; import ca.uhn.fhir.rest.annotation.*; -import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; -import ca.uhn.fhir.rest.api.SummaryEnum; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.param.*; +import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.hl7.fhir.r4.model.*; -import java.util.Set; - /* * #%L * HAPI FHIR JPA Server @@ -50,17 +47,9 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails, - @Description(shortDefinition="Search the contents of the resource's data using a filter") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_FILTER) - StringAndListParam theFtFilter, - - @Description(shortDefinition="Search the contents of the resource's data using a fulltext search") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT) - StringAndListParam theFtContent, - - @Description(shortDefinition="Search the contents of the resource's narrative using a fulltext search") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_TEXT) - StringAndListParam theFtText, + @Description(formalDefinition = "Results from this method are returned across multiple pages. This parameter controls the size of those pages.") + @OperationParam(name = Constants.PARAM_COUNT) + UnsignedIntType theCount, @Description(shortDefinition="The classification of the type of observation") @OperationParam(name="category") @@ -70,10 +59,6 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< @OperationParam(name="code") TokenAndListParam theCode, - @Description(shortDefinition="Obtained date/time. If the obtained element is a period, a date that falls in the period") - @OperationParam(name="date") - DateRangeParam theDate, - @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -82,67 +67,29 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< @OperationParam(name="subject" ) ReferenceAndListParam theSubject, - @IncludeParam(reverse=true) - Set theRevIncludes, - @Description(shortDefinition="Only return resources which were last updated as specified by the given range") - @OperationParam(name="_lastUpdated") - DateRangeParam theLastUpdated, - - @IncludeParam(allow= { - "Observation:based-on", - "Observation:derived-from", - "Observation:device", - "Observation:encounter", - "Observation:focus", - "Observation:has-member", - "Observation:part-of", - "Observation:patient", - "Observation:performer", - "Observation:specimen", - "Observation:subject", - "*" - }) - Set theIncludes, + @Description(shortDefinition="The maximum number of observations to return for each observation code") + @OperationParam(name = "max", typeName = "integer", min = 0, max = 1) + IPrimitiveType theMax, @Sort - SortSpec theSort, - - @ca.uhn.fhir.rest.annotation.Count - Integer theCount, - - SummaryEnum theSummaryMode, - - SearchTotalModeEnum theSearchTotalMode + SortSpec theSort ) { startRequest(theServletRequest); try { SearchParameterMap paramMap = new SearchParameterMap(); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_FILTER, theFtFilter); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, theFtContent); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_TEXT, theFtText); paramMap.add("category", theCategory); paramMap.add("code", theCode); - paramMap.add("date", theDate); paramMap.add("patient", thePatient); paramMap.add("subject", theSubject); - paramMap.setRevIncludes(theRevIncludes); - paramMap.setLastUpdated(theLastUpdated); - paramMap.setIncludes(theIncludes); -/* paramMap.setLastN(true); - if (theSort == null) { - SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); - SortSpec observationCode = new SortSpec("code").setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); - if (thePatient != null && theSubject == null) { - theSort = new SortSpec("patient").setChain(observationCode); - } else { - theSort = new SortSpec("subject").setChain(observationCode); - } - } */ - paramMap.setSort(theSort); - paramMap.setCount(theCount); - paramMap.setSummaryMode(theSummaryMode); - paramMap.setSearchTotalMode(theSearchTotalMode); + paramMap.setLastNMax(theMax.getValue()); + if (theCount != null) { + paramMap.setCount(theCount.getValue()); + } + + if (theSort != null) { + paramMap.setSort(theSort); + } return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); } finally { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java index 53f75ba210b..95e80a775ed 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java @@ -3,20 +3,18 @@ package ca.uhn.fhir.jpa.provider.r5; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.model.api.annotation.Description; import ca.uhn.fhir.model.valueset.BundleTypeEnum; import ca.uhn.fhir.rest.annotation.*; -import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; -import ca.uhn.fhir.rest.api.SummaryEnum; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.param.*; +import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.hl7.fhir.r5.model.UnsignedIntType; import org.hl7.fhir.r5.model.Observation; -import java.util.Set; - /* * #%L * HAPI FHIR JPA Server @@ -50,17 +48,9 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails, - @Description(shortDefinition="Search the contents of the resource's data using a filter") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_FILTER) - StringAndListParam theFtFilter, - - @Description(shortDefinition="Search the contents of the resource's data using a fulltext search") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT) - StringAndListParam theFtContent, - - @Description(shortDefinition="Search the contents of the resource's narrative using a fulltext search") - @OperationParam(name=ca.uhn.fhir.rest.api.Constants.PARAM_TEXT) - StringAndListParam theFtText, + @Description(formalDefinition = "Results from this method are returned across multiple pages. This parameter controls the size of those pages.") + @OperationParam(name = Constants.PARAM_COUNT) + UnsignedIntType theCount, @Description(shortDefinition="The classification of the type of observation") @OperationParam(name="category") @@ -70,10 +60,6 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< @OperationParam(name="code") TokenAndListParam theCode, - @Description(shortDefinition="Obtained date/time. If the obtained element is a period, a date that falls in the period") - @OperationParam(name="date") - DateRangeParam theDate, - @Description(shortDefinition="The subject that the observation is about (if patient)") @OperationParam(name="patient") ReferenceAndListParam thePatient, @@ -82,69 +68,31 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< @OperationParam(name="subject" ) ReferenceAndListParam theSubject, - @IncludeParam(reverse=true) - Set theRevIncludes, - @Description(shortDefinition="Only return resources which were last updated as specified by the given range") - @OperationParam(name="_lastUpdated") - DateRangeParam theLastUpdated, - - @IncludeParam(allow= { - "Observation:based-on", - "Observation:derived-from", - "Observation:device", - "Observation:encounter", - "Observation:focus", - "Observation:has-member", - "Observation:part-of", - "Observation:patient", - "Observation:performer", - "Observation:specimen", - "Observation:subject", - "*" - }) - Set theIncludes, + @Description(shortDefinition="The maximum number of observations to return for each observation code") + @OperationParam(name = "max", typeName = "integer", min = 0, max = 1) + IPrimitiveType theMax, @Sort - SortSpec theSort, - - @ca.uhn.fhir.rest.annotation.Count - Integer theCount, - - SummaryEnum theSummaryMode, - - SearchTotalModeEnum theSearchTotalMode + SortSpec theSort ) { startRequest(theServletRequest); try { SearchParameterMap paramMap = new SearchParameterMap(); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_FILTER, theFtFilter); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, theFtContent); - paramMap.add(ca.uhn.fhir.rest.api.Constants.PARAM_TEXT, theFtText); paramMap.add("category", theCategory); paramMap.add("code", theCode); - paramMap.add("date", theDate); paramMap.add("patient", thePatient); paramMap.add("subject", theSubject); - paramMap.setRevIncludes(theRevIncludes); - paramMap.setLastUpdated(theLastUpdated); - paramMap.setIncludes(theIncludes); - paramMap.setLastN(true); - if (theSort == null) { - SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); - SortSpec observationCode = new SortSpec("code").setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); - if (thePatient != null && theSubject == null) { - theSort = new SortSpec("patient").setChain(observationCode); - } else { - theSort = new SortSpec("subject").setChain(observationCode); - } + paramMap.setLastNMax(theMax.getValue()); + if (theCount != null) { + paramMap.setCount(theCount.getValue()); } - paramMap.setSort(theSort); - paramMap.setCount(theCount); - paramMap.setSummaryMode(theSummaryMode); - paramMap.setSearchTotalMode(theSearchTotalMode); - return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); + if (theSort != null) { + paramMap.setSort(theSort); + } + + return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); } finally { endRequest(theServletRequest); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 52f2ab09f79..9a730b6bb69 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.search.lastn; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.IQueryParameterType; +import ca.uhn.fhir.model.dstu2.resource.Observation; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; @@ -42,6 +43,7 @@ import org.shadehapi.elasticsearch.search.sort.SortOrder; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -200,21 +202,64 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { @Override public List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; - SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(theMaxObservationsPerCode, topHitsInclude)); try { - SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); - return buildObservationIdList(lastnResponse); + List responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, topHitsInclude); + List observationIds = new ArrayList<>(); + for (SearchResponse response : responses) { +// observationIds.addAll(buildObservationIdList(response)); + observationIds.addAll(buildObservationList(response, t -> t.getIdentifier(), theSearchParameterMap)); + } + return observationIds; } catch (IOException theE) { throw new InvalidRequestException("Unable to execute LastN request", theE); } } + private List buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, String[] topHitsInclude) { + List responses = new ArrayList<>(); + if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { + ArrayList subjectReferenceCriteria = new ArrayList<>(); + List> patientParams = new ArrayList<>(); + if (theSearchParameterMap.get(IndexConstants.PATIENT_SEARCH_PARAM) != null) { + patientParams.addAll(theSearchParameterMap.get(IndexConstants.PATIENT_SEARCH_PARAM)); + } + if (theSearchParameterMap.get(IndexConstants.SUBJECT_SEARCH_PARAM) != null) { + patientParams.addAll(theSearchParameterMap.get(IndexConstants.SUBJECT_SEARCH_PARAM)); + } + for (List nextSubjectList : patientParams) { + subjectReferenceCriteria.addAll(getReferenceValues(nextSubjectList)); + } + for (String subject : subjectReferenceCriteria) { + SearchRequest myLastNRequest = buildObservationsSearchRequest(subject, theSearchParameterMap, createCompositeAggregationBuilder(theMaxObservationsPerCode, topHitsInclude)); + try { + SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); + responses.add(lastnResponse); + } catch (IOException theE) { + throw new InvalidRequestException("Unable to execute LastN request", theE); + } + } + } else { + SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, createObservationCodeAggregationBuilder(theMaxObservationsPerCode, topHitsInclude)); + try { + SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); + responses.add(lastnResponse); + } catch (IOException theE) { + throw new InvalidRequestException("Unable to execute LastN request", theE); + } + + } + return responses; + } + @VisibleForTesting List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { - SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(theMaxObservationsPerCode, null)); try { - SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); - return buildObservationDocumentList(lastnResponse); + List responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, null); + List observationDocuments = new ArrayList<>(); + for (SearchResponse response : responses) { + observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap)); + } + return observationDocuments; } catch (IOException theE) { throw new InvalidRequestException("Unable to execute LastN request", theE); } @@ -227,8 +272,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return buildCodeResult(codeSearchResponse); } - @VisibleForTesting - SearchRequest buildObservationCodesSearchRequest(int theMaxResultSetSize) { + private SearchRequest buildObservationCodesSearchRequest(int theMaxResultSetSize) { SearchRequest searchRequest = new SearchRequest(IndexConstants.CODE_INDEX); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // Query @@ -239,27 +283,30 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } private CompositeAggregationBuilder createCompositeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { + CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); + List> compositeAggSubjectSources = new ArrayList(); + compositeAggSubjectSources.add(subjectValuesBuilder); + CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder(GROUP_BY_SUBJECT, compositeAggSubjectSources); + compositeAggregationSubjectBuilder.subAggregation(createObservationCodeAggregationBuilder(theMaxNumberObservationsPerCode, theTopHitsInclude)); + compositeAggregationSubjectBuilder.size(10000); + + return compositeAggregationSubjectBuilder; + } + + private TermsAggregationBuilder createObservationCodeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { TermsAggregationBuilder observationCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE, ValueType.STRING).field("codeconceptid"); // Top Hits Aggregation observationCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") .sort("effectivedtm", SortOrder.DESC) .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); observationCodeAggregationBuilder.size(10000); - CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); - List> compositeAggSubjectSources = new ArrayList(); - compositeAggSubjectSources.add(subjectValuesBuilder); - CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder(GROUP_BY_SUBJECT, compositeAggSubjectSources); - compositeAggregationSubjectBuilder.subAggregation(observationCodeAggregationBuilder); - compositeAggregationSubjectBuilder.size(10000); - - return compositeAggregationSubjectBuilder; + return observationCodeAggregationBuilder; } - private SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { + public SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); } - private List buildObservationIdList(SearchResponse theSearchResponse) throws IOException { List theObservationList = new ArrayList<>(); for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { @@ -288,12 +335,43 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return theObservationList; } + private List buildObservationList(SearchResponse theSearchResponse, Function setValue, SearchParameterMap theSearchParameterMap) throws IOException { + List theObservationList = new ArrayList<>(); + if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { + for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { + for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) { + for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { + String indexedObservation = lastNMatch.getSourceAsString(); + ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); + theObservationList.add(setValue.apply(observationJson)); + } + } + } + } else { + for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(theSearchResponse)) { + for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { + String indexedObservation = lastNMatch.getSourceAsString(); + ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); + theObservationList.add(setValue.apply(observationJson)); + } + } + } + + return theObservationList; + } + private List getSubjectBuckets(SearchResponse theSearchResponse) { Aggregations responseAggregations = theSearchResponse.getAggregations(); ParsedComposite aggregatedSubjects = responseAggregations.get(GROUP_BY_SUBJECT); return aggregatedSubjects.getBuckets(); } + private List getObservationCodeBuckets(SearchResponse theSearchResponse) { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedTerms aggregatedObservationCodes = responseAggregations.get(GROUP_BY_CODE); + return aggregatedObservationCodes.getBuckets(); + } + private List getObservationCodeBuckets(ParsedComposite.ParsedBucket theSubjectBucket) { Aggregations observationCodeAggregations = theSubjectBucket.getAggregations(); ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get(GROUP_BY_CODE); @@ -338,6 +416,24 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return searchRequest; } + private SearchRequest buildObservationsSearchRequest(String theSubjectParam, SearchParameterMap theSearchParameterMap, AggregationBuilder theAggregationBuilder) { + SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + // Query + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(QueryBuilders.termQuery("subject", theSubjectParam)); + addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap); + addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap); + searchSourceBuilder.query(boolQueryBuilder); + searchSourceBuilder.size(0); + + // Aggregation by order codes + searchSourceBuilder.aggregation(theAggregationBuilder); + searchRequest.source(searchSourceBuilder); + + return searchRequest; + } + private Boolean searchParamsHaveLastNCriteria(SearchParameterMap theSearchParameterMap) { return theSearchParameterMap != null && (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM) || @@ -519,11 +615,12 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); } - public void deleteObservationIndex(String theObservationIdentifier) throws IOException { +/* public void deleteObservationIndex(String theObservationIdentifier) throws IOException { DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(IndexConstants.OBSERVATION_DOCUMENT_TYPE); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_IDENTIFIER_FIELD_NAME, theObservationIdentifier)); deleteByQueryRequest.setQuery(boolQueryBuilder); myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); } + */ } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java index 58f16a68383..e5b40a53e56 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java @@ -5,6 +5,7 @@ import ca.uhn.fhir.jpa.api.dao.*; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; import ca.uhn.fhir.jpa.dao.BaseJpaTest; +import ca.uhn.fhir.jpa.rp.r4.ObservationResourceProvider; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.rest.param.*; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; @@ -57,6 +58,8 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { return myPlatformTransactionManager; } + ObservationResourceProvider observationRp = new ObservationResourceProvider(); + private final String observationCd0 = "code0"; private final String observationCd1 = "code1"; private final String observationCd2 = "code2"; @@ -101,6 +104,8 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { dataLoaded = true; } + observationRp.setDao(myObservationDao); + } private void createObservationsForPatient(IIdType thePatientId) { @@ -138,9 +143,13 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { } @Test - public void testLastNNoParams() { + public void testLastNAllPatients() { SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); List sortedPatients = new ArrayList<>(); sortedPatients.add(patient0Id.getValue()); @@ -155,14 +164,39 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { executeTestCase(params, sortedPatients, sortedObservationCodes, null,90); } + @Test + public void testLastNNoPatients() { + + SearchParameterMap params = new SearchParameterMap(); + params.setLastNMax(1); + + List sortedPatients = new ArrayList<>(); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + +// executeTestCase(params, sortedPatients, sortedObservationCodes, null,3); + params.setLastN(true); + Map requestParameters = new HashMap<>(); + when(mySrd.getParameters()).thenReturn(requestParameters); + + List actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + + assertEquals(3, actual.size()); + } + private void executeTestCase(SearchParameterMap params, List sortedPatients, List sortedObservationCodes, List theCategories, int expectedObservationCount) { List actual; params.setLastN(true); Map requestParameters = new HashMap<>(); - String[] maxParam = new String[1]; - maxParam[0] = "100"; - requestParameters.put("max", maxParam); +// String[] maxParam = new String[1]; +// maxParam[0] = "100"; +// requestParameters.put("max", maxParam); + params.setLastNMax(100); + when(mySrd.getParameters()).thenReturn(requestParameters); actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); @@ -297,6 +331,11 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { // One category parameter. SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0); params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); List myCategories = new ArrayList<>(); @@ -315,6 +354,7 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { // Another category parameter. params = new SearchParameterMap(); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); categoryParam = new TokenParam(categorySystem, categoryCd2); params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); myCategories = new ArrayList<>(); @@ -333,6 +373,11 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { // Two category parameters. SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd0); TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd1); params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); @@ -357,6 +402,11 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { // One code parameter. SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + TokenParam code = new TokenParam(codeSystem, observationCd0); params.add(Observation.SP_CODE, buildTokenAndListParam(code)); List sortedObservationCodes = new ArrayList<>(); @@ -371,6 +421,7 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { // Another code parameter. params = new SearchParameterMap(); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); code = new TokenParam(codeSystem, observationCd2); params.add(Observation.SP_CODE, buildTokenAndListParam(code)); sortedObservationCodes = new ArrayList<>(); @@ -385,6 +436,11 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { // Two code parameters. SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1); params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java index 9a552c09a0d..944dae05e7d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java @@ -79,6 +79,8 @@ public class PersistObservationIndexedSearchParamLastNR4IT { private final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; private final String CODEFIRSTCODINGCODE = "test-code"; + private ReferenceAndListParam multiSubjectParams = null; + @Test public void testIndexObservationSingle() { indexSingleObservation(); @@ -177,6 +179,8 @@ public class PersistObservationIndexedSearchParamLastNR4IT { // Check that all observations were indexed. SearchParameterMap searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams); + //searchParameterMap. List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); assertEquals(100, observationIdsOnly.size()); @@ -223,10 +227,14 @@ public class PersistObservationIndexedSearchParamLastNR4IT { categoryCodeableConcept2.setCoding(category2); categoryConcepts2.add(categoryCodeableConcept2); + ReferenceOrListParam subjectParams = new ReferenceOrListParam(); for (int patientCount = 0; patientCount < 10; patientCount++) { String subjectId = String.valueOf(patientCount); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", subjectId); + subjectParams.addOr(subjectParam); + for (int entryCount = 0; entryCount < 10; entryCount++) { Observation observation = new Observation(); @@ -254,6 +262,9 @@ public class PersistObservationIndexedSearchParamLastNR4IT { } } + + multiSubjectParams = new ReferenceAndListParam().addAnd(subjectParams); + } @Test @@ -265,6 +276,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { assertNotNull(observation); SearchParameterMap searchParameterMap = new SearchParameterMap(); + searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams); List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); assertEquals(100, observationIdsOnly.size()); assertTrue(observationIdsOnly.contains("55")); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index 7eb78281ec3..21a7e08b8dc 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -56,23 +56,40 @@ public class LastNElasticsearchSvcMultipleObservationsIT { } @Test - public void testLastNNoCriteriaQuery() { + public void testLastNAllPatientsQuery() { // execute Observation ID search (Composite Aggregation) last 3 observations for each patient - List observations = elasticsearchSvc.executeLastNWithAllFields(null, 3); + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "0"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + subjectParam = new ReferenceParam("Patient", "", "1"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + subjectParam = new ReferenceParam("Patient", "", "2"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + subjectParam = new ReferenceParam("Patient", "", "4"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + subjectParam = new ReferenceParam("Patient", "", "5"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + subjectParam = new ReferenceParam("Patient", "", "6"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + subjectParam = new ReferenceParam("Patient", "", "7"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + subjectParam = new ReferenceParam("Patient", "", "8"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + subjectParam = new ReferenceParam("Patient", "", "9"); + searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - validateQueryResponse(observations); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3); - } - - private void validateQueryResponse(List observationIdsOnly) { - assertEquals(60, observationIdsOnly.size()); + assertEquals(60, observations.size()); // Observation documents should be grouped by subject, then by observation code, and then sorted by effective date/time // within each observation code. Verify the grouping by creating a nested Map. Map>> queriedPatientObservationMap = new HashMap<>(); ObservationJson previousObservationJson = null; - for (ObservationJson observationJson : observationIdsOnly) { + for (ObservationJson observationJson : observations) { assertNotNull(observationJson.getIdentifier()); assertNotNull(observationJson.getSubject()); assertNotNull(observationJson.getCode_concept_id()); @@ -373,4 +390,19 @@ public class LastNElasticsearchSvcMultipleObservationsIT { } + @Test + public void testLastNNoParamsQuery() { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 1); + + assertEquals(2, observations.size()); + + String observationCode1 = observations.get(0).getCode_coding_code_system_hash(); + String observationCode2 = observations.get(1).getCode_coding_code_system_hash(); + + assertNotEquals(observationCode1, observationCode2); + + } + + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java index 1e5116e3f7d..624a05d38b4 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java @@ -47,9 +47,9 @@ public class TestElasticsearchConfig { return embeddedElastic; } - @PreDestroy - public void stop() { - embeddedElasticSearch().stop(); - } +// @PreDestroy +// public void stop() { +// embeddedElasticSearch().stop(); +// } } diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java index becf093ce6a..a9c6b340c48 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java @@ -62,6 +62,7 @@ public class SearchParameterMap implements Serializable { private SearchTotalModeEnum mySearchTotalMode; private QuantityParam myNearDistanceParam; private boolean myLastN; + private Integer myLastNMax; /** * Constructor @@ -322,6 +323,24 @@ public class SearchParameterMap implements Serializable { } + /** + * If set, tells the server the maximum number of observations to return for each + * observation code in the result set of a lastn operation + */ + public Integer getLastNMax() { + return myLastNMax; + } + + /** + * If set, tells the server the maximum number of observations to return for each + * observation code in the result set of a lastn operation + */ + public SearchParameterMap setLastNMax(Integer theLastNMax) { + myLastNMax = theLastNMax; + return this; + } + + /** * This method creates a URL query string representation of the parameters in this * object, excluding the part before the parameters, e.g. From 1e882d640d8125ecab399b37ad7fe4bb0623f3f4 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 5 May 2020 17:35:22 -0400 Subject: [PATCH 14/31] Merging with latest from Master. --- .../extractor/BaseSearchParamExtractor.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java index 71140dcdda6..da9e4354e6c 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java @@ -44,7 +44,6 @@ import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry; import ca.uhn.fhir.model.primitive.BoundCodeDt; import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.Validate; import org.hibernate.search.spatial.impl.Point; @@ -375,9 +374,10 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor Date start = extractValueAsDate(myPeriodStartValueChild, theValue); String startAsString = extractValueAsString(myPeriodStartValueChild, theValue); Date end = extractValueAsDate(myPeriodEndValueChild, theValue); + String endAsString = extractValueAsString(myPeriodEndValueChild, theValue); if (start != null || end != null) { - myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), start, end, startAsString); + myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), start, startAsString, end, endAsString, startAsString); theParams.add(myIndexedSearchParamDate); } } @@ -387,12 +387,14 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor TreeSet dates = new TreeSet<>(); String firstValue = null; + String finalValue = null; for (IPrimitiveType nextEvent : values) { if (nextEvent.getValue() != null) { dates.add(nextEvent.getValue()); if (firstValue == null) { firstValue = nextEvent.getValueAsString(); } + finalValue = nextEvent.getValueAsString(); } } @@ -411,7 +413,7 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } if (!dates.isEmpty()) { - myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), dates.first(), dates.last(), firstValue); + myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), dates.first(), firstValue, dates.last(), finalValue, firstValue); theParams.add(myIndexedSearchParamDate); } } @@ -420,7 +422,7 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor private void addDateTimeTypes(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { IPrimitiveType nextBaseDateTime = (IPrimitiveType) theValue; if (nextBaseDateTime.getValue() != null) { - myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), nextBaseDateTime.getValue(), nextBaseDateTime.getValue(), nextBaseDateTime.getValueAsString()); + myIndexedSearchParamDate = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), nextBaseDateTime.getValue(), nextBaseDateTime.getValueAsString(), nextBaseDateTime.getValue(), nextBaseDateTime.getValueAsString(), nextBaseDateTime.getValueAsString()); theParams.add(myIndexedSearchParamDate); } } From c290fa3493d61112601bfeab0bd3fddb005a8406 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Fri, 8 May 2020 09:19:14 -0400 Subject: [PATCH 15/31] Fixes to enable $lastn to return more than 32K records. --- DeleteConflictService_fix.patch | 15 ++ .../BaseHapiFhirResourceDaoObservation.java | 1 + .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 173 ++++++++++++++---- ...ervationIndexedSearchParamLastNEntity.java | 111 +++++------ .../search/lastn/ElasticsearchSvcImpl.java | 2 + .../fhir/jpa/search/lastn/IndexConstants.java | 1 + .../ca/uhn/fhir/jpa/util/QueryChunker.java | 4 +- .../r4/FhirResourceDaoR4SearchLastNIT.java | 92 +++++++++- .../util/LastNParameterHelper.java | 38 ++++ 9 files changed, 340 insertions(+), 97 deletions(-) create mode 100644 DeleteConflictService_fix.patch create mode 100644 hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java diff --git a/DeleteConflictService_fix.patch b/DeleteConflictService_fix.patch new file mode 100644 index 00000000000..c88dbd45c52 --- /dev/null +++ b/DeleteConflictService_fix.patch @@ -0,0 +1,15 @@ +diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java +index e575041cd9..93e364bc93 100644 +--- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java ++++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java +@@ -49,8 +49,8 @@ import java.util.List; + public class DeleteConflictService { + private static final Logger ourLog = LoggerFactory.getLogger(DeleteConflictService.class); + public static final int FIRST_QUERY_RESULT_COUNT = 1; +- public static final int RETRY_QUERY_RESULT_COUNT = 60; +- public static final int MAX_RETRY_ATTEMPTS = 10; ++ public static final int RETRY_QUERY_RESULT_COUNT = 100; ++ public static final int MAX_RETRY_ATTEMPTS = 100; + + @Autowired + DeleteConflictFinderService myDeleteConflictFinderService; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java index 5f9239d208c..391e34db536 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java @@ -37,6 +37,7 @@ public abstract class BaseHapiFhirResourceDaoObservation EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<>()); private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class); @@ -182,6 +184,18 @@ public class SearchBuilder implements ISearchBuilder { myResourceType = theResourceType; } + public static int getMaximumPageSize() { + if (myIsTest) { + return MAXIMUM_PAGE_SIZE_FOR_TESTING; + } else { + return MAXIMUM_PAGE_SIZE; + } + } + + public static void setIsTest(boolean theIsTest) { + myIsTest = theIsTest; + } + @Override public void setMaxResultsToFetch(Integer theMaxResultsToFetch) { myMaxResultsToFetch = theMaxResultsToFetch; @@ -210,6 +224,10 @@ public class SearchBuilder implements ISearchBuilder { // Handle each parameter for (Map.Entry>> nextParamEntry : myParams.entrySet()) { String nextParamName = nextParamEntry.getKey(); + if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) { + // Skip parameters for Subject, Patient, Code and Category for LastN + continue; + } List> andOrParams = nextParamEntry.getValue(); searchForIdsWithAndOr(myResourceName, nextParamName, andOrParams, theRequest); } @@ -231,8 +249,8 @@ public class SearchBuilder implements ISearchBuilder { init(theParams, theSearchUuid, theRequestPartitionId); - TypedQuery query = createQuery(null, null, true, theRequest); - return new CountQueryIterator(query); + List> queries = createQuery(null, null, true, theRequest); + return new CountQueryIterator(queries.get(0)); } /** @@ -265,7 +283,72 @@ public class SearchBuilder implements ISearchBuilder { } - private TypedQuery createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) { + private List> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) { + List pids = new ArrayList<>(); + + /* + * Fulltext or lastn search + */ + if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { + if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { + if (myFulltextSearchSvc == null) { + if (myParams.containsKey(Constants.PARAM_TEXT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + } + } + + if (myParams.getEverythingMode() != null) { + pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); + } else { + pids = myFulltextSearchSvc.search(myResourceName, myParams); + } + } else if (myParams.isLastN()) { + if (myIElasticsearchSvc == null) { + if (myParams.isLastN()) { + throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); + } + } + Integer myMaxObservationsPerCode = null; + if(myParams.getLastNMax() != null) { + myMaxObservationsPerCode = myParams.getLastNMax(); + } else { + throw new InvalidRequestException("Max parameter is required for $lastn operation"); + } + List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); + for (String lastnResourceId : lastnResourceIds) { + pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); + } + } + if (pids.isEmpty()) { + // Will never match + pids = Collections.singletonList(new ResourcePersistentId(-1L)); + } + + } + + ArrayList> myQueries = new ArrayList<>(); + + if (!pids.isEmpty()) { + new QueryChunker().chunk(ResourcePersistentId.toLongList(pids), t->{ + doCreateChunkedQueries(t, sort, theMaximumResults, theCount, theRequest, myQueries); + }); + } else { + myQueries.add(createQuery(sort,theMaximumResults, theCount, theRequest, null)); + } + + return myQueries; + } + + private void doCreateChunkedQueries(List thePids, SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, ArrayList> theQueries) { + if(thePids.size() < MAXIMUM_PAGE_SIZE) { + thePids = normalizeIdListForLastNInClause(thePids); + } + theQueries.add(createQuery(sort, theMaximumResults, theCount, theRequest, thePids)); + } + + private TypedQuery createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List thePidList) { CriteriaQuery outerQuery; /* * Sort @@ -329,7 +412,7 @@ public class SearchBuilder implements ISearchBuilder { /* * Fulltext or lastn search */ - if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { +/* if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { List pids = new ArrayList<>(); if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { if (myFulltextSearchSvc == null) { @@ -352,19 +435,16 @@ public class SearchBuilder implements ISearchBuilder { } } Integer myMaxObservationsPerCode = null; -// String[] maxCountParams = theRequest.getParameters().get("max"); -// if (maxCountParams != null && maxCountParams.length > 0) { -// myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]); if(myParams.getLastNMax() != null) { myMaxObservationsPerCode = myParams.getLastNMax(); } else { throw new InvalidRequestException("Max parameter is required for $lastn operation"); } List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); -// for (String lastnResourceId : lastnResourceIds) { -// pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); -// } - pids = normalizeIdListForLastNInClause(lastnResourceIds); + for (String lastnResourceId : lastnResourceIds) { + pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); + } +// pids = normalizeIdListForLastNInClause(lastnResourceIds); } if (pids.isEmpty()) { // Will never match @@ -374,6 +454,11 @@ public class SearchBuilder implements ISearchBuilder { myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids))); } +*/ + // Add PID list predicate for full text search and/or lastn operation + if (thePidList != null && thePidList.size() > 0) { + myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(thePidList)); + } /* * Add a predicate to make sure we only include non-deleted resources, and only include @@ -415,10 +500,10 @@ public class SearchBuilder implements ISearchBuilder { return query; } - private List normalizeIdListForLastNInClause(List lastnResourceIds) { - List retVal = new ArrayList<>(); - for (String lastnResourceId : lastnResourceIds) { - retVal.add(new ResourcePersistentId(Long.parseLong(lastnResourceId))); + private List normalizeIdListForLastNInClause(List lastnResourceIds) { + List retVal = new ArrayList<>(); + for (Long lastnResourceId : lastnResourceIds) { + retVal.add(lastnResourceId); } /* @@ -430,32 +515,27 @@ public class SearchBuilder implements ISearchBuilder { arguments never exceeds the maximum specified below. */ int listSize = retVal.size(); + if(listSize > 1 && listSize < 10) { padIdListWithPlaceholders(retVal, 10); - } else if (listSize > 10 && listSize < 100) { + } else if (listSize > 10 && listSize < 50) { + padIdListWithPlaceholders(retVal, 50); + } else if (listSize > 50 && listSize < 100) { padIdListWithPlaceholders(retVal, 100); } else if (listSize > 100 && listSize < 200) { padIdListWithPlaceholders(retVal, 200); } else if (listSize > 200 && listSize < 500) { padIdListWithPlaceholders(retVal, 500); - } else if (listSize > 500 && listSize < 1000) { - padIdListWithPlaceholders(retVal, 1000); - } else if (listSize > 1000 && listSize < 500) { - padIdListWithPlaceholders(retVal, 5000); - } else if (listSize > 5000 && listSize < 10000) { - padIdListWithPlaceholders(retVal, 10000); - } else if (listSize > 10000 && listSize < 20000) { - padIdListWithPlaceholders(retVal, 20000); - } else if (listSize > 20000 && listSize < 30000) { - padIdListWithPlaceholders(retVal, 30000); + } else if (listSize > 500 && listSize < 800) { + padIdListWithPlaceholders(retVal, 800); } return retVal; } - private void padIdListWithPlaceholders(List theIdList, int preferredListSize) { + private void padIdListWithPlaceholders(List theIdList, int preferredListSize) { while(theIdList.size() < preferredListSize) { - theIdList.add(new ResourcePersistentId(-1L)); + theIdList.add(-1L); } } @@ -733,7 +813,7 @@ public class SearchBuilder implements ISearchBuilder { if (matchAll) { String sql; sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r." + searchFieldName + " IN (:target_pids) "; - List> partitions = partition(nextRoundMatches, MAXIMUM_PAGE_SIZE); + List> partitions = partition(nextRoundMatches, getMaximumPageSize()); for (Collection nextPartition : partitions) { TypedQuery q = theEntityManager.createQuery(sql, Long.class); q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition)); @@ -786,7 +866,7 @@ public class SearchBuilder implements ISearchBuilder { sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids)"; } - List> partitions = partition(nextRoundMatches, MAXIMUM_PAGE_SIZE); + List> partitions = partition(nextRoundMatches, getMaximumPageSize()); for (Collection nextPartition : partitions) { TypedQuery q = theEntityManager.createQuery(sql, Long.class); q.setParameter("src_path", nextPath); @@ -1076,6 +1156,8 @@ public class SearchBuilder implements ISearchBuilder { private int mySkipCount = 0; private int myNonSkipCount = 0; + private List> myQueryList; + private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) { mySearchRuntimeDetails = theSearchRuntimeDetails; mySort = myParams.getSort(); @@ -1126,7 +1208,12 @@ public class SearchBuilder implements ISearchBuilder { } if (myNext == null) { - while (myResultsIterator.hasNext()) { + while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) { + // Update iterator with next chunk if necessary. + if (!myResultsIterator.hasNext() && !myQueryList.isEmpty()) { + retrieveNextIteratorQuery(); + } + Long nextLong = myResultsIterator.next(); if (myHavePerfTraceFoundIdHook) { HookParams params = new HookParams() @@ -1225,19 +1312,31 @@ public class SearchBuilder implements ISearchBuilder { } private void initializeIteratorQuery(Integer theMaxResultsToFetch) { - final TypedQuery query = createQuery(mySort, theMaxResultsToFetch, false, myRequest); + if (myQueryList == null || myQueryList.isEmpty()) { + myQueryList = createQuery(mySort, theMaxResultsToFetch, false, myRequest); + } mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); - Query hibernateQuery = (Query) query; - hibernateQuery.setFetchSize(myFetchSize); - ScrollableResults scroll = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY); - myResultsIterator = new ScrollableResultsIterator<>(scroll); + retrieveNextIteratorQuery(); mySkipCount = 0; myNonSkipCount = 0; } + private void retrieveNextIteratorQuery() { + if (myQueryList != null && myQueryList.size() > 0) { + final TypedQuery query = myQueryList.remove(0); + Query hibernateQuery = (Query) (query); + hibernateQuery.setFetchSize(myFetchSize); + ScrollableResults scroll = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY); + myResultsIterator = new ScrollableResultsIterator<>(scroll); + } else { + myResultsIterator = null; + } + + } + @Override public boolean hasNext() { if (myNext == null) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java index f3e26cdb847..5fddc842c30 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java @@ -13,79 +13,80 @@ import java.util.*; @Indexed(index = "observation_index") public class ObservationIndexedSearchParamLastNEntity { - @Id - @SequenceGenerator(name = "SEQ_LASTN", sequenceName = "SEQ_LASTN") - @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_LASTN") - @Column(name = "LASTN_ID") - private Long myId; + @Id + @SequenceGenerator(name = "SEQ_LASTN", sequenceName = "SEQ_LASTN") + @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_LASTN") + @Column(name = "LASTN_ID") + private Long myId; @Field(name = "subject", analyze = Analyze.NO) - @Column(name = "LASTN_SUBJECT_ID", nullable = true) - private String mySubject; + @Column(name = "LASTN_SUBJECT_ID", nullable = true) + private String mySubject; - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_OBSERVATION_CODE_FK")) - @IndexedEmbedded(depth = 2, prefix = "codeconcept") - private ObservationIndexedCodeCodeableConceptEntity myObservationCode; + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_OBSERVATION_CODE_FK")) + @IndexedEmbedded(depth = 2, prefix = "codeconcept") + private ObservationIndexedCodeCodeableConceptEntity myObservationCode; - @Field(name = "codeconceptid", analyze = Analyze.NO) - @Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false) - private String myCodeNormalizedId; + @Field(name = "codeconceptid", analyze = Analyze.NO) + @Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false) + private String myCodeNormalizedId; - @IndexedEmbedded(depth = 2, prefix = "categoryconcept") - @Transient - private Set myCategoryCodeableConcepts; + @IndexedEmbedded(depth = 2, prefix = "categoryconcept") + @Transient + private Set myCategoryCodeableConcepts; - @Field(name = "effectivedtm", analyze = Analyze.NO) - @Temporal(TemporalType.TIMESTAMP) - @Column(name = "LASTN_EFFECTIVE_DATETIME", nullable = true) - private Date myEffectiveDtm; + @Field(name = "effectivedtm", analyze = Analyze.NO) + @Temporal(TemporalType.TIMESTAMP) + @Column(name = "LASTN_EFFECTIVE_DATETIME", nullable = true) + private Date myEffectiveDtm; - @DocumentId(name = "identifier") - @Column(name = "RESOURCE_IDENTIFIER", nullable = false) - private String myIdentifier; + @DocumentId(name = "identifier") + @Column(name = "RESOURCE_IDENTIFIER", nullable = false) + private String myIdentifier; - public ObservationIndexedSearchParamLastNEntity() {} + public ObservationIndexedSearchParamLastNEntity() { + } - public String getSubject() { - return mySubject; - } + public String getSubject() { + return mySubject; + } - public void setSubject(String theSubject) { - mySubject = theSubject; - } + public void setSubject(String theSubject) { + mySubject = theSubject; + } - public String getIdentifier() { - return myIdentifier; - } + public String getIdentifier() { + return myIdentifier; + } - public void setIdentifier(String theIdentifier) { - myIdentifier = theIdentifier; - } + public void setIdentifier(String theIdentifier) { + myIdentifier = theIdentifier; + } - public void setEffectiveDtm(Date theEffectiveDtm) { - myEffectiveDtm = theEffectiveDtm; - } + public void setEffectiveDtm(Date theEffectiveDtm) { + myEffectiveDtm = theEffectiveDtm; + } - public Date getEffectiveDtm() { - return myEffectiveDtm; - } + public Date getEffectiveDtm() { + return myEffectiveDtm; + } - public void setCodeNormalizedId(String theCodeNormalizedId) { - myCodeNormalizedId = theCodeNormalizedId; - } + public void setCodeNormalizedId(String theCodeNormalizedId) { + myCodeNormalizedId = theCodeNormalizedId; + } - public String getCodeNormalizedId() { - return myCodeNormalizedId; - } + public String getCodeNormalizedId() { + return myCodeNormalizedId; + } - public void setObservationCode(ObservationIndexedCodeCodeableConceptEntity theObservationCode) { - myObservationCode = theObservationCode; - } + public void setObservationCode(ObservationIndexedCodeCodeableConceptEntity theObservationCode) { + myObservationCode = theObservationCode; + } - public void setCategoryCodeableConcepts(Set theCategoryCodeableConcepts) { - myCategoryCodeableConcepts = theCategoryCodeableConcepts; - } + public void setCategoryCodeableConcepts(Set theCategoryCodeableConcepts) { + myCategoryCodeableConcepts = theCategoryCodeableConcepts; + } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 9a730b6bb69..47c42cd6b6e 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -200,6 +200,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } @Override + // TODO: Should eliminate dependency on SearchParameterMap in API. public List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; try { @@ -252,6 +253,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } @VisibleForTesting + // TODO: Should eliminate dependency on SearchParameterMap in API. List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { try { List responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, null); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java index 9a48d3fc3b5..535d299fd9f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.search.lastn; public class IndexConstants { + // TODO: These should all be moved into ElasticSearchSvcImpl. public static final String OBSERVATION_INDEX = "observation_index"; public static final String CODE_INDEX = "code_index"; public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/QueryChunker.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/QueryChunker.java index c549c9fd481..31b172f5d8a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/QueryChunker.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/QueryChunker.java @@ -34,8 +34,8 @@ import java.util.function.Consumer; public class QueryChunker { public void chunk(List theInput, Consumer> theBatchConsumer) { - for (int i = 0; i < theInput.size(); i += SearchBuilder.MAXIMUM_PAGE_SIZE) { - int to = i + SearchBuilder.MAXIMUM_PAGE_SIZE; + for (int i = 0; i < theInput.size(); i += SearchBuilder.getMaximumPageSize()) { + int to = i + SearchBuilder.getMaximumPageSize(); to = Math.min(to, theInput.size()); List batch = theInput.subList(i, to); theBatchConsumer.accept(batch); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java index e5b40a53e56..20ab38370c9 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java @@ -5,13 +5,16 @@ import ca.uhn.fhir.jpa.api.dao.*; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; import ca.uhn.fhir.jpa.dao.BaseJpaTest; +import ca.uhn.fhir.jpa.dao.SearchBuilder; import ca.uhn.fhir.jpa.rp.r4.ObservationResourceProvider; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener; import ca.uhn.fhir.rest.param.*; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.util.TestUtil; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.*; +import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; @@ -23,7 +26,9 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.transaction.PlatformTransactionManager; import java.util.*; +import java.util.stream.Collectors; +import static org.hamcrest.Matchers.matchesPattern; import static org.junit.Assert.*; import static org.mockito.Mockito.when; @@ -58,6 +63,9 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { return myPlatformTransactionManager; } + @Autowired + protected CircularQueueCaptureQueriesListener myCaptureQueriesListener; + ObservationResourceProvider observationRp = new ObservationResourceProvider(); private final String observationCd0 = "code0"; @@ -108,6 +116,11 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { } + @After + public void resetMaximumPageSize() { + SearchBuilder.setIsTest(false); + } + private void createObservationsForPatient(IIdType thePatientId) { createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15); createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10); @@ -192,9 +205,6 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { params.setLastN(true); Map requestParameters = new HashMap<>(); -// String[] maxParam = new String[1]; -// maxParam[0] = "100"; -// requestParameters.put("max", maxParam); params.setLastNMax(100); when(mySrd.getParameters()).thenReturn(requestParameters); @@ -520,6 +530,82 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { return new TokenAndListParam().addAnd(myTokenOrListParam); } + @Test + public void testLastNWithChunkedQuery() { + SearchBuilder.setIsTest(true); + Integer numberOfObservations = SearchBuilder.getMaximumPageSize()+1; + Calendar observationDate = new GregorianCalendar(); + + List myObservationIds = new ArrayList<>(); + List myPatientIds = new ArrayList<>(); + List myPatientReferences = new ArrayList<>(); + for (int idx=0; idx actual; + params.setLastN(true); + + Map requestParameters = new HashMap<>(); + params.setLastNMax(1); + + params.setCount(numberOfObservations); + + when(mySrd.getParameters()).thenReturn(requestParameters); + + myCaptureQueriesListener.clear(); + actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + List queries = myCaptureQueriesListener + .getSelectQueriesForCurrentThread() + .stream() + .map(t -> t.getSql(true, false)) + .collect(Collectors.toList()); + + // First chunked query + String resultingQueryNotFormatted = queries.get(0); + assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '[0-9]+' , '[0-9]+' , '[0-9]+'\\).*")); + + // Second chunked query chunk + resultingQueryNotFormatted = queries.get(1); + assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '-1' , '-1' , '-1'\\).*")); + + assertEquals(numberOfObservations, (Integer)actual.size()); + for(IIdType observationId : myObservationIds) { + myObservationDao.delete(observationId); + } + + for (IIdType patientId : myPatientIds) { + myPatientDao.delete(patientId); + } + + } + @AfterClass public static void afterClassClearContext() { TestUtil.clearAllStaticFieldsForUnitTest(); diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java new file mode 100644 index 00000000000..c999194e926 --- /dev/null +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java @@ -0,0 +1,38 @@ +package ca.uhn.fhir.jpa.searchparam.util; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.FhirVersionEnum; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; + +public class LastNParameterHelper { + + public static boolean isLastNParameter(String theParamName, FhirContext theContext) { + if (theParamName == null) { + return false; + } + if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) { + if (theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_PATIENT) + || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)) { + return true; + } else { + return false; + } + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.R4) { + if (theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_PATIENT) + || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CODE)) { + return true; + } else { + return false; + } + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) { + if (theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_PATIENT) + || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CODE)) { + return true; + } else { + return false; + } + } else { + throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString()); + } + } +} From 8fcdc780777a0e746a6190403180a1e9401571fb Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Fri, 8 May 2020 09:31:05 -0400 Subject: [PATCH 16/31] Fix issues resulting from latest merge. --- .../jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java | 7 ++++--- .../uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java | 7 ++++--- .../uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java index db7a0cd47bf..b147bb62598 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java @@ -29,6 +29,7 @@ import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; import ca.uhn.fhir.rest.param.ReferenceAndListParam; import ca.uhn.fhir.rest.param.TokenAndListParam; import org.hl7.fhir.instance.model.api.IBaseResource; @@ -52,9 +53,9 @@ public class FhirResourceDaoObservationDstu3 extends BaseHapiFhirResourceDaoObse } @Override - public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, - boolean theUpdateVersion, Date theUpdateTime, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { - ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theUpdateTime, theForceUpdate, theCreateNewHistoryEntry); + public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, + boolean theUpdateVersion, TransactionDetails theTransactionDetails, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { + ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theTransactionDetails, theForceUpdate, theCreateNewHistoryEntry); if (!retVal.isUnchangedInCurrentOperation()) { if (retVal.getDeleted() == null) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index 55f47758b3f..3290c91fc38 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -29,6 +29,7 @@ import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.Observation; import org.springframework.beans.factory.annotation.Autowired; @@ -57,9 +58,9 @@ public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDaoObserva } @Override - public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, - boolean theUpdateVersion, Date theUpdateTime, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { - ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theUpdateTime, theForceUpdate, theCreateNewHistoryEntry); + public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, + boolean theUpdateVersion, TransactionDetails theTransactionDetails, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { + ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theTransactionDetails, theForceUpdate, theCreateNewHistoryEntry); if (!retVal.isUnchangedInCurrentOperation()) { if (retVal.getDeleted() == null) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java index 63d8333e763..7a693c91999 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java @@ -29,6 +29,7 @@ import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; import ca.uhn.fhir.rest.param.ReferenceAndListParam; import ca.uhn.fhir.rest.param.TokenAndListParam; import org.hl7.fhir.instance.model.api.IBaseResource; @@ -52,9 +53,9 @@ public class FhirResourceDaoObservationR5 extends BaseHapiFhirResourceDaoObserva } @Override - public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, - boolean theUpdateVersion, Date theUpdateTime, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { - ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theUpdateTime, theForceUpdate, theCreateNewHistoryEntry); + public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, + boolean theUpdateVersion, TransactionDetails theTransactionDetails, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { + ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theTransactionDetails, theForceUpdate, theCreateNewHistoryEntry); if (!retVal.isUnchangedInCurrentOperation()) { if (retVal.getDeleted() == null) { From eb1d1c2b27c31edd0a10d83d857af63e01db6506 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Thu, 14 May 2020 12:08:55 -0400 Subject: [PATCH 17/31] Changes to better support chunking of queries and paging of results for lastn operation and to monitor performance. --- .../ca/uhn/fhir/interceptor/api/Pointcut.java | 40 +- .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 86 +-- .../PerformanceTracingLoggingInterceptor.java | 5 + .../search/lastn/ElasticsearchSvcImpl.java | 35 +- .../jpa/search/lastn/IElasticsearchSvc.java | 3 +- .../ca/uhn/fhir/jpa/config/TestR4Config.java | 5 + .../config/TestR4ConfigWithElasticSearch.java | 3 + .../TestR4ConfigWithElasticsearchClient.java | 1 + .../fhir/jpa/dao/r4/BaseR4SearchLastN.java | 543 ++++++++++++++++ .../FhirResourceDaoR4SearchLastNAsyncIT.java | 133 ++++ .../r4/FhirResourceDaoR4SearchLastNIT.java | 578 +----------------- ...bservationIndexedSearchParamLastNR4IT.java | 20 +- ...lasticsearchSvcMultipleObservationsIT.java | 22 +- ...tNElasticsearchSvcSingleObservationIT.java | 4 +- .../model/search/SearchRuntimeDetails.java | 9 + 15 files changed, 867 insertions(+), 620 deletions(-) create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java index 672fa91f0d7..3ac4cffd0e1 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java @@ -1719,7 +1719,7 @@ public enum Pointcut { /** * Performance Tracing Hook: - * This hook is invoked when a search has failed for any reason. When this pointcut + * This hook is invoked when a search has completed. When this pointcut * is invoked, a pass in the Search Coordinator has completed successfully, but * not all possible resources have been loaded yet so a future paging request * may trigger a new task that will load further resources. @@ -1757,6 +1757,44 @@ public enum Pointcut { "ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails" ), + /** + * Performance Tracing Hook: + * This hook is invoked when a query involving an external index (e.g. Elasticsearch) has completed. When this pointcut + * is invoked, an initial list of resource IDs has been generated which will be used as part of a subsequent database query. + *

+ * Note that this is a performance tracing hook. Use with caution in production + * systems, since calling it may (or may not) carry a cost. + *

+ * Hooks may accept the following parameters: + *
    + *
  • + * ca.uhn.fhir.rest.api.server.RequestDetails - A bean containing details about the request that is about to be processed, including details such as the + * resource type and logical ID (if any) and other FHIR-specific aspects of the request which have been + * pulled out of the servlet request. Note that the bean + * properties are not all guaranteed to be populated, depending on how early during processing the + * exception occurred. + *
    • + *
  • + * ca.uhn.fhir.rest.server.servlet.ServletRequestDetails - A bean containing details about the request that is about to be processed, including details such as the + * resource type and logical ID (if any) and other FHIR-specific aspects of the request which have been + * pulled out of the servlet request. This parameter is identical to the RequestDetails parameter above but will + * only be populated when operating in a RestfulServer implementation. It is provided as a convenience. + *
    • + *
  • + * ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails - Contains details about the search being + * performed. Hooks should not modify this object. + *
    • + *
+ *

+ * Hooks should return void. + *

+ */ + JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE(void.class, + "ca.uhn.fhir.rest.api.server.RequestDetails", + "ca.uhn.fhir.rest.server.servlet.ServletRequestDetails", + "ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails" + ), + /** * Performance Tracing Hook: * Invoked when the storage engine is about to reuse the results of diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index b03011ce626..3027019dc1f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -131,7 +131,7 @@ public class SearchBuilder implements ISearchBuilder { */ // NB: keep public public static final int MAXIMUM_PAGE_SIZE = 800; - public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 4; + public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50; public static boolean myIsTest = false; private static final List EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<>()); @@ -249,7 +249,7 @@ public class SearchBuilder implements ISearchBuilder { init(theParams, theSearchUuid, theRequestPartitionId); - List> queries = createQuery(null, null, true, theRequest); + List> queries = createQuery(null, null, true, theRequest, null); return new CountQueryIterator(queries.get(0)); } @@ -283,7 +283,8 @@ public class SearchBuilder implements ISearchBuilder { } - private List> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) { + private List> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, + SearchRuntimeDetails theSearchRuntimeDetails) { List pids = new ArrayList<>(); /* @@ -310,17 +311,26 @@ public class SearchBuilder implements ISearchBuilder { throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); } } - Integer myMaxObservationsPerCode = null; + Integer myMaxObservationsPerCode; if(myParams.getLastNMax() != null) { myMaxObservationsPerCode = myParams.getLastNMax(); } else { throw new InvalidRequestException("Max parameter is required for $lastn operation"); } - List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); + List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode, theMaximumResults); for (String lastnResourceId : lastnResourceIds) { pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); } } + if (theSearchRuntimeDetails != null) { + theSearchRuntimeDetails.setFoundIndexMatchesCount(pids.size()); + HookParams params = new HookParams() + .add(RequestDetails.class, theRequest) + .addIfMatchesType(ServletRequestDetails.class, theRequest) + .add(SearchRuntimeDetails.class, theSearchRuntimeDetails); + JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params); + } + if (pids.isEmpty()) { // Will never match pids = Collections.singletonList(new ResourcePersistentId(-1L)); @@ -331,24 +341,27 @@ public class SearchBuilder implements ISearchBuilder { ArrayList> myQueries = new ArrayList<>(); if (!pids.isEmpty()) { + if (theMaximumResults != null && pids.size() > theMaximumResults) { + pids.subList(0,theMaximumResults-1); + } new QueryChunker().chunk(ResourcePersistentId.toLongList(pids), t->{ - doCreateChunkedQueries(t, sort, theMaximumResults, theCount, theRequest, myQueries); + doCreateChunkedQueries(t, sort, theCount, theRequest, myQueries); }); } else { - myQueries.add(createQuery(sort,theMaximumResults, theCount, theRequest, null)); + myQueries.add(createChunkedQuery(sort,theMaximumResults, theCount, theRequest, null)); } return myQueries; } - private void doCreateChunkedQueries(List thePids, SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, ArrayList> theQueries) { - if(thePids.size() < MAXIMUM_PAGE_SIZE) { - thePids = normalizeIdListForLastNInClause(thePids); + private void doCreateChunkedQueries(List thePids, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList> theQueries) { + if(thePids.size() < getMaximumPageSize()) { + normalizeIdListForLastNInClause(thePids); } - theQueries.add(createQuery(sort, theMaximumResults, theCount, theRequest, thePids)); + theQueries.add(createChunkedQuery(sort, thePids.size(), theCount, theRequest, thePids)); } - private TypedQuery createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List thePidList) { + private TypedQuery createChunkedQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List thePidList) { CriteriaQuery outerQuery; /* * Sort @@ -501,11 +514,6 @@ public class SearchBuilder implements ISearchBuilder { } private List normalizeIdListForLastNInClause(List lastnResourceIds) { - List retVal = new ArrayList<>(); - for (Long lastnResourceId : lastnResourceIds) { - retVal.add(lastnResourceId); - } - /* The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info: @@ -514,23 +522,23 @@ public class SearchBuilder implements ISearchBuilder { Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of arguments never exceeds the maximum specified below. */ - int listSize = retVal.size(); + int listSize = lastnResourceIds.size(); if(listSize > 1 && listSize < 10) { - padIdListWithPlaceholders(retVal, 10); + padIdListWithPlaceholders(lastnResourceIds, 10); } else if (listSize > 10 && listSize < 50) { - padIdListWithPlaceholders(retVal, 50); + padIdListWithPlaceholders(lastnResourceIds, 50); } else if (listSize > 50 && listSize < 100) { - padIdListWithPlaceholders(retVal, 100); + padIdListWithPlaceholders(lastnResourceIds, 100); } else if (listSize > 100 && listSize < 200) { - padIdListWithPlaceholders(retVal, 200); + padIdListWithPlaceholders(lastnResourceIds, 200); } else if (listSize > 200 && listSize < 500) { - padIdListWithPlaceholders(retVal, 500); + padIdListWithPlaceholders(lastnResourceIds, 500); } else if (listSize > 500 && listSize < 800) { - padIdListWithPlaceholders(retVal, 800); + padIdListWithPlaceholders(lastnResourceIds, 800); } - return retVal; + return lastnResourceIds; } private void padIdListWithPlaceholders(List theIdList, int preferredListSize) { @@ -664,10 +672,17 @@ public class SearchBuilder implements ISearchBuilder { private void doLoadPids(Collection thePids, Collection theIncludedPids, List theResourceListToPopulate, boolean theForHistoryOperation, - Map thePosition, RequestDetails theRequest) { + Map thePosition) { + + List myLongPersistentIds; + if(thePids.size() < getMaximumPageSize()) { + myLongPersistentIds = normalizeIdListForLastNInClause(ResourcePersistentId.toLongList(thePids)); + } else { + myLongPersistentIds = ResourcePersistentId.toLongList(thePids); + } // -- get the resource from the searchView - Collection resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(ResourcePersistentId.toLongList(thePids)); + Collection resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(myLongPersistentIds); //-- preload all tags with tag definition if any Map> tagMap = getResourceTagMap(resourceSearchViewList); @@ -768,7 +783,7 @@ public class SearchBuilder implements ISearchBuilder { List pids = new ArrayList<>(thePids); new QueryChunker().chunk(pids, t->{ - doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position, theDetails); + doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position); }); } @@ -1096,9 +1111,8 @@ public class SearchBuilder implements ISearchBuilder { private final RequestDetails myRequest; private Iterator myCurrentIterator; - private Set myCurrentPids; + private final Set myCurrentPids; private ResourcePersistentId myNext; - private int myPageSize = myDaoConfig.getEverythingIncludesFetchPageSize(); IncludesIterator(Set thePidSet, RequestDetails theRequest) { myCurrentPids = new HashSet<>(thePidSet); @@ -1151,12 +1165,12 @@ public class SearchBuilder implements ISearchBuilder { private ResourcePersistentId myNext; private Iterator myPreResultsIterator; private ScrollableResultsIterator myResultsIterator; - private SortSpec mySort; + private final SortSpec mySort; private boolean myStillNeedToFetchIncludes; private int mySkipCount = 0; private int myNonSkipCount = 0; - private List> myQueryList; + private List> myQueryList = new ArrayList<>(); private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) { mySearchRuntimeDetails = theSearchRuntimeDetails; @@ -1210,7 +1224,7 @@ public class SearchBuilder implements ISearchBuilder { if (myNext == null) { while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) { // Update iterator with next chunk if necessary. - if (!myResultsIterator.hasNext() && !myQueryList.isEmpty()) { + if (!myResultsIterator.hasNext()) { retrieveNextIteratorQuery(); } @@ -1312,8 +1326,10 @@ public class SearchBuilder implements ISearchBuilder { } private void initializeIteratorQuery(Integer theMaxResultsToFetch) { - if (myQueryList == null || myQueryList.isEmpty()) { - myQueryList = createQuery(mySort, theMaxResultsToFetch, false, myRequest); + if (myQueryList.isEmpty()) { + // Capture times for Lucene/Elasticsearch queries as well + mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); + myQueryList = createQuery(mySort, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails); } mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/PerformanceTracingLoggingInterceptor.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/PerformanceTracingLoggingInterceptor.java index a01b6f08203..6cca9668499 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/PerformanceTracingLoggingInterceptor.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/PerformanceTracingLoggingInterceptor.java @@ -79,6 +79,11 @@ public class PerformanceTracingLoggingInterceptor { log("SqlQuery {} failed in {} - Found {} matches", theOutcome.getSearchUuid(), theOutcome.getQueryStopwatch(), theOutcome.getFoundMatchesCount()); } + @Hook(value = Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE) + public void indexSearchQueryComplete(SearchRuntimeDetails theOutcome) { + log("Index query for {} completed in {} - Found {} matches", theOutcome.getSearchUuid(), theOutcome.getQueryStopwatch(), theOutcome.getFoundIndexMatchesCount()); + } + @Hook(value = Pointcut.JPA_PERFTRACE_INFO) public void info(StorageProcessingMessage theMessage) { log("[INFO] " + theMessage); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 47c42cd6b6e..49af7d3510d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -2,7 +2,6 @@ package ca.uhn.fhir.jpa.search.lastn; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.IQueryParameterType; -import ca.uhn.fhir.model.dstu2.resource.Observation; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; @@ -49,9 +48,9 @@ import static org.apache.commons.lang3.StringUtils.isBlank; public class ElasticsearchSvcImpl implements IElasticsearchSvc { - RestHighLevelClient myRestHighLevelClient; + private final RestHighLevelClient myRestHighLevelClient; - ObjectMapper objectMapper = new ObjectMapper(); + private final ObjectMapper objectMapper = new ObjectMapper(); private final String GROUP_BY_SUBJECT = "group_by_subject"; private final String GROUP_BY_CODE = "group_by_code"; @@ -201,14 +200,15 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { @Override // TODO: Should eliminate dependency on SearchParameterMap in API. - public List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { + public List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch) { String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; try { List responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, topHitsInclude); List observationIds = new ArrayList<>(); for (SearchResponse response : responses) { // observationIds.addAll(buildObservationIdList(response)); - observationIds.addAll(buildObservationList(response, t -> t.getIdentifier(), theSearchParameterMap)); + Integer maxResultsToAdd = theMaxResultsToFetch - observationIds.size(); + observationIds.addAll(buildObservationList(response, t -> t.getIdentifier(), theSearchParameterMap, maxResultsToAdd)); } return observationIds; } catch (IOException theE) { @@ -216,7 +216,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } } - private List buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, String[] topHitsInclude) { + private List buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, + String[] topHitsInclude) { List responses = new ArrayList<>(); if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { ArrayList subjectReferenceCriteria = new ArrayList<>(); @@ -254,12 +255,12 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { @VisibleForTesting // TODO: Should eliminate dependency on SearchParameterMap in API. - List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { + List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch) { try { List responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, null); List observationDocuments = new ArrayList<>(); for (SearchResponse response : responses) { - observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap)); + observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap, theMaxResultsToFetch)); } return observationDocuments; } catch (IOException theE) { @@ -337,12 +338,22 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return theObservationList; } - private List buildObservationList(SearchResponse theSearchResponse, Function setValue, SearchParameterMap theSearchParameterMap) throws IOException { + private List buildObservationList(SearchResponse theSearchResponse, Function setValue, + SearchParameterMap theSearchParameterMap, Integer theMaxResultsToFetch) throws IOException { List theObservationList = new ArrayList<>(); if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } String indexedObservation = lastNMatch.getSourceAsString(); ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); theObservationList.add(setValue.apply(observationJson)); @@ -351,7 +362,13 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } } else { for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(theSearchResponse)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { + if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { + break; + } String indexedObservation = lastNMatch.getSourceAsString(); ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); theObservationList.add(setValue.apply(observationJson)); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java index 091d85b742f..592b6939d6d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -5,6 +5,5 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import java.util.List; public interface IElasticsearchSvc { - - List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode); + List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java index 3955361d5fd..dfa4c95bd21 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java @@ -102,10 +102,14 @@ public class TestR4Config extends BaseJavaConfigR4 { }; retVal.setDriver(new org.h2.Driver()); +// retVal.setDriver(new org.postgresql.Driver()); retVal.setUrl("jdbc:h2:mem:testdb_r4"); +// retVal.setUrl("jdbc:postgresql://localhost:5432/hapi"); retVal.setMaxWaitMillis(10000); retVal.setUsername(""); +// retVal.setUsername("hapi"); retVal.setPassword(""); +// retVal.setPassword("HapiFHIR"); retVal.setMaxTotal(ourMaxThreads); SLF4JLogLevel level = SLF4JLogLevel.INFO; @@ -145,6 +149,7 @@ public class TestR4Config extends BaseJavaConfigR4 { extraProperties.put("hibernate.show_sql", "false"); extraProperties.put("hibernate.hbm2ddl.auto", "update"); extraProperties.put("hibernate.dialect", H2Dialect.class.getName()); +// extraProperties.put("hibernate.dialect", org.hibernate.dialect.PostgreSQL95Dialect.class.getName()); extraProperties.put("hibernate.search.model_mapping", ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory.class.getName()); extraProperties.put("hibernate.search.default.directory_provider", "local-heap"); extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java index ebf3fc9e094..1b90253383d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java @@ -24,7 +24,9 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config { private static final String ELASTIC_VERSION = "6.5.4"; protected final String elasticsearchHost = "localhost"; protected final String elasticsearchUserId = ""; +// protected final String elasticsearchUserId = "elastic"; protected final String elasticsearchPassword = ""; +// protected final String elasticsearchPassword = "changeme"; @Override @@ -34,6 +36,7 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config { // Force elasticsearch to start first int httpPort = embeddedElasticSearch().getHttpPort(); +// int httpPort = 9301; ourLog.info("ElasticSearch started on port: {}", httpPort); new ElasticsearchHibernatePropertiesBuilder() diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java index 9ff6bc4c491..607457c8edd 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java @@ -10,6 +10,7 @@ public class TestR4ConfigWithElasticsearchClient extends TestR4ConfigWithElastic @Bean() public ElasticsearchSvcImpl myElasticsearchSvc() { int elasticsearchPort = embeddedElasticSearch().getHttpPort(); +// int elasticsearchPort = 9301; return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java new file mode 100644 index 00000000000..bcfde6bac4f --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java @@ -0,0 +1,543 @@ +package ca.uhn.fhir.jpa.dao.r4; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoPatient; +import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; +import ca.uhn.fhir.jpa.dao.BaseJpaTest; +import ca.uhn.fhir.jpa.dao.SearchBuilder; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceAndListParam; +import ca.uhn.fhir.rest.param.ReferenceOrListParam; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenAndListParam; +import ca.uhn.fhir.rest.param.TokenOrListParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.server.RestfulServer; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.util.TestUtil; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.StringType; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.transaction.PlatformTransactionManager; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.matchesPattern; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class }) +public class BaseR4SearchLastN extends BaseJpaTest { + + @Autowired + @Qualifier("myPatientDaoR4") + protected IFhirResourceDaoPatient myPatientDao; + + @Autowired + @Qualifier("myObservationDaoR4") + protected IFhirResourceDaoObservation myObservationDao; + + @Autowired + protected DaoConfig myDaoConfig; + + @Autowired + protected FhirContext myFhirCtx; + + @Autowired + protected PlatformTransactionManager myPlatformTransactionManager; + + @Override + protected FhirContext getContext() { + return myFhirCtx; + } + + @Override + protected PlatformTransactionManager getTxManager() { + return myPlatformTransactionManager; + } + + protected final String observationCd0 = "code0"; + protected final String observationCd1 = "code1"; + protected final String observationCd2 = "code2"; + private final String observationCd3 = "code3"; + + protected final String categoryCd0 = "category0"; + private final String categoryCd1 = "category1"; + private final String categoryCd2 = "category2"; + private final String categoryCd3 = "category3"; + + protected final String codeSystem = "http://mycode.com"; + private final String categorySystem = "http://mycategory.com"; + + // Using static variables including the flag below so that we can initalize the database and indexes once + // (all of the tests only read from the DB and indexes and so no need to re-initialze them for each test). + private static boolean dataLoaded = false; + + protected static IIdType patient0Id = null; + protected static IIdType patient1Id = null; + protected static IIdType patient2Id = null; + + private static final Map observationPatientMap = new HashMap<>(); + private static final Map observationCategoryMap = new HashMap<>(); + private static final Map observationCodeMap = new HashMap<>(); + private static final Map observationEffectiveMap = new HashMap<>(); + + @Before + public void beforeCreateTestPatientsAndObservations() { + // Using a static flag to ensure that test data and elasticsearch index is only created once. + // Creating this data and the index is time consuming and as such want to avoid having to repeat for each test. + // Normally would use a static @BeforeClass method for this purpose, but Autowired objects cannot be accessed in static methods. + if(!dataLoaded) { + Patient pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Arthur"); + patient0Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + createObservationsForPatient(patient0Id); + pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Johnathan"); + patient1Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + createObservationsForPatient(patient1Id); + pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Michael"); + patient2Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); + createObservationsForPatient(patient2Id); + dataLoaded = true; + + } + + } + + private void createObservationsForPatient(IIdType thePatientId) { + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd2, 5); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd0, 10); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd1, 5); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd2, categoryCd2, 5); + createFiveObservationsForPatientCodeCategory(thePatientId,observationCd3, categoryCd3, 5); + } + + private void createFiveObservationsForPatientCodeCategory(IIdType thePatientId, String theObservationCode, String theCategoryCode, + Integer theTimeOffset) { + Calendar observationDate = new GregorianCalendar(); + + for (int idx=0; idx<5; idx++ ) { + Observation obs = new Observation(); + obs.getSubject().setReferenceElement(thePatientId); + obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem); + obs.setValue(new StringType(theObservationCode + "_0")); + observationDate.add(Calendar.HOUR, -theTimeOffset+idx); + Date effectiveDtm = observationDate.getTime(); + obs.setEffective(new DateTimeType(effectiveDtm)); + obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem); + String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue(); + observationPatientMap.put(observationId, thePatientId.getValue()); + observationCategoryMap.put(observationId, theCategoryCode); + observationCodeMap.put(observationId, theObservationCode); + observationEffectiveMap.put(observationId, effectiveDtm); + } + } + + protected ServletRequestDetails mockSrd() { + return mySrd; + } + + @Test + public void testLastNAllPatients() { + + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + sortedObservationCodes.add(observationCd3); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null,105); + } + + @Test + public void testLastNNoPatients() { + + SearchParameterMap params = new SearchParameterMap(); + params.setLastNMax(1); + + params.setLastN(true); + Map requestParameters = new HashMap<>(); + when(mySrd.getParameters()).thenReturn(requestParameters); + + List actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + + assertEquals(4, actual.size()); + } + + private void executeTestCase(SearchParameterMap params, List sortedPatients, List sortedObservationCodes, List theCategories, int expectedObservationCount) { + List actual; + params.setLastN(true); + + Map requestParameters = new HashMap<>(); + params.setLastNMax(100); + + when(mySrd.getParameters()).thenReturn(requestParameters); + + actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + + assertEquals(expectedObservationCount, actual.size()); + + validateSorting(actual, sortedPatients, sortedObservationCodes, theCategories); + } + + private void validateSorting(List theObservationIds, List thePatientIds, List theCodes, List theCategores) { + int theNextObservationIdx = 0; + // Validate patient grouping + for (String patientId : thePatientIds) { + assertEquals(patientId, observationPatientMap.get(theObservationIds.get(theNextObservationIdx))); + theNextObservationIdx = validateSortingWithinPatient(theObservationIds,theNextObservationIdx,theCodes, theCategores, patientId); + } + assertEquals(theObservationIds.size(), theNextObservationIdx); + } + + private int validateSortingWithinPatient(List theObservationIds, int theFirstObservationIdxForPatient, List theCodes, + List theCategories, String thePatientId) { + int theNextObservationIdx = theFirstObservationIdxForPatient; + for (String codeValue : theCodes) { + assertEquals(codeValue, observationCodeMap.get(theObservationIds.get(theNextObservationIdx))); + // Validate sorting within code group + theNextObservationIdx = validateSortingWithinCode(theObservationIds,theNextObservationIdx, + observationCodeMap.get(theObservationIds.get(theNextObservationIdx)), theCategories, thePatientId); + } + return theNextObservationIdx; + } + + private int validateSortingWithinCode(List theObservationIds, int theFirstObservationIdxForPatientAndCode, String theObservationCode, + List theCategories, String thePatientId) { + int theNextObservationIdx = theFirstObservationIdxForPatientAndCode; + Date lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); + theNextObservationIdx++; + while(theObservationCode.equals(observationCodeMap.get(theObservationIds.get(theNextObservationIdx))) + && thePatientId.equals(observationPatientMap.get(theObservationIds.get(theNextObservationIdx)))) { + // Check that effective date is before that of the previous observation. + assertTrue(lastEffectiveDt.compareTo(observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx))) > 0); + lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); + + // Check that observation is in one of the specified categories (if applicable) + if (theCategories != null && !theCategories.isEmpty()) { + assertTrue(theCategories.contains(observationCategoryMap.get(theObservationIds.get(theNextObservationIdx)))); + } + theNextObservationIdx++; + if (theNextObservationIdx >= theObservationIds.size()) { + // Have reached the end of the Observation list. + break; + } + } + return theNextObservationIdx; + } + + @Test + public void testLastNSinglePatient() { + + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + sortedObservationCodes.add(observationCd3); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,35); + + params = new SearchParameterMap(); + ReferenceParam patientParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); + + sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + sortedObservationCodes.add(observationCd3); + + executeTestCase(params, sortedPatients,sortedObservationCodes, null,35); + } + + protected ReferenceAndListParam buildReferenceAndListParam(ReferenceParam... theReference) { + ReferenceOrListParam myReferenceOrListParam = new ReferenceOrListParam(); + for (ReferenceParam referenceParam : theReference) { + myReferenceOrListParam.addOr(referenceParam); + } + return new ReferenceAndListParam().addAnd(myReferenceOrListParam); + } + + @Test + public void testLastNMultiplePatients() { + + // Two Subject parameters. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + sortedObservationCodes.add(observationCd2); + sortedObservationCodes.add(observationCd3); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null,70); + + // Two Patient parameters + params = new SearchParameterMap(); + ReferenceParam patientParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam patientParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(patientParam1, patientParam3)); + + sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + executeTestCase(params,sortedPatients, sortedObservationCodes, null,70); + + } + + @Test + public void testLastNSingleCategory() { + + // One category parameter. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd0); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); + + // Another category parameter. + params = new SearchParameterMap(); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + categoryParam = new TokenParam(categorySystem, categoryCd2); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); + myCategories = new ArrayList<>(); + myCategories.add(categoryCd2); + + sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); + + } + + @Test + public void testLastNMultipleCategories() { + + // Two category parameters. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd0); + TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd1); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd0); + myCategories.add(categoryCd1); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 60); + } + + @Test + public void testLastNSingleCode() { + + // One code parameter. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + TokenParam code = new TokenParam(codeSystem, observationCd0); + params.add(Observation.SP_CODE, buildTokenAndListParam(code)); + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null, 45); + + // Another code parameter. + params = new SearchParameterMap(); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + code = new TokenParam(codeSystem, observationCd2); + params.add(Observation.SP_CODE, buildTokenAndListParam(code)); + sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null, 15); + + } + + @Test + public void testLastNMultipleCodes() { + + // Two code parameters. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + + TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); + TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1); + params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd1); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + sortedPatients.add(patient2Id.getValue()); + + executeTestCase(params, sortedPatients, sortedObservationCodes, null, 75); + + } + + @Test + public void testLastNSinglePatientCategoryCode() { + + // One patient, category and code. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + TokenParam code = new TokenParam(codeSystem, observationCd0); + params.add(Observation.SP_CODE, buildTokenAndListParam(code)); + TokenParam category = new TokenParam(categorySystem, categoryCd2); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(category)); + + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 5); + + } + + @Test + public void testLastNMultiplePatientsCategoriesCodes() { + + // Two patients, categories and codes. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); + List sortedPatients = new ArrayList<>(); + sortedPatients.add(patient0Id.getValue()); + sortedPatients.add(patient1Id.getValue()); + + TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); + TokenParam codeParam2 = new TokenParam(codeSystem, observationCd2); + params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + List sortedObservationCodes = new ArrayList<>(); + sortedObservationCodes.add(observationCd0); + sortedObservationCodes.add(observationCd2); + + TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd1); + TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd2); + params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); + List myCategories = new ArrayList<>(); + myCategories.add(categoryCd1); + myCategories.add(categoryCd2); + + executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); + + } + + protected TokenAndListParam buildTokenAndListParam(TokenParam... theToken) { + TokenOrListParam myTokenOrListParam = new TokenOrListParam(); + for (TokenParam tokenParam : theToken) { + myTokenOrListParam.addOr(tokenParam); + } + return new TokenAndListParam().addAnd(myTokenOrListParam); + } + + @AfterClass + public static void afterClassClearContext() { + TestUtil.clearAllStaticFieldsForUnitTest(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java new file mode 100644 index 00000000000..7978c956e15 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java @@ -0,0 +1,133 @@ +package ca.uhn.fhir.jpa.dao.r4; + +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.dao.SearchBuilder; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.server.RestfulServer; +import ca.uhn.fhir.util.TestUtil; +import org.hl7.fhir.r4.model.Observation; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.matchesPattern; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +public class FhirResourceDaoR4SearchLastNAsyncIT extends BaseR4SearchLastN { + + @Autowired + protected DaoConfig myDaoConfig; + + private List originalPreFetchThresholds; + + @Before + public void before() { + + RestfulServer myServer = new RestfulServer(myFhirCtx); + myServer.setPagingProvider(myDatabaseBackedPagingProvider); + + when(mySrd.getServer()).thenReturn(myServer); + + // Set pre-fetch sizes small so that most tests are forced to do multiple fetches. + // This will allow testing a common use case where result set is larger than first fetch size but smaller than the normal query chunk size. + originalPreFetchThresholds = myDaoConfig.getSearchPreFetchThresholds(); + List mySmallerPreFetchThresholds = new ArrayList<>(); + mySmallerPreFetchThresholds.add(20); + mySmallerPreFetchThresholds.add(400); + mySmallerPreFetchThresholds.add(-1); + myDaoConfig.setSearchPreFetchThresholds(mySmallerPreFetchThresholds); + + SearchBuilder.setIsTest(true); + + } + + @After + public void after() { + myDaoConfig.setSearchPreFetchThresholds(originalPreFetchThresholds); + SearchBuilder.setIsTest(false); + } + + @Test + public void testLastNChunking() { + + // Set up search parameters that will return 75 Observations. + SearchParameterMap params = new SearchParameterMap(); + ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); + ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); + ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); + params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); + TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); + TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1); + params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + + params.setLastN(true); + params.setLastNMax(100); + + Map requestParameters = new HashMap<>(); + when(mySrd.getParameters()).thenReturn(requestParameters); + + // Set chunk size to 50 + SearchBuilder.setIsTest(true); + + // Expand default fetch sizes to ensure all observations are returned in first page: + List myBiggerPreFetchThresholds = new ArrayList<>(); + myBiggerPreFetchThresholds.add(100); + myBiggerPreFetchThresholds.add(1000); + myBiggerPreFetchThresholds.add(-1); + myDaoConfig.setSearchPreFetchThresholds(myBiggerPreFetchThresholds); + + myCaptureQueriesListener.clear(); + List results = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + assertEquals(75, results.size()); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + List queries = myCaptureQueriesListener + .getSelectQueriesForCurrentThread() + .stream() + .map(t -> t.getSql(true, false)) + .collect(Collectors.toList()); + + // 1 query to lookup up Search from cache, and 2 chunked queries to retrieve resources by PID. + assertEquals(3, queries.size()); + + // The first chunked query should have a full complement of PIDs + StringBuilder firstQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'"); + for (int pidIndex = 1; pidIndex<50; pidIndex++) { + firstQueryPattern.append(" , '[0-9]+'"); + } + firstQueryPattern.append("\\).*"); + assertThat(queries.get(1), matchesPattern(firstQueryPattern.toString())); + + // the second chunked query should be padded with "-1". + StringBuilder secondQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'"); + for (int pidIndex = 1; pidIndex<25; pidIndex++) { + secondQueryPattern.append(" , '[0-9]+'"); + } + for (int pidIndex = 0; pidIndex<25; pidIndex++) { + secondQueryPattern.append(" , '-1'"); + } + secondQueryPattern.append("\\).*"); + assertThat(queries.get(2), matchesPattern(secondQueryPattern.toString())); + + } + + @AfterClass + public static void afterClassClearContext() { + TestUtil.clearAllStaticFieldsForUnitTest(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java index 20ab38370c9..8614a9e7c7e 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java @@ -1,29 +1,15 @@ package ca.uhn.fhir.jpa.dao.r4; -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.jpa.api.dao.*; -import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; -import ca.uhn.fhir.jpa.dao.BaseJpaTest; import ca.uhn.fhir.jpa.dao.SearchBuilder; -import ca.uhn.fhir.jpa.rp.r4.ObservationResourceProvider; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener; import ca.uhn.fhir.rest.param.*; -import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.util.TestUtil; -import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.*; import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.PlatformTransactionManager; import java.util.*; import java.util.stream.Collectors; @@ -33,553 +19,38 @@ import static org.junit.Assert.*; import static org.mockito.Mockito.when; @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { TestR4ConfigWithElasticsearchClient.class }) -public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { - - @Autowired - @Qualifier("myPatientDaoR4") - protected IFhirResourceDaoPatient myPatientDao; - - @Autowired - @Qualifier("myObservationDaoR4") - protected IFhirResourceDaoObservation myObservationDao; - - @Autowired - protected DaoConfig myDaoConfig; - - @Autowired - protected FhirContext myFhirCtx; - - @Autowired - protected PlatformTransactionManager myPlatformTransactionManager; - - @Override - protected FhirContext getContext() { - return myFhirCtx; - } - - @Override - protected PlatformTransactionManager getTxManager() { - return myPlatformTransactionManager; - } - - @Autowired - protected CircularQueueCaptureQueriesListener myCaptureQueriesListener; - - ObservationResourceProvider observationRp = new ObservationResourceProvider(); - - private final String observationCd0 = "code0"; - private final String observationCd1 = "code1"; - private final String observationCd2 = "code2"; - - private final String categoryCd0 = "category0"; - private final String categoryCd1 = "category1"; - private final String categoryCd2 = "category2"; - - private final String codeSystem = "http://mycode.com"; - private final String categorySystem = "http://mycategory.com"; - - // Using static variables including the flag below so that we can initalize the database and indexes once - // (all of the tests only read from the DB and indexes and so no need to re-initialze them for each test). - private static boolean dataLoaded = false; - - private static IIdType patient0Id = null; - private static IIdType patient1Id = null; - private static IIdType patient2Id = null; - - private static final Map observationPatientMap = new HashMap<>(); - private static final Map observationCategoryMap = new HashMap<>(); - private static final Map observationCodeMap = new HashMap<>(); - private static final Map observationEffectiveMap = new HashMap<>(); - - @Before - public void beforeCreateTestPatientsAndObservations() { - // Using a static flag here to ensure that load is only done once. Reason for this is that we cannot - // access Autowired objects in @BeforeClass method. - if(!dataLoaded) { - Patient pt = new Patient(); - pt.addName().setFamily("Lastn").addGiven("Arthur"); - patient0Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); - createObservationsForPatient(patient0Id); - pt = new Patient(); - pt.addName().setFamily("Lastn").addGiven("Johnathan"); - patient1Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); - createObservationsForPatient(patient1Id); - pt = new Patient(); - pt.addName().setFamily("Lastn").addGiven("Michael"); - patient2Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless(); - createObservationsForPatient(patient2Id); - dataLoaded = true; - } - - observationRp.setDao(myObservationDao); - - } +public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN { @After public void resetMaximumPageSize() { SearchBuilder.setIsTest(false); } - private void createObservationsForPatient(IIdType thePatientId) { - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd2, 5); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd0, 10); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd1, categoryCd1, 5); - createFiveObservationsForPatientCodeCategory(thePatientId,observationCd2, categoryCd2, 5); - } - - private void createFiveObservationsForPatientCodeCategory(IIdType thePatientId, String theObservationCode, String theCategoryCode, - Integer theTimeOffset) { - Calendar observationDate = new GregorianCalendar(); - - for (int idx=0; idx<5; idx++ ) { - Observation obs = new Observation(); - obs.getSubject().setReferenceElement(thePatientId); - obs.getCode().addCoding().setCode(theObservationCode).setSystem(codeSystem); - obs.setValue(new StringType(theObservationCode + "_0")); - observationDate.add(Calendar.HOUR, -theTimeOffset+idx); - Date effectiveDtm = observationDate.getTime(); - obs.setEffective(new DateTimeType(effectiveDtm)); - obs.getCategoryFirstRep().addCoding().setCode(theCategoryCode).setSystem(categorySystem); - String observationId = myObservationDao.create(obs, mockSrd()).getId().toUnqualifiedVersionless().getValue(); - observationPatientMap.put(observationId, thePatientId.getValue()); - observationCategoryMap.put(observationId, theCategoryCode); - observationCodeMap.put(observationId, theObservationCode); - observationEffectiveMap.put(observationId, effectiveDtm); - } - } - - private ServletRequestDetails mockSrd() { - return mySrd; - } - @Test - public void testLastNAllPatients() { + public void testLastNChunking() { + // Set up search parameters that will return 75 Observations. SearchParameterMap params = new SearchParameterMap(); ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null,90); - } - - @Test - public void testLastNNoPatients() { - - SearchParameterMap params = new SearchParameterMap(); - params.setLastNMax(1); - - List sortedPatients = new ArrayList<>(); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - -// executeTestCase(params, sortedPatients, sortedObservationCodes, null,3); - params.setLastN(true); - Map requestParameters = new HashMap<>(); - when(mySrd.getParameters()).thenReturn(requestParameters); - - List actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); - - assertEquals(3, actual.size()); - } - - private void executeTestCase(SearchParameterMap params, List sortedPatients, List sortedObservationCodes, List theCategories, int expectedObservationCount) { - List actual; - params.setLastN(true); - - Map requestParameters = new HashMap<>(); - params.setLastNMax(100); - - when(mySrd.getParameters()).thenReturn(requestParameters); - - actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); - - assertEquals(expectedObservationCount, actual.size()); - - validateSorting(actual, sortedPatients, sortedObservationCodes, theCategories); - } - - private void validateSorting(List theObservationIds, List thePatientIds, List theCodes, List theCategores) { - int theNextObservationIdx = 0; - // Validate patient grouping - for (String patientId : thePatientIds) { - assertEquals(patientId, observationPatientMap.get(theObservationIds.get(theNextObservationIdx))); - theNextObservationIdx = validateSortingWithinPatient(theObservationIds,theNextObservationIdx,theCodes, theCategores, patientId); - } - assertEquals(theObservationIds.size(), theNextObservationIdx); - } - - private int validateSortingWithinPatient(List theObservationIds, int theFirstObservationIdxForPatient, List theCodes, - List theCategories, String thePatientId) { - int theNextObservationIdx = theFirstObservationIdxForPatient; - for (String codeValue : theCodes) { - assertEquals(codeValue, observationCodeMap.get(theObservationIds.get(theNextObservationIdx))); - // Validate sorting within code group - theNextObservationIdx = validateSortingWithinCode(theObservationIds,theNextObservationIdx, - observationCodeMap.get(theObservationIds.get(theNextObservationIdx)), theCategories, thePatientId); - } - return theNextObservationIdx; - } - - private int validateSortingWithinCode(List theObservationIds, int theFirstObservationIdxForPatientAndCode, String theObservationCode, - List theCategories, String thePatientId) { - int theNextObservationIdx = theFirstObservationIdxForPatientAndCode; - Date lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); - theNextObservationIdx++; - while(theObservationCode.equals(observationCodeMap.get(theObservationIds.get(theNextObservationIdx))) - && thePatientId.equals(observationPatientMap.get(theObservationIds.get(theNextObservationIdx)))) { - // Check that effective date is before that of the previous observation. - assertTrue(lastEffectiveDt.compareTo(observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx))) > 0); - lastEffectiveDt = observationEffectiveMap.get(theObservationIds.get(theNextObservationIdx)); - - // Check that observation is in one of the specified categories (if applicable) - if (theCategories != null && !theCategories.isEmpty()) { - assertTrue(theCategories.contains(observationCategoryMap.get(theObservationIds.get(theNextObservationIdx)))); - } - theNextObservationIdx++; - if (theNextObservationIdx >= theObservationIds.size()) { - // Have reached the end of the Observation list. - break; - } - } - return theNextObservationIdx; - } - - @Test - public void testLastNSinglePatient() { - - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients,sortedObservationCodes, null,30); - - params = new SearchParameterMap(); - ReferenceParam patientParam = new ReferenceParam("Patient", "", patient0Id.getValue()); - params.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam)); - - sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - - sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients,sortedObservationCodes, null,30); - } - - private ReferenceAndListParam buildReferenceAndListParam(ReferenceParam... theReference) { - ReferenceOrListParam myReferenceOrListParam = new ReferenceOrListParam(); - for (ReferenceParam referenceParam : theReference) { - myReferenceOrListParam.addOr(referenceParam); - } - return new ReferenceAndListParam().addAnd(myReferenceOrListParam); - } - - @Test - public void testLastNMultiplePatients() { - - // Two Subject parameters. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null,60); - - // Two Patient parameters - params = new SearchParameterMap(); - ReferenceParam patientParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam patientParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(patientParam1, patientParam3)); - - sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - executeTestCase(params,sortedPatients, sortedObservationCodes, null,60); - - } - - @Test - public void testLastNSingleCategory() { - - // One category parameter. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - - TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - List myCategories = new ArrayList<>(); - myCategories.add(categoryCd0); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); - - // Another category parameter. - params = new SearchParameterMap(); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - categoryParam = new TokenParam(categorySystem, categoryCd2); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); - myCategories = new ArrayList<>(); - myCategories.add(categoryCd2); - - sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); - - } - - @Test - public void testLastNMultipleCategories() { - - // Two category parameters. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - - TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd0); - TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd1); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); - List myCategories = new ArrayList<>(); - myCategories.add(categoryCd0); - myCategories.add(categoryCd1); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 60); - } - - @Test - public void testLastNSingleCode() { - - // One code parameter. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - - TokenParam code = new TokenParam(codeSystem, observationCd0); - params.add(Observation.SP_CODE, buildTokenAndListParam(code)); - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null, 45); - - // Another code parameter. - params = new SearchParameterMap(); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - code = new TokenParam(codeSystem, observationCd2); - params.add(Observation.SP_CODE, buildTokenAndListParam(code)); - sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null, 15); - - } - - @Test - public void testLastNMultipleCodes() { - - // Two code parameters. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - ReferenceParam subjectParam3 = new ReferenceParam("Patient", "", patient2Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2, subjectParam3)); - TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); TokenParam codeParam2 = new TokenParam(codeSystem, observationCd1); params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd1); - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - sortedPatients.add(patient2Id.getValue()); - - executeTestCase(params, sortedPatients, sortedObservationCodes, null, 75); - - } - - @Test - public void testLastNSinglePatientCategoryCode() { - - // One patient, category and code. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam = new ReferenceParam("Patient", "", patient0Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - TokenParam code = new TokenParam(codeSystem, observationCd0); - params.add(Observation.SP_CODE, buildTokenAndListParam(code)); - TokenParam category = new TokenParam(categorySystem, categoryCd2); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(category)); - - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - - List myCategories = new ArrayList<>(); - myCategories.add(categoryCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 5); - - } - - @Test - public void testLastNMultiplePatientsCategoriesCodes() { - - // Two patients, categories and codes. - SearchParameterMap params = new SearchParameterMap(); - ReferenceParam subjectParam1 = new ReferenceParam("Patient", "", patient0Id.getValue()); - ReferenceParam subjectParam2 = new ReferenceParam("Patient", "", patient1Id.getValue()); - params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam1, subjectParam2)); - List sortedPatients = new ArrayList<>(); - sortedPatients.add(patient0Id.getValue()); - sortedPatients.add(patient1Id.getValue()); - - TokenParam codeParam1 = new TokenParam(codeSystem, observationCd0); - TokenParam codeParam2 = new TokenParam(codeSystem, observationCd2); - params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - List sortedObservationCodes = new ArrayList<>(); - sortedObservationCodes.add(observationCd0); - sortedObservationCodes.add(observationCd2); - - TokenParam categoryParam1 = new TokenParam(categorySystem, categoryCd1); - TokenParam categoryParam2 = new TokenParam(categorySystem, categoryCd2); - params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); - List myCategories = new ArrayList<>(); - myCategories.add(categoryCd1); - myCategories.add(categoryCd2); - - executeTestCase(params, sortedPatients, sortedObservationCodes, myCategories, 30); - - } - - private TokenAndListParam buildTokenAndListParam(TokenParam... theToken) { - TokenOrListParam myTokenOrListParam = new TokenOrListParam(); - for (TokenParam tokenParam : theToken) { - myTokenOrListParam.addOr(tokenParam); - } - return new TokenAndListParam().addAnd(myTokenOrListParam); - } - - @Test - public void testLastNWithChunkedQuery() { - SearchBuilder.setIsTest(true); - Integer numberOfObservations = SearchBuilder.getMaximumPageSize()+1; - Calendar observationDate = new GregorianCalendar(); - - List myObservationIds = new ArrayList<>(); - List myPatientIds = new ArrayList<>(); - List myPatientReferences = new ArrayList<>(); - for (int idx=0; idx actual; params.setLastN(true); + params.setLastNMax(100); Map requestParameters = new HashMap<>(); - params.setLastNMax(1); - - params.setCount(numberOfObservations); - when(mySrd.getParameters()).thenReturn(requestParameters); - myCaptureQueriesListener.clear(); - actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + // Set chunk size to 50 + SearchBuilder.setIsTest(true); + myCaptureQueriesListener.clear(); + List results = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); + assertEquals(75, results.size()); myCaptureQueriesListener.logSelectQueriesForCurrentThread(); List queries = myCaptureQueriesListener .getSelectQueriesForCurrentThread() @@ -587,22 +58,29 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest { .map(t -> t.getSql(true, false)) .collect(Collectors.toList()); - // First chunked query - String resultingQueryNotFormatted = queries.get(0); - assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '[0-9]+' , '[0-9]+' , '[0-9]+'\\).*")); + // Two chunked queries executed by the QueryIterator (in current thread) and two chunked queries to retrieve resources by PID. + assertEquals(4, queries.size()); - // Second chunked query chunk - resultingQueryNotFormatted = queries.get(1); - assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '-1' , '-1' , '-1'\\).*")); - - assertEquals(numberOfObservations, (Integer)actual.size()); - for(IIdType observationId : myObservationIds) { - myObservationDao.delete(observationId); + // The first and third chunked queries should have a full complement of PIDs + StringBuilder firstQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'"); + for (int pidIndex = 1; pidIndex<50; pidIndex++) { + firstQueryPattern.append(" , '[0-9]+'"); } + firstQueryPattern.append("\\).*"); + assertThat(queries.get(0), matchesPattern(firstQueryPattern.toString())); + assertThat(queries.get(2), matchesPattern(firstQueryPattern.toString())); - for (IIdType patientId : myPatientIds) { - myPatientDao.delete(patientId); + // the second and fourth chunked queries should be padded with "-1". + StringBuilder secondQueryPattern = new StringBuilder(".*RES_ID in \\('[0-9]+'"); + for (int pidIndex = 1; pidIndex<25; pidIndex++) { + secondQueryPattern.append(" , '[0-9]+'"); } + for (int pidIndex = 0; pidIndex<25; pidIndex++) { + secondQueryPattern.append(" , '-1'"); + } + secondQueryPattern.append("\\).*"); + assertThat(queries.get(1), matchesPattern(secondQueryPattern.toString())); + assertThat(queries.get(3), matchesPattern(secondQueryPattern.toString())); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java index 944dae05e7d..fd1942840da 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java @@ -107,7 +107,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 100); assertEquals(1, observationIdsOnly.size()); assertEquals(SINGLE_OBSERVATION_PID, observationIdsOnly.get(0)); @@ -181,14 +181,14 @@ public class PersistObservationIndexedSearchParamLastNR4IT { SearchParameterMap searchParameterMap = new SearchParameterMap(); searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams); //searchParameterMap. - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(100, observationIdsOnly.size()); // Filter the results by category code. TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 100); assertEquals(50, observationIdsOnly.size()); @@ -277,7 +277,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { SearchParameterMap searchParameterMap = new SearchParameterMap(); searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams); - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(100, observationIdsOnly.size()); assertTrue(observationIdsOnly.contains("55")); @@ -295,7 +295,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { observation = myResourceIndexedObservationLastNDao.findForIdentifier("55"); assertNull(observation); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(99, observationIdsOnly.size()); assertTrue(!observationIdsOnly.contains("55")); @@ -316,7 +316,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(1, observationIdsOnly.size()); assertTrue(observationIdsOnly.contains(SINGLE_OBSERVATION_PID)); @@ -339,7 +339,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { assertEquals(newEffectiveDtm.getValue(), updatedObservationEntity.getEffectiveDtm()); // Repeat earlier Elasticsearch query. This time, should return no matches. - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); assertEquals(0, observationIdsOnly.size()); // Try again with the new patient ID. @@ -348,7 +348,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam))); searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); // Should see the observation returned now. assertEquals(1, observationIdsOnly.size()); @@ -398,11 +398,11 @@ public class PersistObservationIndexedSearchParamLastNR4IT { SearchParameterMap searchParameterMap = new SearchParameterMap(); // execute Observation ID search - Composite Aggregation - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1, 200); assertEquals(20, observationIdsOnly.size()); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 200); assertEquals(38, observationIdsOnly.size()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index 21a7e08b8dc..25215936831 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -81,7 +81,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { subjectParam = new ReferenceParam("Patient", "", "9"); searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); - List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3, 100); assertEquals(60, observations.size()); @@ -153,7 +153,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { TokenParam codeParam2 = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-2"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(20, observations.size()); @@ -165,7 +165,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(20, observations.size()); @@ -198,7 +198,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { TokenParam codeParam = new TokenParam("test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(5, observations.size()); @@ -215,7 +215,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(10, observations.size()); } @@ -232,7 +232,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { codeParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(5, observations.size()); @@ -248,7 +248,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(0, observations.size()); // Invalid subject @@ -259,7 +259,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(0, observations.size()); // Invalid observation code @@ -270,7 +270,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-999"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(0, observations.size()); // Invalid category code @@ -281,7 +281,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); assertEquals(0, observations.size()); } @@ -393,7 +393,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { @Test public void testLastNNoParamsQuery() { SearchParameterMap searchParameterMap = new SearchParameterMap(); - List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 1); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 1, 100); assertEquals(2, observations.size()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java index b78e14821b3..416c949f6ce 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java @@ -103,13 +103,13 @@ public class LastNElasticsearchSvcSingleObservationIT { searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); // execute Observation ID search - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 100); assertEquals(1, observationIdsOnly.size()); assertEquals(RESOURCEPID, observationIdsOnly.get(0)); // execute Observation search for all search fields - List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3, 100); validateFullObservationSearch(observations); } diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java index 858dcc0c7ec..3acb7743771 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java @@ -37,6 +37,7 @@ public class SearchRuntimeDetails { private boolean myLoadSynchronous; private String myQueryString; private SearchStatusEnum mySearchStatus; + private int myFoundIndexMatchesCount; public SearchRuntimeDetails(RequestDetails theRequestDetails, String theSearchUuid) { myRequestDetails = theRequestDetails; mySearchUuid = theSearchUuid; @@ -67,6 +68,14 @@ public class SearchRuntimeDetails { myFoundMatchesCount = theFoundMatchesCount; } + public int getFoundIndexMatchesCount() { + return myFoundIndexMatchesCount; + } + + public void setFoundIndexMatchesCount(int theFoundIndexMatchesCount) { + myFoundIndexMatchesCount = theFoundIndexMatchesCount; + } + public boolean getLoadSynchronous() { return myLoadSynchronous; } From 6a27192a96290fb44040a022657dc62ee292bac7 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 19 May 2020 14:48:26 -0400 Subject: [PATCH 18/31] Fixed sorting with chunked and paged queries. --- hapi-fhir-jpaserver-base/pom.xml | 5 + .../BaseHapiFhirResourceDaoObservation.java | 17 +- .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 48 +--- .../FhirResourceDaoObservationDstu3.java | 22 +- .../dao/r4/FhirResourceDaoObservationR4.java | 20 ++ .../dao/r5/FhirResourceDaoObservationR5.java | 22 +- ...seJpaResourceProviderObservationDstu3.java | 17 +- .../BaseJpaResourceProviderObservationR4.java | 21 +- .../BaseJpaResourceProviderObservationR5.java | 21 +- .../search/lastn/ElasticsearchSvcImpl.java | 210 ++++++++---------- .../jpa/search/lastn/IElasticsearchSvc.java | 3 +- .../fhir/jpa/search/lastn/IndexConstants.java | 16 -- .../ca/uhn/fhir/jpa/config/TestR4Config.java | 6 +- ...bservationIndexedSearchParamLastNR4IT.java | 35 ++- ...lasticsearchSvcMultipleObservationsIT.java | 51 +++-- ...tNElasticsearchSvcSingleObservationIT.java | 31 +-- .../util/LastNParameterHelper.java | 62 ++++++ 17 files changed, 334 insertions(+), 273 deletions(-) delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index a9f0297e403..3f6223d8827 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -585,6 +585,11 @@ 1.0-SNAPSHOT shaded6 + + org.postgresql + postgresql + 42.2.9 + diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java index 391e34db536..ed06b3c4eaf 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java @@ -21,7 +21,6 @@ package ca.uhn.fhir.jpa.dao; */ import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; -import ca.uhn.fhir.jpa.search.lastn.IndexConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.rest.api.*; import ca.uhn.fhir.rest.api.server.*; @@ -35,12 +34,18 @@ public abstract class BaseHapiFhirResourceDaoObservation lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode, theMaximumResults); + List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults); for (String lastnResourceId : lastnResourceIds) { pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); } @@ -422,52 +422,6 @@ public class SearchBuilder implements ISearchBuilder { searchForIdsWithAndOr(myParams, theRequest); } - /* - * Fulltext or lastn search - */ -/* if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { - List pids = new ArrayList<>(); - if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { - if (myFulltextSearchSvc == null) { - if (myParams.containsKey(Constants.PARAM_TEXT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); - } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); - } - } - - if (myParams.getEverythingMode() != null) { - pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); - } else { - pids = myFulltextSearchSvc.search(myResourceName, myParams); - } - } else if (myParams.isLastN()) { - if (myIElasticsearchSvc == null) { - if (myParams.isLastN()) { - throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); - } - } - Integer myMaxObservationsPerCode = null; - if(myParams.getLastNMax() != null) { - myMaxObservationsPerCode = myParams.getLastNMax(); - } else { - throw new InvalidRequestException("Max parameter is required for $lastn operation"); - } - List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); - for (String lastnResourceId : lastnResourceIds) { - pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); - } -// pids = normalizeIdListForLastNInClause(lastnResourceIds); - } - if (pids.isEmpty()) { - // Will never match - pids = Collections.singletonList(new ResourcePersistentId(-1L)); - } - - myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids))); - - } -*/ // Add PID list predicate for full text search and/or lastn operation if (thePidList != null && thePidList.size() > 0) { myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(thePidList)); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java index b147bb62598..c31c77cd7a0 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java @@ -30,8 +30,6 @@ import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; -import ca.uhn.fhir.rest.param.ReferenceAndListParam; -import ca.uhn.fhir.rest.param.TokenAndListParam; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.dstu3.model.Observation; import org.springframework.beans.factory.annotation.Autowired; @@ -52,6 +50,26 @@ public class FhirResourceDaoObservationDstu3 extends BaseHapiFhirResourceDaoObse return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); } + @Override + protected String getEffectiveParamName() { + return Observation.SP_DATE; + } + + @Override + protected String getCodeParamName() { + return Observation.SP_CODE; + } + + @Override + protected String getSubjectParamName() { + return Observation.SP_SUBJECT; + } + + @Override + protected String getPatientParamName() { + return Observation.SP_PATIENT; + } + @Override public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, boolean theUpdateVersion, TransactionDetails theTransactionDetails, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index 3290c91fc38..c25ce16ea54 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -57,6 +57,26 @@ public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDaoObserva return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); } + @Override + protected String getEffectiveParamName() { + return Observation.SP_DATE; + } + + @Override + protected String getCodeParamName() { + return Observation.SP_CODE; + } + + @Override + protected String getSubjectParamName() { + return Observation.SP_SUBJECT; + } + + @Override + protected String getPatientParamName() { + return Observation.SP_PATIENT; + } + @Override public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, boolean theUpdateVersion, TransactionDetails theTransactionDetails, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java index 7a693c91999..8d9b7f2acfe 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java @@ -30,8 +30,6 @@ import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; -import ca.uhn.fhir.rest.param.ReferenceAndListParam; -import ca.uhn.fhir.rest.param.TokenAndListParam; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r5.model.Observation; import org.springframework.beans.factory.annotation.Autowired; @@ -52,6 +50,26 @@ public class FhirResourceDaoObservationR5 extends BaseHapiFhirResourceDaoObserva return mySearchCoordinatorSvc.registerSearch(this, theSearchParameterMap, getResourceName(), new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequestDetails); } + @Override + protected String getEffectiveParamName() { + return Observation.SP_DATE; + } + + @Override + protected String getCodeParamName() { + return Observation.SP_CODE; + } + + @Override + protected String getSubjectParamName() { + return Observation.SP_SUBJECT; + } + + @Override + protected String getPatientParamName() { + return Observation.SP_PATIENT; + } + @Override public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, boolean theUpdateVersion, TransactionDetails theTransactionDetails, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java index a70b7a61d89..4a1e3189a67 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/BaseJpaResourceProviderObservationDstu3.java @@ -70,10 +70,7 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider @Description(shortDefinition="The maximum number of observations to return for each observation code") @OperationParam(name = "max", typeName = "integer", min = 0, max = 1) - IPrimitiveType theMax, - - @Sort - SortSpec theSort + IPrimitiveType theMax ) { startRequest(theServletRequest); @@ -81,17 +78,17 @@ public class BaseJpaResourceProviderObservationDstu3 extends JpaResourceProvider SearchParameterMap paramMap = new SearchParameterMap(); paramMap.add(Observation.SP_CATEGORY, theCategory); paramMap.add(Observation.SP_CODE, theCode); - paramMap.add(Observation.SP_PATIENT, thePatient); - paramMap.add(Observation.SP_SUBJECT, theSubject); + if (thePatient != null) { + paramMap.add("patient", thePatient); + } + if (theSubject != null) { + paramMap.add("subject", theSubject); + } paramMap.setLastNMax(theMax.getValue()); if (theCount != null) { paramMap.setCount(theCount.getValue()); } - if (theSort != null) { - paramMap.setSort(theSort); - } - return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); } finally { endRequest(theServletRequest); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java index a4b81125af9..2ad20aa1974 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/BaseJpaResourceProviderObservationR4.java @@ -69,28 +69,25 @@ public class BaseJpaResourceProviderObservationR4 extends JpaResourceProviderR4< @Description(shortDefinition="The maximum number of observations to return for each observation code") @OperationParam(name = "max", typeName = "integer", min = 0, max = 1) - IPrimitiveType theMax, - - @Sort - SortSpec theSort + IPrimitiveType theMax ) { startRequest(theServletRequest); try { SearchParameterMap paramMap = new SearchParameterMap(); - paramMap.add("category", theCategory); - paramMap.add("code", theCode); - paramMap.add("patient", thePatient); - paramMap.add("subject", theSubject); + paramMap.add(Observation.SP_CATEGORY, theCategory); + paramMap.add(Observation.SP_CODE, theCode); + if (thePatient != null) { + paramMap.add(Observation.SP_PATIENT, thePatient); + } + if (theSubject != null) { + paramMap.add(Observation.SP_SUBJECT, theSubject); + } paramMap.setLastNMax(theMax.getValue()); if (theCount != null) { paramMap.setCount(theCount.getValue()); } - if (theSort != null) { - paramMap.setSort(theSort); - } - return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); } finally { endRequest(theServletRequest); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java index 95e80a775ed..b5cacc7c571 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/BaseJpaResourceProviderObservationR5.java @@ -70,28 +70,25 @@ public class BaseJpaResourceProviderObservationR5 extends JpaResourceProviderR5< @Description(shortDefinition="The maximum number of observations to return for each observation code") @OperationParam(name = "max", typeName = "integer", min = 0, max = 1) - IPrimitiveType theMax, - - @Sort - SortSpec theSort + IPrimitiveType theMax ) { startRequest(theServletRequest); try { SearchParameterMap paramMap = new SearchParameterMap(); - paramMap.add("category", theCategory); - paramMap.add("code", theCode); - paramMap.add("patient", thePatient); - paramMap.add("subject", theSubject); + paramMap.add(Observation.SP_CATEGORY, theCategory); + paramMap.add(Observation.SP_CODE, theCode); + if (thePatient != null) { + paramMap.add(Observation.SP_PATIENT, thePatient); + } + if (theSubject != null) { + paramMap.add(Observation.SP_SUBJECT, theSubject); + } paramMap.setLastNMax(theMax.getValue()); if (theCount != null) { paramMap.setCount(theCount.getValue()); } - if (theSort != null) { - paramMap.setSort(theSort); - } - return ((IFhirResourceDaoObservation) getDao()).observationsLastN(paramMap, theRequestDetails, theServletResponse); } finally { endRequest(theServletRequest); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 49af7d3510d..40655996679 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -1,6 +1,8 @@ package ca.uhn.fhir.jpa.search.lastn; +import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper; import ca.uhn.fhir.model.api.IQueryParameterType; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.TokenParam; @@ -48,13 +50,18 @@ import static org.apache.commons.lang3.StringUtils.isBlank; public class ElasticsearchSvcImpl implements IElasticsearchSvc { + public static final String OBSERVATION_INDEX = "observation_index"; + public static final String CODE_INDEX = "code_index"; + public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; + public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity"; + private final RestHighLevelClient myRestHighLevelClient; private final ObjectMapper objectMapper = new ObjectMapper(); private final String GROUP_BY_SUBJECT = "group_by_subject"; + private final String GROUP_BY_SYSTEM = "group_by_system"; private final String GROUP_BY_CODE = "group_by_code"; - private final String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier"; public ElasticsearchSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { @@ -69,7 +76,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } private void createObservationIndexIfMissing() throws IOException { - if (indexExists(IndexConstants.OBSERVATION_INDEX)) { + if (indexExists(OBSERVATION_INDEX)) { return; } String observationMapping = "{\n" + @@ -124,14 +131,14 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { " }\n" + " }\n" + "}\n"; - if (!createIndex(IndexConstants.OBSERVATION_INDEX, observationMapping)) { + if (!createIndex(OBSERVATION_INDEX, observationMapping)) { throw new RuntimeException("Failed to create observation index"); } } private void createCodeIndexIfMissing() throws IOException { - if (indexExists(IndexConstants.CODE_INDEX)) { + if (indexExists(CODE_INDEX)) { return; } String codeMapping = "{\n" + @@ -161,7 +168,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { " }\n" + " }\n" + "}\n"; - if (!createIndex(IndexConstants.CODE_INDEX, codeMapping)) { + if (!createIndex(CODE_INDEX, codeMapping)) { throw new RuntimeException("Failed to create code index"); } @@ -199,16 +206,18 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } @Override - // TODO: Should eliminate dependency on SearchParameterMap in API. - public List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch) { + public List executeLastN(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, Integer theMaxResultsToFetch) { + String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier"; String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; try { - List responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, topHitsInclude); + List responses = buildAndExecuteSearch(theSearchParameterMap, theFhirContext, topHitsInclude); List observationIds = new ArrayList<>(); for (SearchResponse response : responses) { -// observationIds.addAll(buildObservationIdList(response)); - Integer maxResultsToAdd = theMaxResultsToFetch - observationIds.size(); - observationIds.addAll(buildObservationList(response, t -> t.getIdentifier(), theSearchParameterMap, maxResultsToAdd)); + Integer maxResultsToAdd = null; + if (theMaxResultsToFetch != null) { + maxResultsToAdd = theMaxResultsToFetch - observationIds.size(); + } + observationIds.addAll(buildObservationList(response, ObservationJson::getIdentifier, theSearchParameterMap, theFhirContext, maxResultsToAdd)); } return observationIds; } catch (IOException theE) { @@ -216,23 +225,27 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } } - private List buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, + private List buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, String[] topHitsInclude) { List responses = new ArrayList<>(); - if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { + String patientParamName = LastNParameterHelper.getPatientParamName(theFhirContext); + String subjectParamName = LastNParameterHelper.getSubjectParamName(theFhirContext); + if (theSearchParameterMap.containsKey(patientParamName) + || theSearchParameterMap.containsKey(subjectParamName)) { ArrayList subjectReferenceCriteria = new ArrayList<>(); List> patientParams = new ArrayList<>(); - if (theSearchParameterMap.get(IndexConstants.PATIENT_SEARCH_PARAM) != null) { - patientParams.addAll(theSearchParameterMap.get(IndexConstants.PATIENT_SEARCH_PARAM)); + if (theSearchParameterMap.get(patientParamName) != null) { + patientParams.addAll(theSearchParameterMap.get(patientParamName)); } - if (theSearchParameterMap.get(IndexConstants.SUBJECT_SEARCH_PARAM) != null) { - patientParams.addAll(theSearchParameterMap.get(IndexConstants.SUBJECT_SEARCH_PARAM)); + if (theSearchParameterMap.get(subjectParamName) != null) { + patientParams.addAll(theSearchParameterMap.get(subjectParamName)); } for (List nextSubjectList : patientParams) { subjectReferenceCriteria.addAll(getReferenceValues(nextSubjectList)); } for (String subject : subjectReferenceCriteria) { - SearchRequest myLastNRequest = buildObservationsSearchRequest(subject, theSearchParameterMap, createCompositeAggregationBuilder(theMaxObservationsPerCode, topHitsInclude)); + SearchRequest myLastNRequest = buildObservationsSearchRequest(subject, theSearchParameterMap, theFhirContext, + createCompositeAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); try { SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); responses.add(lastnResponse); @@ -241,7 +254,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } } } else { - SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, createObservationCodeAggregationBuilder(theMaxObservationsPerCode, topHitsInclude)); + SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, theFhirContext, createObservationCodeAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); try { SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); responses.add(lastnResponse); @@ -254,13 +267,12 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } @VisibleForTesting - // TODO: Should eliminate dependency on SearchParameterMap in API. - List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch) { + List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { try { - List responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, null); + List responses = buildAndExecuteSearch(theSearchParameterMap, theFhirContext, null); List observationDocuments = new ArrayList<>(); for (SearchResponse response : responses) { - observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap, theMaxResultsToFetch)); + observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap, theFhirContext, 100)); } return observationDocuments; } catch (IOException theE) { @@ -269,20 +281,15 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } @VisibleForTesting - List queryAllIndexedObservationCodes(int theMaxResultSetSize) throws IOException { - SearchRequest codeSearchRequest = buildObservationCodesSearchRequest(theMaxResultSetSize); - SearchResponse codeSearchResponse = executeSearchRequest(codeSearchRequest); - return buildCodeResult(codeSearchResponse); - } - - private SearchRequest buildObservationCodesSearchRequest(int theMaxResultSetSize) { - SearchRequest searchRequest = new SearchRequest(IndexConstants.CODE_INDEX); + List queryAllIndexedObservationCodes() throws IOException { + SearchRequest codeSearchRequest = new SearchRequest(CODE_INDEX); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // Query searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - searchSourceBuilder.size(theMaxResultSetSize); - searchRequest.source(searchSourceBuilder); - return searchRequest; + searchSourceBuilder.size(1000); + codeSearchRequest.source(searchSourceBuilder); + SearchResponse codeSearchResponse = executeSearchRequest(codeSearchRequest); + return buildCodeResult(codeSearchResponse); } private CompositeAggregationBuilder createCompositeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { @@ -297,51 +304,27 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } private TermsAggregationBuilder createObservationCodeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { - TermsAggregationBuilder observationCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE, ValueType.STRING).field("codeconceptid"); + TermsAggregationBuilder observationCodeCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE, ValueType.STRING).field("codeconceptcodingcode"); // Top Hits Aggregation - observationCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + observationCodeCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") .sort("effectivedtm", SortOrder.DESC) .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); - observationCodeAggregationBuilder.size(10000); - return observationCodeAggregationBuilder; + observationCodeCodeAggregationBuilder.size(10000); + TermsAggregationBuilder observationCodeSystemAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_SYSTEM, ValueType.STRING).field("codeconceptcodingsystem"); + observationCodeSystemAggregationBuilder.subAggregation(observationCodeCodeAggregationBuilder); + return observationCodeSystemAggregationBuilder; } - public SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { + private SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); } - private List buildObservationIdList(SearchResponse theSearchResponse) throws IOException { - List theObservationList = new ArrayList<>(); - for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { - for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) { - for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { - String indexedObservation = lastNMatch.getSourceAsString(); - ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); - theObservationList.add(observationJson.getIdentifier()); - } - } - } - return theObservationList; - } - - private List buildObservationDocumentList(SearchResponse theSearchResponse) throws IOException { - List theObservationList = new ArrayList<>(); - for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { - for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) { - for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { - String indexedObservation = lastNMatch.getSourceAsString(); - ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); - theObservationList.add(observationJson); - } - } - } - return theObservationList; - } - private List buildObservationList(SearchResponse theSearchResponse, Function setValue, - SearchParameterMap theSearchParameterMap, Integer theMaxResultsToFetch) throws IOException { + SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, + Integer theMaxResultsToFetch) throws IOException { List theObservationList = new ArrayList<>(); - if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { + if (theSearchParameterMap.containsKey(LastNParameterHelper.getPatientParamName(theFhirContext)) + || theSearchParameterMap.containsKey(LastNParameterHelper.getSubjectParamName(theFhirContext))) { for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { break; @@ -387,14 +370,23 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { private List getObservationCodeBuckets(SearchResponse theSearchResponse) { Aggregations responseAggregations = theSearchResponse.getAggregations(); - ParsedTerms aggregatedObservationCodes = responseAggregations.get(GROUP_BY_CODE); - return aggregatedObservationCodes.getBuckets(); + return getObservationCodeBuckets(responseAggregations); } private List getObservationCodeBuckets(ParsedComposite.ParsedBucket theSubjectBucket) { - Aggregations observationCodeAggregations = theSubjectBucket.getAggregations(); - ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get(GROUP_BY_CODE); - return aggregatedObservationCodes.getBuckets(); + Aggregations observationCodeSystemAggregations = theSubjectBucket.getAggregations(); + return getObservationCodeBuckets(observationCodeSystemAggregations); + } + + private List getObservationCodeBuckets(Aggregations theObservationCodeSystemAggregations) { + List retVal = new ArrayList<>(); + ParsedTerms aggregatedObservationCodeSystems = theObservationCodeSystemAggregations.get(GROUP_BY_SYSTEM); + for(Terms.Bucket observationCodeSystem : aggregatedObservationCodeSystems.getBuckets()) { + Aggregations observationCodeCodeAggregations = observationCodeSystem.getAggregations(); + ParsedTerms aggregatedObservationCodeCodes = observationCodeCodeAggregations.get(GROUP_BY_CODE); + retVal.addAll(aggregatedObservationCodeCodes.getBuckets()); + } + return retVal; } private SearchHit[] getLastNMatches(Terms.Bucket theObservationCodeBucket) { @@ -413,17 +405,16 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return codes; } - private SearchRequest buildObservationsSearchRequest(SearchParameterMap theSearchParameterMap, AggregationBuilder theAggregationBuilder) { - SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + private SearchRequest buildObservationsSearchRequest(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, AggregationBuilder theAggregationBuilder) { + SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // Query - if (!searchParamsHaveLastNCriteria(theSearchParameterMap)) { + if (!searchParamsHaveLastNCriteria(theSearchParameterMap, theFhirContext)) { searchSourceBuilder.query(QueryBuilders.matchAllQuery()); } else { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - addSubjectsCriteria(boolQueryBuilder, theSearchParameterMap); - addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap); - addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap); + addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); + addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); searchSourceBuilder.query(boolQueryBuilder); } searchSourceBuilder.size(0); @@ -435,14 +426,15 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return searchRequest; } - private SearchRequest buildObservationsSearchRequest(String theSubjectParam, SearchParameterMap theSearchParameterMap, AggregationBuilder theAggregationBuilder) { - SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + private SearchRequest buildObservationsSearchRequest(String theSubjectParam, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, + AggregationBuilder theAggregationBuilder) { + SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // Query BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.must(QueryBuilders.termQuery("subject", theSubjectParam)); - addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap); - addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap); + addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); + addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); searchSourceBuilder.query(boolQueryBuilder); searchSourceBuilder.size(0); @@ -453,30 +445,12 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return searchRequest; } - private Boolean searchParamsHaveLastNCriteria(SearchParameterMap theSearchParameterMap) { + private Boolean searchParamsHaveLastNCriteria(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { return theSearchParameterMap != null && - (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM) || - theSearchParameterMap.containsKey(IndexConstants.CATEGORY_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.CODE_SEARCH_PARAM)); - } - - private void addSubjectsCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { - if (theSearchParameterMap.containsKey(IndexConstants.PATIENT_SEARCH_PARAM) || theSearchParameterMap.containsKey(IndexConstants.SUBJECT_SEARCH_PARAM)) { - ArrayList subjectReferenceCriteria = new ArrayList<>(); - List> andOrParams = new ArrayList<>(); - if (theSearchParameterMap.get(IndexConstants.PATIENT_SEARCH_PARAM) != null) { - andOrParams.addAll(theSearchParameterMap.get(IndexConstants.PATIENT_SEARCH_PARAM)); - } - if (theSearchParameterMap.get(IndexConstants.SUBJECT_SEARCH_PARAM) != null) { - andOrParams.addAll(theSearchParameterMap.get(IndexConstants.SUBJECT_SEARCH_PARAM)); - } - for (List nextAnd : andOrParams) { - subjectReferenceCriteria.addAll(getReferenceValues(nextAnd)); - } - if (subjectReferenceCriteria.size() > 0) { - theBoolQueryBuilder.must(QueryBuilders.termsQuery("subject", subjectReferenceCriteria)); - } - } - + (theSearchParameterMap.containsKey(LastNParameterHelper.getPatientParamName(theFhirContext)) + || theSearchParameterMap.containsKey(LastNParameterHelper.getSubjectParamName(theFhirContext)) + || theSearchParameterMap.containsKey(LastNParameterHelper.getCategoryParamName(theFhirContext)) + || theSearchParameterMap.containsKey(LastNParameterHelper.getCodeParamName(theFhirContext))); } private List getReferenceValues(List referenceParams) { @@ -496,13 +470,14 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return referenceList; } - private void addCategoriesCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { - if (theSearchParameterMap.containsKey(IndexConstants.CATEGORY_SEARCH_PARAM)) { + private void addCategoriesCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { + String categoryParamName = LastNParameterHelper.getCategoryParamName(theFhirContext); + if (theSearchParameterMap.containsKey(categoryParamName)) { ArrayList codeSystemHashList = new ArrayList<>(); ArrayList codeOnlyList = new ArrayList<>(); ArrayList systemOnlyList = new ArrayList<>(); ArrayList textOnlyList = new ArrayList<>(); - List> andOrParams = theSearchParameterMap.get(IndexConstants.CATEGORY_SEARCH_PARAM); + List> andOrParams = theSearchParameterMap.get(categoryParamName); for (List nextAnd : andOrParams) { codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); @@ -593,13 +568,14 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return textOnlyList; } - private void addObservationCodeCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { - if (theSearchParameterMap.containsKey(IndexConstants.CODE_SEARCH_PARAM)) { + private void addObservationCodeCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { + String codeParamName = LastNParameterHelper.getCodeParamName(theFhirContext); + if (theSearchParameterMap.containsKey(codeParamName)) { ArrayList codeSystemHashList = new ArrayList<>(); ArrayList codeOnlyList = new ArrayList<>(); ArrayList systemOnlyList = new ArrayList<>(); ArrayList textOnlyList = new ArrayList<>(); - List> andOrParams = theSearchParameterMap.get(IndexConstants.CODE_SEARCH_PARAM); + List> andOrParams = theSearchParameterMap.get(codeParamName); for (List nextAnd : andOrParams) { codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); @@ -634,12 +610,4 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); } -/* public void deleteObservationIndex(String theObservationIdentifier) throws IOException { - DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(IndexConstants.OBSERVATION_DOCUMENT_TYPE); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_IDENTIFIER_FIELD_NAME, theObservationIdentifier)); - deleteByQueryRequest.setQuery(boolQueryBuilder); - myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); - } - */ } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java index 592b6939d6d..f994dc7cf18 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -1,9 +1,10 @@ package ca.uhn.fhir.jpa.search.lastn; +import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import java.util.List; public interface IElasticsearchSvc { - List executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode, Integer theMaxResultsToFetch); + List executeLastN(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, Integer theMaxResultsToFetch); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java deleted file mode 100644 index 535d299fd9f..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java +++ /dev/null @@ -1,16 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn; - -public class IndexConstants { - - // TODO: These should all be moved into ElasticSearchSvcImpl. - public static final String OBSERVATION_INDEX = "observation_index"; - public static final String CODE_INDEX = "code_index"; - public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; - public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity"; - - public static final String SUBJECT_SEARCH_PARAM = "subject"; - public static final String PATIENT_SEARCH_PARAM = "patient"; - public static final String CODE_SEARCH_PARAM = "code"; - public static final String CATEGORY_SEARCH_PARAM = "category"; - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java index dfa4c95bd21..48d2a12d247 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java @@ -104,12 +104,12 @@ public class TestR4Config extends BaseJavaConfigR4 { retVal.setDriver(new org.h2.Driver()); // retVal.setDriver(new org.postgresql.Driver()); retVal.setUrl("jdbc:h2:mem:testdb_r4"); -// retVal.setUrl("jdbc:postgresql://localhost:5432/hapi"); +// retVal.setUrl("jdbc:postgresql://localhost:5432/cdr"); retVal.setMaxWaitMillis(10000); retVal.setUsername(""); -// retVal.setUsername("hapi"); +// retVal.setUsername("cdr"); retVal.setPassword(""); -// retVal.setPassword("HapiFHIR"); +// retVal.setPassword("SmileCDR"); retVal.setMaxTotal(ourMaxThreads); SLF4JLogLevel level = SLF4JLogLevel.INFO; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java index fd1942840da..bafdfe53fee 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java @@ -60,6 +60,9 @@ public class PersistObservationIndexedSearchParamLastNR4IT { @Autowired ObservationLastNIndexPersistSvc testObservationPersist; + @Autowired + protected FhirContext myFhirCtx; + @Before public void before() { @@ -106,8 +109,9 @@ public class PersistObservationIndexedSearchParamLastNR4IT { searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); + searchParameterMap.setLastNMax(3); - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 100); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 100); assertEquals(1, observationIdsOnly.size()); assertEquals(SINGLE_OBSERVATION_PID, observationIdsOnly.get(0)); @@ -180,15 +184,18 @@ public class PersistObservationIndexedSearchParamLastNR4IT { // Check that all observations were indexed. SearchParameterMap searchParameterMap = new SearchParameterMap(); searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams); - //searchParameterMap. - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); + + searchParameterMap.setLastNMax(10); + + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 200); assertEquals(100, observationIdsOnly.size()); // Filter the results by category code. TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 100); + + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 100); assertEquals(50, observationIdsOnly.size()); @@ -277,7 +284,8 @@ public class PersistObservationIndexedSearchParamLastNR4IT { SearchParameterMap searchParameterMap = new SearchParameterMap(); searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams); - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); + searchParameterMap.setLastNMax(10); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 200); assertEquals(100, observationIdsOnly.size()); assertTrue(observationIdsOnly.contains("55")); @@ -295,7 +303,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { observation = myResourceIndexedObservationLastNDao.findForIdentifier("55"); assertNull(observation); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 200); assertEquals(99, observationIdsOnly.size()); assertTrue(!observationIdsOnly.contains("55")); @@ -316,7 +324,9 @@ public class PersistObservationIndexedSearchParamLastNR4IT { searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); + searchParameterMap.setLastNMax(10); + + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 200); assertEquals(1, observationIdsOnly.size()); assertTrue(observationIdsOnly.contains(SINGLE_OBSERVATION_PID)); @@ -339,7 +349,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { assertEquals(newEffectiveDtm.getValue(), updatedObservationEntity.getEffectiveDtm()); // Repeat earlier Elasticsearch query. This time, should return no matches. - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 200); assertEquals(0, observationIdsOnly.size()); // Try again with the new patient ID. @@ -348,7 +358,8 @@ public class PersistObservationIndexedSearchParamLastNR4IT { searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam))); searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam))); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 10, 200); + searchParameterMap.setLastNMax(10); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 200); // Should see the observation returned now. assertEquals(1, observationIdsOnly.size()); @@ -398,11 +409,13 @@ public class PersistObservationIndexedSearchParamLastNR4IT { SearchParameterMap searchParameterMap = new SearchParameterMap(); // execute Observation ID search - Composite Aggregation - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,1, 200); + searchParameterMap.setLastNMax(1); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap,myFhirCtx, 200); assertEquals(20, observationIdsOnly.size()); - observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 200); + searchParameterMap.setLastNMax(3); + observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 200); assertEquals(38, observationIdsOnly.size()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index 25215936831..4b16c31c095 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -1,5 +1,6 @@ package ca.uhn.fhir.jpa.search.lastn; +import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.rest.param.*; import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchConfig; @@ -21,7 +22,6 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; import java.util.*; -import static ca.uhn.fhir.jpa.search.lastn.IndexConstants.*; import static org.junit.Assert.*; @RunWith(SpringJUnit4ClassRunner.class) @@ -35,6 +35,8 @@ public class LastNElasticsearchSvcMultipleObservationsIT { private final Map>> createdPatientObservationMap = new HashMap<>(); + private FhirContext myFhirContext = FhirContext.forR4(); + @BeforeClass public static void beforeClass() { @@ -51,8 +53,8 @@ public class LastNElasticsearchSvcMultipleObservationsIT { @After public void after() throws IOException { - elasticsearchSvc.deleteAllDocuments(OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(CODE_INDEX); + elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.CODE_INDEX); } @Test @@ -80,8 +82,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); subjectParam = new ReferenceParam("Patient", "", "9"); searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam)); + searchParameterMap.setLastNMax(3); - List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3, 100); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, myFhirContext); assertEquals(60, observations.size()); @@ -152,8 +155,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { TokenParam codeParam1 = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); TokenParam codeParam2 = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-2"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + searchParameterMap.setLastNMax(100); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(20, observations.size()); @@ -164,8 +168,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_PATIENT, buildReferenceAndListParam(patientParam1, patientParam2)); searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam1, categoryParam2)); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam1, codeParam2)); + searchParameterMap.setLastNMax(100); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(20, observations.size()); @@ -197,8 +202,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); TokenParam codeParam = new TokenParam("test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.setLastNMax(100); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(5, observations.size()); @@ -214,8 +220,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.setLastNMax(100); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(10, observations.size()); } @@ -231,8 +238,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { TokenParam codeParam = new TokenParam("test-code-1 display"); codeParam.setModifier(TokenParamModifier.TEXT); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); + searchParameterMap.setLastNMax(100); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); + List observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(5, observations.size()); @@ -248,7 +256,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - List observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); + searchParameterMap.setLastNMax(100); + + List observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(0, observations.size()); // Invalid subject @@ -259,7 +269,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(0, observations.size()); // Invalid observation code @@ -270,7 +282,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-999"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(0, observations.size()); // Invalid category code @@ -281,7 +295,9 @@ public class LastNElasticsearchSvcMultipleObservationsIT { searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam)); codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1"); searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(codeParam)); - observations = elasticsearchSvc.executeLastN(searchParameterMap, 100, 100); + searchParameterMap.setLastNMax(100); + + observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(0, observations.size()); } @@ -341,12 +357,12 @@ public class LastNElasticsearchSvcMultipleObservationsIT { observationJson.setCategories(categoryConcepts1); observationJson.setCode(codeableConceptField1); observationJson.setCode_concept_id(codeableConceptId1); - assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId1, codeJson1Document, CODE_DOCUMENT_TYPE)); + assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.CODE_INDEX, codeableConceptId1, codeJson1Document, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); } else { observationJson.setCategories(categoryConcepts2); observationJson.setCode(codeableConceptField2); observationJson.setCode_concept_id(codeableConceptId2); - assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId2, codeJson2Document, CODE_DOCUMENT_TYPE)); + assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.CODE_INDEX, codeableConceptId2, codeJson2Document, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); } Calendar observationDate = new GregorianCalendar(); @@ -355,7 +371,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { observationJson.setEffectiveDtm(effectiveDtm); String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationJson); - assertTrue(elasticsearchSvc.performIndex(OBSERVATION_INDEX, identifier, observationDocument, OBSERVATION_DOCUMENT_TYPE)); + assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX, identifier, observationDocument, ElasticsearchSvcImpl.OBSERVATION_DOCUMENT_TYPE)); if (createdPatientObservationMap.containsKey(subject)) { Map> observationCodeMap = createdPatientObservationMap.get(subject); @@ -393,7 +409,8 @@ public class LastNElasticsearchSvcMultipleObservationsIT { @Test public void testLastNNoParamsQuery() { SearchParameterMap searchParameterMap = new SearchParameterMap(); - List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 1, 100); + searchParameterMap.setLastNMax(1); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, myFhirContext); assertEquals(2, observations.size()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java index 416c949f6ce..e220bf96569 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java @@ -1,5 +1,6 @@ package ca.uhn.fhir.jpa.search.lastn; +import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchConfig; import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; @@ -67,12 +68,14 @@ public class LastNElasticsearchSvcSingleObservationIT { final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; final String CODEFIRSTCODINGCODE = "test-code"; final String CODEFIRSTCODINGDISPLAY = "test-code display"; - final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; - final String CODESECONDCODINGCODE = "test-alt-code"; - final String CODESECONDCODINGDISPLAY = "test-alt-code display"; - final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; - final String CODETHIRDCODINGCODE = "test-second-alt-code"; - final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; +// final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; +// final String CODESECONDCODINGCODE = "test-alt-code"; +// final String CODESECONDCODINGDISPLAY = "test-alt-code display"; +// final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; +// final String CODETHIRDCODINGCODE = "test-second-alt-code"; +// final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; + + final FhirContext myFhirContext = FhirContext.forR4(); @BeforeClass public static void beforeClass() { @@ -85,8 +88,8 @@ public class LastNElasticsearchSvcSingleObservationIT { @After public void after() throws IOException { - elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.CODE_INDEX); } @Test @@ -102,14 +105,16 @@ public class LastNElasticsearchSvcSingleObservationIT { TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam))); + searchParameterMap.setLastNMax(3); + // execute Observation ID search - List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, 3, 100); + List observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100); assertEquals(1, observationIdsOnly.size()); assertEquals(RESOURCEPID, observationIdsOnly.get(0)); // execute Observation search for all search fields - List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, 3, 100); + List observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, myFhirContext); validateFullObservationSearch(observations); } @@ -243,7 +248,7 @@ public class LastNElasticsearchSvcSingleObservationIT { assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash); // Retrieve all Observation codes - List codes = elasticsearchSvc.queryAllIndexedObservationCodes(1000); + List codes = elasticsearchSvc.queryAllIndexedObservationCodes(); assertEquals(1, codes.size()); CodeJson persistedObservationCode = codes.get(0); @@ -331,11 +336,11 @@ public class LastNElasticsearchSvcSingleObservationIT { indexedObservation.setCode(codeableConceptField); String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); - assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); + assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX, RESOURCEPID, observationDocument, ElasticsearchSvcImpl.OBSERVATION_DOCUMENT_TYPE)); CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); - assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); + assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); try { Thread.sleep(1000L); diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java index c999194e926..9792226d71b 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java @@ -35,4 +35,66 @@ public class LastNParameterHelper { throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString()); } } + + public static String getSubjectParamName(FhirContext theContext) { + if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) { + return org.hl7.fhir.r5.model.Observation.SP_SUBJECT; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.R4) { + return org.hl7.fhir.r4.model.Observation.SP_SUBJECT; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) { + return org.hl7.fhir.dstu3.model.Observation.SP_SUBJECT; + } else { + throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString()); + } + } + + public static String getPatientParamName(FhirContext theContext) { + if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) { + return org.hl7.fhir.r5.model.Observation.SP_PATIENT; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.R4) { + return org.hl7.fhir.r4.model.Observation.SP_PATIENT; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) { + return org.hl7.fhir.dstu3.model.Observation.SP_PATIENT; + } else { + throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString()); + } + } + + public static String getEffectiveParamName(FhirContext theContext) { + if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) { + return org.hl7.fhir.r5.model.Observation.SP_DATE; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.R4) { + return org.hl7.fhir.r4.model.Observation.SP_DATE; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) { + return org.hl7.fhir.dstu3.model.Observation.SP_DATE; + } else { + throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString()); + } + } + + public static String getCategoryParamName(FhirContext theContext) { + if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) { + return org.hl7.fhir.r5.model.Observation.SP_CATEGORY; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.R4) { + return org.hl7.fhir.r4.model.Observation.SP_CATEGORY; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) { + return org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY; + } else { + throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString()); + } + } + + public static String getCodeParamName(FhirContext theContext) { + if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) { + return org.hl7.fhir.r5.model.Observation.SP_CODE; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.R4) { + return org.hl7.fhir.r4.model.Observation.SP_CODE; + } else if (theContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) { + return org.hl7.fhir.dstu3.model.Observation.SP_CODE; + } else { + throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString()); + } + } + + } From 3b7f106052dba7ced263e9c889a158e1c3929b78 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 19 May 2020 16:34:05 -0400 Subject: [PATCH 19/31] Fixed latest merge conflicts. --- .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 107 +++++++++++++----- 1 file changed, 78 insertions(+), 29 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 4aea0cce29e..f52f3119558 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -81,7 +81,6 @@ import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.util.StopWatch; import ca.uhn.fhir.util.UrlUtil; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; import org.apache.commons.lang3.Validate; import org.hibernate.ScrollMode; import org.hibernate.ScrollableResults; @@ -283,8 +282,81 @@ public class SearchBuilder implements ISearchBuilder { myRequestPartitionId = theRequestPartitionId; } + private List> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, + SearchRuntimeDetails theSearchRuntimeDetails) { - private TypedQuery createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) { + List pids = new ArrayList<>(); + + /* + * Fulltext or lastn search + */ + if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { + if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { + if (myFulltextSearchSvc == null) { + if (myParams.containsKey(Constants.PARAM_TEXT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + } + } + + if (myParams.getEverythingMode() != null) { + pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); + } else { + pids = myFulltextSearchSvc.search(myResourceName, myParams); + } + } else if (myParams.isLastN()) { + if (myIElasticsearchSvc == null) { + if (myParams.isLastN()) { + throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request"); + } + } + if(myParams.getLastNMax() == null) { + throw new InvalidRequestException("Max parameter is required for $lastn operation"); + } + List lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults); + for (String lastnResourceId : lastnResourceIds) { + pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); + } + } + if (theSearchRuntimeDetails != null) { + theSearchRuntimeDetails.setFoundIndexMatchesCount(pids.size()); + HookParams params = new HookParams() + .add(RequestDetails.class, theRequest) + .addIfMatchesType(ServletRequestDetails.class, theRequest) + .add(SearchRuntimeDetails.class, theSearchRuntimeDetails); + JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params); + } + + if (pids.isEmpty()) { + // Will never match + pids = Collections.singletonList(new ResourcePersistentId(-1L)); + } + + } + + ArrayList> myQueries = new ArrayList<>(); + + if (!pids.isEmpty()) { + if (theMaximumResults != null && pids.size() > theMaximumResults) { + pids.subList(0,theMaximumResults-1); + } + new QueryChunker().chunk(ResourcePersistentId.toLongList(pids), t-> doCreateChunkedQueries(t, sort, theCount, theRequest, myQueries)); + } else { + myQueries.add(createChunkedQuery(sort,theMaximumResults, theCount, theRequest, null)); + } + + return myQueries; + } + + private void doCreateChunkedQueries(List thePids, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList> theQueries) { + if(thePids.size() < getMaximumPageSize()) { + normalizeIdListForLastNInClause(thePids); + } + theQueries.add(createChunkedQuery(sort, thePids.size(), theCount, theRequest, thePids)); + } + + private TypedQuery createChunkedQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List thePidList) { /* * Sort * @@ -332,30 +404,9 @@ public class SearchBuilder implements ISearchBuilder { searchForIdsWithAndOr(myParams, theRequest); } - /* - * Fulltext search - */ - if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { - if (myFulltextSearchSvc == null) { - if (myParams.containsKey(Constants.PARAM_TEXT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); - } else if (myParams.containsKey(Constants.PARAM_CONTENT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); - } - } - - List pids; - if (myParams.getEverythingMode() != null) { - pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest); - } else { - pids = myFulltextSearchSvc.search(myResourceName, myParams); - } - if (pids.isEmpty()) { - // Will never match - pids = Collections.singletonList(new ResourcePersistentId(-1L)); - } - - myQueryStack.addPredicate(myQueryStack.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids))); + // Add PID list predicate for full text search and/or lastn operation + if (thePidList != null && thePidList.size() > 0) { + myQueryStack.addPredicate(myQueryStack.get("myId").as(Long.class).in(thePidList)); } // Last updated @@ -644,9 +695,7 @@ public class SearchBuilder implements ISearchBuilder { } List pids = new ArrayList<>(thePids); - new QueryChunker().chunk(pids, t -> { - doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position, theDetails); - }); + new QueryChunker().chunk(pids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position)); } From 645a0d3681bc088c8040ddb19989f1f9961fe820 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 19 May 2020 16:37:40 -0400 Subject: [PATCH 20/31] Removing extra file that was mistakenly committed. --- DeleteConflictService_fix.patch | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 DeleteConflictService_fix.patch diff --git a/DeleteConflictService_fix.patch b/DeleteConflictService_fix.patch deleted file mode 100644 index c88dbd45c52..00000000000 --- a/DeleteConflictService_fix.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java -index e575041cd9..93e364bc93 100644 ---- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java -+++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java -@@ -49,8 +49,8 @@ import java.util.List; - public class DeleteConflictService { - private static final Logger ourLog = LoggerFactory.getLogger(DeleteConflictService.class); - public static final int FIRST_QUERY_RESULT_COUNT = 1; -- public static final int RETRY_QUERY_RESULT_COUNT = 60; -- public static final int MAX_RETRY_ATTEMPTS = 10; -+ public static final int RETRY_QUERY_RESULT_COUNT = 100; -+ public static final int MAX_RETRY_ATTEMPTS = 100; - - @Autowired - DeleteConflictFinderService myDeleteConflictFinderService; From 816f409a8a8ce0931cb586e28cd4628e1d40fdf8 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Mon, 25 May 2020 11:37:01 -0400 Subject: [PATCH 21/31] Re-factor lastn entities to locate them is same package as other Search Parameter entities. --- .../src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java | 3 +-- .../java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java | 2 +- .../dao/{lastn => }/ObservationLastNIndexPersistSvc.java | 8 ++++++-- ...servationIndexedCodeCodeableConceptSearchParamDao.java | 2 +- .../data/IObservationIndexedCodeCodingSearchParamDao.java | 2 +- .../dao/data/IObservationIndexedSearchParamLastNDao.java | 2 +- .../jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java | 2 +- .../uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java | 2 +- .../uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java | 2 +- .../uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java | 8 ++++---- .../r4/PersistObservationIndexedSearchParamLastNR4IT.java | 6 +++--- .../ObservationIndexedCategoryCodeableConceptEntity.java | 2 +- .../entity/ObservationIndexedCategoryCodingEntity.java | 4 ++-- .../ObservationIndexedCodeCodeableConceptEntity.java | 2 +- .../model}/entity/ObservationIndexedCodeCodingEntity.java | 4 ++-- .../entity/ObservationIndexedSearchParamLastNEntity.java | 2 +- .../java/ca/uhn/fhir/jpa/model}/util/CodeSystemHash.java | 2 +- .../spring/boot/autoconfigure/FhirAutoConfiguration.java | 2 +- 18 files changed, 30 insertions(+), 27 deletions(-) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/{lastn => }/ObservationLastNIndexPersistSvc.java (95%) rename {hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn => hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model}/entity/ObservationIndexedCategoryCodeableConceptEntity.java (96%) rename {hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn => hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model}/entity/ObservationIndexedCategoryCodingEntity.java (89%) rename {hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn => hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model}/entity/ObservationIndexedCodeCodeableConceptEntity.java (98%) rename {hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn => hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model}/entity/ObservationIndexedCodeCodingEntity.java (93%) rename {hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn => hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model}/entity/ObservationIndexedSearchParamLastNEntity.java (98%) rename {hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn => hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model}/util/CodeSystemHash.java (96%) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index 992f9aa8a93..259a04db35c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -355,8 +355,7 @@ public abstract class BaseConfig { public static void configureEntityManagerFactory(LocalContainerEntityManagerFactoryBean theFactory, FhirContext theCtx) { theFactory.setJpaDialect(hibernateJpaDialect(theCtx.getLocalizer())); - // TODO: Looking at moving the lastn entities into jpa.model.entity package. Note that moving the lastn entities may require re-building elasticsearch indexes. - theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.dao.lastn.entity"); + theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity"); theFactory.setPersistenceProvider(new HibernatePersistenceProvider()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java index 4bc9abbb563..8f27e2f0a41 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java @@ -23,7 +23,7 @@ package ca.uhn.fhir.jpa.config; import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; import ca.uhn.fhir.context.support.IValidationSupport; import ca.uhn.fhir.jpa.dao.JpaPersistedResourceValidationSupport; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; +import ca.uhn.fhir.jpa.dao.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.term.TermCodeSystemStorageSvcImpl; import ca.uhn.fhir.jpa.term.TermDeferredStorageSvcImpl; import ca.uhn.fhir.jpa.term.TermReindexingSvcImpl; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/ObservationLastNIndexPersistSvc.java similarity index 95% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/ObservationLastNIndexPersistSvc.java index 105fc81efda..488041b90e6 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/ObservationLastNIndexPersistSvc.java @@ -1,10 +1,14 @@ -package ca.uhn.fhir.jpa.dao.lastn; +package ca.uhn.fhir.jpa.dao; import ca.uhn.fhir.context.*; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; -import ca.uhn.fhir.jpa.dao.lastn.entity.*; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCategoryCodeableConceptEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCategoryCodingEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodingEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity; import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken; import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor; import ca.uhn.fhir.jpa.searchparam.extractor.PathAndRef; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java index 2bde707c764..84c6c0e680f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java index d5d46f47e16..c4ca00b856a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodingEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodingEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java index b4a6209795f..5e393b0909d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java index c31c77cd7a0..689c6bc8abb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java @@ -21,7 +21,7 @@ package ca.uhn.fhir.jpa.dao.dstu3; */ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; +import ca.uhn.fhir.jpa.dao.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java index c25ce16ea54..fa00864ebad 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -21,7 +21,7 @@ package ca.uhn.fhir.jpa.dao.r4; */ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; +import ca.uhn.fhir.jpa.dao.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java index 8d9b7f2acfe..4c2d9fd79a7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java @@ -21,7 +21,7 @@ package ca.uhn.fhir.jpa.dao.r5; */ import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; +import ca.uhn.fhir.jpa.dao.ObservationLastNIndexPersistSvc; import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 40655996679..814815e6223 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -52,8 +52,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { public static final String OBSERVATION_INDEX = "observation_index"; public static final String CODE_INDEX = "code_index"; - public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; - public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity"; + public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity"; + public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity"; private final RestHighLevelClient myRestHighLevelClient; @@ -81,7 +81,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } String observationMapping = "{\n" + " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + + " \"ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + " \"properties\" : {\n" + " \"codeconceptid\" : {\n" + " \"type\" : \"keyword\",\n" + @@ -143,7 +143,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } String codeMapping = "{\n" + " \"mappings\" : {\n" + - " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + + " \"ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + " \"properties\" : {\n" + " \"codeable_concept_id\" : {\n" + " \"type\" : \"keyword\",\n" + diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java index bafdfe53fee..1f29f9bf978 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java @@ -4,9 +4,9 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao; import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; -import ca.uhn.fhir.jpa.dao.lastn.ObservationLastNIndexPersistSvc; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; -import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; +import ca.uhn.fhir.jpa.dao.ObservationLastNIndexPersistSvc; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity; +import ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity; import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodeableConceptEntity.java similarity index 96% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java rename to hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodeableConceptEntity.java index 281b5e59b91..d985da9b316 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodeableConceptEntity.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; import org.hibernate.search.annotations.Field; import org.hibernate.search.annotations.IndexedEmbedded; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodingEntity.java similarity index 89% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java rename to hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodingEntity.java index 9bc95ed657c..9773e47a529 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCategoryCodingEntity.java @@ -1,6 +1,6 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; -import ca.uhn.fhir.jpa.dao.lastn.util.CodeSystemHash; +import ca.uhn.fhir.jpa.model.util.CodeSystemHash; import org.hibernate.search.annotations.Analyze; import org.hibernate.search.annotations.Field; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java similarity index 98% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java rename to hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java index 2757173505d..4dea98a0002 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; import org.hibernate.search.annotations.DocumentId; import org.hibernate.search.annotations.Field; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java similarity index 93% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java rename to hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java index ecdbfd6e7e3..5722022af75 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java @@ -1,6 +1,6 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; -import ca.uhn.fhir.jpa.dao.lastn.util.CodeSystemHash; +import ca.uhn.fhir.jpa.model.util.CodeSystemHash; import org.hibernate.search.annotations.Analyze; import org.hibernate.search.annotations.Field; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java similarity index 98% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java rename to hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java index 5fddc842c30..1d0695c1537 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao.lastn.entity; +package ca.uhn.fhir.jpa.model.entity; import org.hibernate.search.annotations.*; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/CodeSystemHash.java similarity index 96% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java rename to hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/CodeSystemHash.java index 6cac1cd3627..a1e4b1fe264 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/CodeSystemHash.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.dao.lastn.util; +package ca.uhn.fhir.jpa.model.util; import ca.uhn.fhir.util.UrlUtil; import com.google.common.base.Charsets; diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/src/main/java/ca/uhn/fhir/spring/boot/autoconfigure/FhirAutoConfiguration.java b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/src/main/java/ca/uhn/fhir/spring/boot/autoconfigure/FhirAutoConfiguration.java index 0b3115e8efa..10b72a1e851 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/src/main/java/ca/uhn/fhir/spring/boot/autoconfigure/FhirAutoConfiguration.java +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/src/main/java/ca/uhn/fhir/spring/boot/autoconfigure/FhirAutoConfiguration.java @@ -170,7 +170,7 @@ public class FhirAutoConfiguration { private ScheduledExecutorService myScheduledExecutorService; @Configuration - @EntityScan(basePackages = {"ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.dao.lastn.entity"}) + @EntityScan(basePackages = {"ca.uhn.fhir.jpa.entity", "ca.uhn.fhir.jpa.model.entity"}) @Import({ SubscriptionChannelConfig.class, SubscriptionProcessorConfig.class, From 1e554731bbbac773667e3f9fd0b630c170d9b065 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Mon, 25 May 2020 18:25:25 -0400 Subject: [PATCH 22/31] Additional cleanup and test fixes. --- hapi-fhir-elasticsearch-6/pom.xml | 309 ++++++++---------- hapi-fhir-jpaserver-base/pom.xml | 19 +- .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 2 +- .../search/lastn/ElasticsearchSvcImpl.java | 195 ++++++----- .../java/ca/uhn/fhir/jpa/util/TestUtil.java | 4 +- .../ca/uhn/fhir/jpa/config/TestR4Config.java | 7 - .../config/TestR4ConfigWithElasticSearch.java | 3 - .../TestR4ConfigWithElasticsearchClient.java | 1 - ...ationIndexedCodeCodeableConceptEntity.java | 10 +- .../ObservationIndexedCodeCodingEntity.java | 4 +- ...ervationIndexedSearchParamLastNEntity.java | 8 +- .../extractor/BaseSearchParamExtractor.java | 124 ++----- 12 files changed, 282 insertions(+), 404 deletions(-) diff --git a/hapi-fhir-elasticsearch-6/pom.xml b/hapi-fhir-elasticsearch-6/pom.xml index c63846bc972..495be468d38 100644 --- a/hapi-fhir-elasticsearch-6/pom.xml +++ b/hapi-fhir-elasticsearch-6/pom.xml @@ -1,183 +1,144 @@ - 4.0.0 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - ca.uhn.hapi.fhir - hapi-fhir-elasticsearch-6 - 1.0-SNAPSHOT + + ca.uhn.hapi.fhir + hapi-fhir + 5.1.0-SNAPSHOT + ../pom.xml + - hapi-fhir-elasticsearch-6 - - http://www.example.com + hapi-fhir-elasticsearch-6 - - UTF-8 - 1.7 - 1.7 - + hapi-fhir-elasticsearch-6 - - - junit - junit - 4.12 - test - - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - 6.5.4 - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.dataformat - * - - - com.github.spullara.mustache.java - compiler - - - com.tdunning - t-digest - - - commons-codec - commons-codec - - - commons-logging - commons-logging - - - net.bytebuddy - byte-buddy - - - net.sf.jopt-simple - jopt-simple - - - org.apache.httpcomponents - * - - - org.apache.lucene - lucene-analyzers-common - - - org.apache.lucene - lucene-backward-codecs - - - org.apache.lucene - lucene-sandbox - - - org.elasticsearch - jna - - - org.hdrhistogram - HdrHistogram - - - org.yaml - snakeyaml - - - - + + UTF-8 + 1.7 + 1.7 + - - - - - - maven-clean-plugin - 3.1.0 - - - - maven-resources-plugin - 3.0.2 - - - maven-compiler-plugin - 3.8.0 - - - maven-surefire-plugin - 2.22.1 - - - maven-jar-plugin - 3.0.2 - - - maven-install-plugin - 2.5.2 - - - maven-deploy-plugin - 2.8.2 - - - - maven-site-plugin - 3.7.1 - - - maven-project-info-reports-plugin - 3.0.0 - - - - - - maven-shade-plugin - 3.2.1 - - - package - - shade - - - true - shaded6 - - - com.carrotsearch.hppc - com.shadehapi.carrotsearch.hppc - - - org.apache.logging.log4j - org.shadehapi.apache.logging.log4j - - - org.apache.lucene - org.shadehapi.apache.lucene - - - org.elasticsearch - org.shadehapi.elasticsearch - - - org.joda - org.shadehapi.joda - - - - - - - - + + + junit + junit + 4.12 + test + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.5.4 + + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.dataformat + * + + + com.github.spullara.mustache.java + compiler + + + com.tdunning + t-digest + + + commons-codec + commons-codec + + + commons-logging + commons-logging + + + net.bytebuddy + byte-buddy + + + net.sf.jopt-simple + jopt-simple + + + org.apache.httpcomponents + * + + + org.apache.lucene + lucene-analyzers-common + + + org.apache.lucene + lucene-backward-codecs + + + org.apache.lucene + lucene-sandbox + + + org.elasticsearch + jna + + + org.hdrhistogram + HdrHistogram + + + org.yaml + snakeyaml + + + + + + + + + maven-shade-plugin + 3.2.1 + + + package + + shade + + + true + shaded6 + + + com.carrotsearch.hppc + com.shadehapi.carrotsearch.hppc + + + org.apache.logging.log4j + org.shadehapi.apache.logging.log4j + + + org.apache.lucene + org.shadehapi.apache.lucene + + + org.elasticsearch + org.shadehapi.elasticsearch + + + org.joda + org.shadehapi.joda + + + + + + + + diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index ab11a7d0598..891079525ac 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -145,6 +145,13 @@ hapi-fhir-validation-resources-r5 ${project.version} + + ca.uhn.hapi.fhir + hapi-fhir-elasticsearch-6 + ${project.version} + shaded6 + + net.ttddyy @@ -587,18 +594,6 @@ org.jetbrains annotations - - ca.uhn.hapi.fhir - hapi-fhir-elasticsearch-6 - 1.0-SNAPSHOT - shaded6 - - - org.postgresql - postgresql - 42.2.9 - - diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index f52f3119558..5bc3325d53d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -224,7 +224,7 @@ public class SearchBuilder implements ISearchBuilder { for (Map.Entry>> nextParamEntry : myParams.entrySet()) { String nextParamName = nextParamEntry.getKey(); if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) { - // Skip parameters for Subject, Patient, Code and Category for LastN + // Skip parameters for Subject, Patient, Code and Category for LastN as these will be filtered by Elasticsearch continue; } List> andOrParams = nextParamEntry.getValue(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index 814815e6223..d653a7f580f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -44,6 +44,7 @@ import org.shadehapi.elasticsearch.search.sort.SortOrder; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.TreeSet; import java.util.function.Function; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -134,7 +135,6 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { if (!createIndex(OBSERVATION_INDEX, observationMapping)) { throw new RuntimeException("Failed to create observation index"); } - } private void createCodeIndexIfMissing() throws IOException { @@ -182,23 +182,6 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } - @VisibleForTesting - boolean performIndex(String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) throws IOException { - IndexResponse indexResponse = myRestHighLevelClient.index(createIndexRequest(theIndexName, theDocumentId, theIndexDocument, theDocumentType), - RequestOptions.DEFAULT); - - return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); - } - - private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { - IndexRequest request = new IndexRequest(theIndexName); - request.id(theDocumentId); - request.type(theDocumentType); - - request.source(theObservationDocument, XContentType.JSON); - return request; - } - private boolean indexExists(String theIndexName) throws IOException { GetIndexRequest request = new GetIndexRequest(); request.indices(theIndexName); @@ -209,90 +192,79 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { public List executeLastN(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, Integer theMaxResultsToFetch) { String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier"; String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; - try { - List responses = buildAndExecuteSearch(theSearchParameterMap, theFhirContext, topHitsInclude); - List observationIds = new ArrayList<>(); - for (SearchResponse response : responses) { - Integer maxResultsToAdd = null; - if (theMaxResultsToFetch != null) { - maxResultsToAdd = theMaxResultsToFetch - observationIds.size(); - } - observationIds.addAll(buildObservationList(response, ObservationJson::getIdentifier, theSearchParameterMap, theFhirContext, maxResultsToAdd)); - } - return observationIds; - } catch (IOException theE) { - throw new InvalidRequestException("Unable to execute LastN request", theE); - } + return buildAndExecuteSearch(theSearchParameterMap, theFhirContext, topHitsInclude, + ObservationJson::getIdentifier, theMaxResultsToFetch); } - private List buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, - String[] topHitsInclude) { - List responses = new ArrayList<>(); + private List buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, + String[] topHitsInclude, Function setValue, Integer theMaxResultsToFetch) { String patientParamName = LastNParameterHelper.getPatientParamName(theFhirContext); String subjectParamName = LastNParameterHelper.getSubjectParamName(theFhirContext); + List searchResults = new ArrayList<>(); if (theSearchParameterMap.containsKey(patientParamName) || theSearchParameterMap.containsKey(subjectParamName)) { - ArrayList subjectReferenceCriteria = new ArrayList<>(); - List> patientParams = new ArrayList<>(); - if (theSearchParameterMap.get(patientParamName) != null) { - patientParams.addAll(theSearchParameterMap.get(patientParamName)); - } - if (theSearchParameterMap.get(subjectParamName) != null) { - patientParams.addAll(theSearchParameterMap.get(subjectParamName)); - } - for (List nextSubjectList : patientParams) { - subjectReferenceCriteria.addAll(getReferenceValues(nextSubjectList)); - } - for (String subject : subjectReferenceCriteria) { + for (String subject : getSubjectReferenceCriteria(patientParamName, subjectParamName, theSearchParameterMap)) { + if (theMaxResultsToFetch != null && searchResults.size() >= theMaxResultsToFetch) { + break; + } SearchRequest myLastNRequest = buildObservationsSearchRequest(subject, theSearchParameterMap, theFhirContext, - createCompositeAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); + createObservationSubjectAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); try { SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); - responses.add(lastnResponse); + searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext, + theMaxResultsToFetch)); } catch (IOException theE) { throw new InvalidRequestException("Unable to execute LastN request", theE); } } } else { - SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, theFhirContext, createObservationCodeAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); + SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, theFhirContext, + createObservationCodeAggregationBuilder(theSearchParameterMap.getLastNMax(), topHitsInclude)); try { SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); - responses.add(lastnResponse); + searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext, + theMaxResultsToFetch)); } catch (IOException theE) { throw new InvalidRequestException("Unable to execute LastN request", theE); } - } - return responses; + return searchResults; } - @VisibleForTesting - List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { - try { - List responses = buildAndExecuteSearch(theSearchParameterMap, theFhirContext, null); - List observationDocuments = new ArrayList<>(); - for (SearchResponse response : responses) { - observationDocuments.addAll(buildObservationList(response, t -> t, theSearchParameterMap, theFhirContext, 100)); + private List getSubjectReferenceCriteria(String thePatientParamName, String theSubjectParamName, SearchParameterMap theSearchParameterMap) { + List subjectReferenceCriteria = new ArrayList<>(); + + List> patientParams = new ArrayList<>(); + if (theSearchParameterMap.get(thePatientParamName) != null) { + patientParams.addAll(theSearchParameterMap.get(thePatientParamName)); + } + if (theSearchParameterMap.get(theSubjectParamName) != null) { + patientParams.addAll(theSearchParameterMap.get(theSubjectParamName)); + } + for (List nextSubjectList : patientParams) { + subjectReferenceCriteria.addAll(getReferenceValues(nextSubjectList)); + } + return subjectReferenceCriteria; + } + + private TreeSet getReferenceValues(List referenceParams) { + TreeSet referenceList = new TreeSet<>(); + + for (IQueryParameterType nextOr : referenceParams) { + + if (nextOr instanceof ReferenceParam) { + ReferenceParam ref = (ReferenceParam) nextOr; + if (isBlank(ref.getChain())) { + referenceList.add(ref.getValue()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting ReferenceParam): " + nextOr.getClass()); } - return observationDocuments; - } catch (IOException theE) { - throw new InvalidRequestException("Unable to execute LastN request", theE); } + return referenceList; } - @VisibleForTesting - List queryAllIndexedObservationCodes() throws IOException { - SearchRequest codeSearchRequest = new SearchRequest(CODE_INDEX); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - // Query - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - searchSourceBuilder.size(1000); - codeSearchRequest.source(searchSourceBuilder); - SearchResponse codeSearchResponse = executeSearchRequest(codeSearchRequest); - return buildCodeResult(codeSearchResponse); - } - - private CompositeAggregationBuilder createCompositeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { + private CompositeAggregationBuilder createObservationSubjectAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); List> compositeAggSubjectSources = new ArrayList(); compositeAggSubjectSources.add(subjectValuesBuilder); @@ -395,16 +367,6 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return parsedTopHits.getHits().getHits(); } - private List buildCodeResult(SearchResponse theSearchResponse) throws JsonProcessingException { - SearchHits codeHits = theSearchResponse.getHits(); - List codes = new ArrayList<>(); - for (SearchHit codeHit : codeHits) { - CodeJson code = objectMapper.readValue(codeHit.getSourceAsString(), CodeJson.class); - codes.add(code); - } - return codes; - } - private SearchRequest buildObservationsSearchRequest(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, AggregationBuilder theAggregationBuilder) { SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -453,23 +415,6 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { || theSearchParameterMap.containsKey(LastNParameterHelper.getCodeParamName(theFhirContext))); } - private List getReferenceValues(List referenceParams) { - ArrayList referenceList = new ArrayList<>(); - - for (IQueryParameterType nextOr : referenceParams) { - - if (nextOr instanceof ReferenceParam) { - ReferenceParam ref = (ReferenceParam) nextOr; - if (isBlank(ref.getChain())) { - referenceList.add(ref.getValue()); - } - } else { - throw new IllegalArgumentException("Invalid token type (expecting ReferenceParam): " + nextOr.getClass()); - } - } - return referenceList; - } - private void addCategoriesCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { String categoryParamName = LastNParameterHelper.getCategoryParamName(theFhirContext); if (theSearchParameterMap.containsKey(categoryParamName)) { @@ -603,6 +548,50 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { } + @VisibleForTesting + List executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { + return buildAndExecuteSearch(theSearchParameterMap, theFhirContext, null, t -> t, 100); + } + + @VisibleForTesting + List queryAllIndexedObservationCodes() throws IOException { + SearchRequest codeSearchRequest = new SearchRequest(CODE_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + // Query + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.size(1000); + codeSearchRequest.source(searchSourceBuilder); + SearchResponse codeSearchResponse = executeSearchRequest(codeSearchRequest); + return buildCodeResult(codeSearchResponse); + } + + private List buildCodeResult(SearchResponse theSearchResponse) throws JsonProcessingException { + SearchHits codeHits = theSearchResponse.getHits(); + List codes = new ArrayList<>(); + for (SearchHit codeHit : codeHits) { + CodeJson code = objectMapper.readValue(codeHit.getSourceAsString(), CodeJson.class); + codes.add(code); + } + return codes; + } + + @VisibleForTesting + boolean performIndex(String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) throws IOException { + IndexResponse indexResponse = myRestHighLevelClient.index(createIndexRequest(theIndexName, theDocumentId, theIndexDocument, theDocumentType), + RequestOptions.DEFAULT); + + return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); + } + + private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { + IndexRequest request = new IndexRequest(theIndexName); + request.id(theDocumentId); + request.type(theDocumentType); + + request.source(theObservationDocument, XContentType.JSON); + return request; + } + @VisibleForTesting void deleteAllDocuments(String theIndexName) throws IOException { DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(theIndexName); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/TestUtil.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/TestUtil.java index c038fdb1399..1bea0ca4b24 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/TestUtil.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/TestUtil.java @@ -131,13 +131,15 @@ public class TestUtil { OneToOne oneToOne = nextField.getAnnotation(OneToOne.class); boolean isOtherSideOfOneToManyMapping = oneToMany != null && isNotBlank(oneToMany.mappedBy()); boolean isOtherSideOfOneToOneMapping = oneToOne != null && isNotBlank(oneToOne.mappedBy()); + boolean isField = nextField.getAnnotation(org.hibernate.search.annotations.Field.class) != null; Validate.isTrue( hasEmbedded || hasColumn || hasJoinColumn || isOtherSideOfOneToManyMapping || isOtherSideOfOneToOneMapping || - hasEmbeddedId, "Non-transient has no @Column or @JoinColumn or @EmbeddedId: " + nextField); + hasEmbeddedId || + isField, "Non-transient has no @Column or @JoinColumn or @EmbeddedId: " + nextField); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java index 48d2a12d247..f479832e04f 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java @@ -102,22 +102,16 @@ public class TestR4Config extends BaseJavaConfigR4 { }; retVal.setDriver(new org.h2.Driver()); -// retVal.setDriver(new org.postgresql.Driver()); retVal.setUrl("jdbc:h2:mem:testdb_r4"); -// retVal.setUrl("jdbc:postgresql://localhost:5432/cdr"); retVal.setMaxWaitMillis(10000); retVal.setUsername(""); -// retVal.setUsername("cdr"); retVal.setPassword(""); -// retVal.setPassword("SmileCDR"); retVal.setMaxTotal(ourMaxThreads); SLF4JLogLevel level = SLF4JLogLevel.INFO; DataSource dataSource = ProxyDataSourceBuilder .create(retVal) -// .logQueryBySlf4j(level, "SQL") .logSlowQueryBySlf4j(10, TimeUnit.SECONDS) -// .countQuery(new ThreadQueryCountHolder()) .beforeQuery(new BlockLargeNumbersOfParamsListener()) .afterQuery(captureQueriesListener()) .afterQuery(new CurrentThreadCaptureQueriesListener()) @@ -149,7 +143,6 @@ public class TestR4Config extends BaseJavaConfigR4 { extraProperties.put("hibernate.show_sql", "false"); extraProperties.put("hibernate.hbm2ddl.auto", "update"); extraProperties.put("hibernate.dialect", H2Dialect.class.getName()); -// extraProperties.put("hibernate.dialect", org.hibernate.dialect.PostgreSQL95Dialect.class.getName()); extraProperties.put("hibernate.search.model_mapping", ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory.class.getName()); extraProperties.put("hibernate.search.default.directory_provider", "local-heap"); extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java index 1b90253383d..ebf3fc9e094 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java @@ -24,9 +24,7 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config { private static final String ELASTIC_VERSION = "6.5.4"; protected final String elasticsearchHost = "localhost"; protected final String elasticsearchUserId = ""; -// protected final String elasticsearchUserId = "elastic"; protected final String elasticsearchPassword = ""; -// protected final String elasticsearchPassword = "changeme"; @Override @@ -36,7 +34,6 @@ public class TestR4ConfigWithElasticSearch extends TestR4Config { // Force elasticsearch to start first int httpPort = embeddedElasticSearch().getHttpPort(); -// int httpPort = 9301; ourLog.info("ElasticSearch started on port: {}", httpPort); new ElasticsearchHibernatePropertiesBuilder() diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java index 607457c8edd..9ff6bc4c491 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticsearchClient.java @@ -10,7 +10,6 @@ public class TestR4ConfigWithElasticsearchClient extends TestR4ConfigWithElastic @Bean() public ElasticsearchSvcImpl myElasticsearchSvc() { int elasticsearchPort = embeddedElasticSearch().getHttpPort(); -// int elasticsearchPort = 9301; return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); } diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java index 4dea98a0002..93d1a916086 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java @@ -10,16 +10,18 @@ import javax.persistence.*; @Entity @Indexed(index = "code_index") @Embeddable -@Table(name = "HFJ_SPIDX_LASTN_CODEABLE_CONCEPT") +@Table(name = "HFJ_SPIDX_LASTN_CODE_CONCEPT") public class ObservationIndexedCodeCodeableConceptEntity { - @Id + public static final int MAX_LENGTH = 200; + + @Id @DocumentId(name = "codeable_concept_id") - @Column(name="CODEABLE_CONCEPT_ID") + @Column(name="CODEABLE_CONCEPT_ID", length = MAX_LENGTH) private String myCodeableConceptId; @Field(name = "text") - @Column(name = "CODEABLE_CONCEPT_TEXT", nullable = true) + @Column(name = "CODEABLE_CONCEPT_TEXT", nullable = true, length = MAX_LENGTH) private String myCodeableConceptText; // TODO: Make coding a Collection. Need to first figure out how to maintain this over time. diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java index 5722022af75..7373f04bc91 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java @@ -11,6 +11,8 @@ import javax.persistence.*; @Table(name = "HFJ_SPIDX_LASTN_CODING") public class ObservationIndexedCodeCodingEntity { + public static final int MAX_LENGTH = 200; + // TODO: Fix this to allow multiple codings for observation code // @Id // @SequenceGenerator(name = "SEQ_CODING_FIELD", sequenceName = "SEQ_CODING_FIELD") @@ -18,7 +20,7 @@ public class ObservationIndexedCodeCodingEntity { // private Long myId; @Id - @Column(name="CODEABLE_CONCEPT_ID") + @Column(name="CODEABLE_CONCEPT_ID", length = MAX_LENGTH) private String myCodeableConceptId; @Field (name = "code", analyze = Analyze.NO) diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java index 1d0695c1537..b99193b653a 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedSearchParamLastNEntity.java @@ -13,6 +13,8 @@ import java.util.*; @Indexed(index = "observation_index") public class ObservationIndexedSearchParamLastNEntity { + public static final int MAX_LENGTH = 200; + @Id @SequenceGenerator(name = "SEQ_LASTN", sequenceName = "SEQ_LASTN") @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_LASTN") @@ -20,7 +22,7 @@ public class ObservationIndexedSearchParamLastNEntity { private Long myId; @Field(name = "subject", analyze = Analyze.NO) - @Column(name = "LASTN_SUBJECT_ID", nullable = true) + @Column(name = "LASTN_SUBJECT_ID", nullable = true, length = MAX_LENGTH) private String mySubject; @ManyToOne(fetch = FetchType.LAZY) @@ -29,7 +31,7 @@ public class ObservationIndexedSearchParamLastNEntity { private ObservationIndexedCodeCodeableConceptEntity myObservationCode; @Field(name = "codeconceptid", analyze = Analyze.NO) - @Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false) + @Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false, length = MAX_LENGTH) private String myCodeNormalizedId; @IndexedEmbedded(depth = 2, prefix = "categoryconcept") @@ -42,7 +44,7 @@ public class ObservationIndexedSearchParamLastNEntity { private Date myEffectiveDtm; @DocumentId(name = "identifier") - @Column(name = "RESOURCE_IDENTIFIER", nullable = false) + @Column(name = "RESOURCE_IDENTIFIER", nullable = false, length = MAX_LENGTH) private String myIdentifier; public ObservationIndexedSearchParamLastNEntity() { diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java index 24cd063e0be..74891a5aa54 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java @@ -165,12 +165,6 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor return new ResourceLinkExtractor(); } - @Override - public PathAndRef extractReferenceLinkFromResource(IBase theValue, String thePath) { - ResourceLinkExtractor extractor = new ResourceLinkExtractor(); - return extractor.get(theValue, thePath); - } - private class ResourceLinkExtractor implements IExtractor { private PathAndRef myPathAndRef = null; @@ -249,16 +243,10 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } } - private List extractParamsAsQueryTokens(RuntimeSearchParam theSearchParam, IBaseResource theResource, IExtractor theExtractor) { - SearchParamSet params = new SearchParamSet<>(); - extractSearchParam(theSearchParam, theResource, theExtractor, params); - return toStringList(params); - } - - private List toStringList(SearchParamSet theParams) { - return theParams.stream() - .map(param -> param.toQueryParameterType().getValueAsQueryToken(myContext)) - .collect(Collectors.toList()); + @Override + public PathAndRef extractReferenceLinkFromResource(IBase theValue, String thePath) { + ResourceLinkExtractor extractor = new ResourceLinkExtractor(); + return extractor.get(theValue, thePath); } @Override @@ -309,6 +297,18 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor .collect(Collectors.toList()); } + private List extractParamsAsQueryTokens(RuntimeSearchParam theSearchParam, IBaseResource theResource, IExtractor theExtractor) { + SearchParamSet params = new SearchParamSet<>(); + extractSearchParam(theSearchParam, theResource, theExtractor, params); + return toStringList(params); + } + + private List toStringList(SearchParamSet theParams) { + return theParams.stream() + .map(param -> param.toQueryParameterType().getValueAsQueryToken(myContext)) + .collect(Collectors.toList()); + } + @Override public SearchParamSet extractSearchParamTokens(IBaseResource theResource) { IExtractor extractor = createTokenExtractor(theResource); @@ -734,6 +734,20 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor return tokenTextIndexingEnabledForSearchParam(myModelConfig, theSearchParam); } + private void addToken_CodeableConcept(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { + List codings = getCodingsFromCodeableConcept(theValue); + for (IBase nextCoding : codings) { + addToken_Coding(theResourceType, theParams, theSearchParam, nextCoding); + } + + if (shouldIndexTextComponentOfToken(theSearchParam)) { + String text = getDisplayTextFromCodeableConcept(theValue); + if (isNotBlank(text)) { + createStringIndexIfNotBlank(theResourceType, theParams, theSearchParam, text); + } + } + } + @Override public List getCodingsFromCodeableConcept(IBase theValue) { String nextType = BaseSearchParamExtractor.this.toRootTypeName(theValue); @@ -754,20 +768,6 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } } - private void addToken_CodeableConcept(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - List codings = getCodingsFromCodeableConcept(theValue); - for (IBase nextCoding : codings) { - addToken_Coding(theResourceType, theParams, theSearchParam, nextCoding); - } - - if (shouldIndexTextComponentOfToken(theSearchParam)) { - String text = getDisplayTextFromCodeableConcept(theValue); - if (isNotBlank(text)) { - createStringIndexIfNotBlank(theResourceType, theParams, theSearchParam, text); - } - } - } - private void addToken_Coding(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { ResourceIndexedSearchParamToken resourceIndexedSearchParamToken = createSearchParamForCoding(theResourceType, theSearchParam, theValue); if (resourceIndexedSearchParamToken != null) { @@ -822,61 +822,6 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } } - private void addDate_Period(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - Date start = extractValueAsDate(myPeriodStartValueChild, theValue); - String startAsString = extractValueAsString(myPeriodStartValueChild, theValue); - Date end = extractValueAsDate(myPeriodEndValueChild, theValue); - String endAsString = extractValueAsString(myPeriodEndValueChild, theValue); - - if (start != null || end != null) { - ResourceIndexedSearchParamDate nextEntity = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), start, startAsString, end, endAsString, startAsString); - theParams.add(nextEntity); - } - } - - private void addDate_Timing(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - List> values = extractValuesAsFhirDates(myTimingEventValueChild, theValue); - - TreeSet dates = new TreeSet<>(); - TreeSet dateStrings = new TreeSet<>(); - String firstValue = null; - String finalValue = null; - for (IPrimitiveType nextEvent : values) { - if (nextEvent.getValue() != null) { - dates.add(nextEvent.getValue()); - if (firstValue == null) { - firstValue = nextEvent.getValueAsString(); - } - finalValue = nextEvent.getValueAsString(); - } - } - - Optional repeat = myTimingRepeatValueChild.getAccessor().getFirstValueOrNull(theValue); - if (repeat.isPresent()) { - Optional bounds = myTimingRepeatBoundsValueChild.getAccessor().getFirstValueOrNull(repeat.get()); - if (bounds.isPresent()) { - String boundsType = toRootTypeName(bounds.get()); - if ("Period".equals(boundsType)) { - Date start = extractValueAsDate(myPeriodStartValueChild, bounds.get()); - Date end = extractValueAsDate(myPeriodEndValueChild, bounds.get()); - String endString = extractValueAsString(myPeriodEndValueChild, bounds.get()); - dates.add(start); - dates.add(end); - //TODO Check if this logic is valid. Does the start of the first period indicate a lower bound?? - if (firstValue == null) { - firstValue = extractValueAsString(myPeriodStartValueChild, bounds.get()); - } - finalValue = endString; - } - } - } - - if (!dates.isEmpty()) { - ResourceIndexedSearchParamDate nextEntity = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), dates.first(), firstValue, dates.last(), finalValue, firstValue); - theParams.add(nextEntity); - } - } - private void addNumber_Duration(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { String system = extractValueAsString(myDurationSystemValueChild, theValue); String code = extractValueAsString(myDurationCodeValueChild, theValue); @@ -1060,15 +1005,6 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor return elementDefinition.getName(); } - @SuppressWarnings("unchecked") - private void addDateTimeTypes(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { - IPrimitiveType nextBaseDateTime = (IPrimitiveType) theValue; - if (nextBaseDateTime.getValue() != null) { - ResourceIndexedSearchParamDate param = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), nextBaseDateTime.getValue(), nextBaseDateTime.getValueAsString(), nextBaseDateTime.getValue(), nextBaseDateTime.getValueAsString(), nextBaseDateTime.getValueAsString()); - theParams.add(param); - } - } - private void addUri_Uri(String theResourceType, Set theParams, RuntimeSearchParam theSearchParam, IBase theValue) { IPrimitiveType value = (IPrimitiveType) theValue; String valueAsString = value.getValueAsString(); From a63d90c1f5c5659eefaa8cbccd32d9bf08a8e1de Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Mon, 25 May 2020 18:36:04 -0400 Subject: [PATCH 23/31] Additional cleanup and test fixes. --- .../extractor/BaseSearchParamExtractor.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java index 74891a5aa54..a4331c1e05f 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java @@ -399,12 +399,6 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor return new DateExtractor(theResource); } - @Override - public Date extractDateFromResource(IBase theValue, String thePath) { - DateExtractor extractor = new DateExtractor("DateType"); - return extractor.get(theValue, thePath).getValueHigh(); - } - private class DateExtractor implements IExtractor { String myResourceType; @@ -508,6 +502,12 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor } } + @Override + public Date extractDateFromResource(IBase theValue, String thePath) { + DateExtractor extractor = new DateExtractor("DateType"); + return extractor.get(theValue, thePath).getValueHigh(); + } + @Override public SearchParamSet extractSearchParamNumber(IBaseResource theResource) { IExtractor extractor = createNumberExtractor(theResource); From b250ac4f3dc151c8a880cc382b239673ee8fe0f5 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Mon, 25 May 2020 18:42:17 -0400 Subject: [PATCH 24/31] Add a change log. --- .../hapi/fhir/changelog/5_1_0/1867-add-lastn-operation.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1867-add-lastn-operation.yaml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1867-add-lastn-operation.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1867-add-lastn-operation.yaml new file mode 100644 index 00000000000..e1ad3ee941b --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1867-add-lastn-operation.yaml @@ -0,0 +1,4 @@ +--- +type: add +issue: 1867 +title: "Initial implementation of lastn operation that uses an Elasticsearch v6 server to index observations." From 5cc77b78d48797949357138b5814153538dfaffd Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 26 May 2020 18:09:54 -0400 Subject: [PATCH 25/31] Fixed problems with results of chunked queries being incorrectly sorted. --- .../BaseHapiFhirResourceDaoObservation.java | 52 +++++++++++++++++++ .../search/lastn/ElasticsearchSvcImpl.java | 14 ++--- .../fhir/jpa/search/lastn/json/CodeJson.java | 2 +- .../lastn/config/TestElasticsearchConfig.java | 2 +- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java index ed06b3c4eaf..d5251ad51b8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java @@ -20,14 +20,27 @@ package ca.uhn.fhir.jpa.dao; * #L% */ +import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.api.IQueryParameterType; import ca.uhn.fhir.rest.api.*; import ca.uhn.fhir.rest.api.server.*; +import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import ca.uhn.fhir.rest.param.ReferenceParam; import org.hl7.fhir.instance.model.api.*; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.ArrayList; +import java.util.List; +import java.util.TreeMap; public abstract class BaseHapiFhirResourceDaoObservation extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { + @Autowired + private IRequestPartitionHelperSvc myRequestPartitionHelperService; + protected void updateSearchParamsForLastn(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails) { if (!isPagingProviderDatabaseBacked(theRequestDetails)) { theSearchParameterMap.setLoadSynchronous(true); @@ -37,12 +50,51 @@ public abstract class BaseHapiFhirResourceDaoObservation orderedSubjectReferenceMap = new TreeMap<>(); + if(theSearchParameterMap.containsKey(getSubjectParamName())) { + + RequestPartitionId requestPartitionId = myRequestPartitionHelperService.determineReadPartitionForRequest(theRequestDetails, getResourceName()); + + List> patientParams = new ArrayList<>(); + if (theSearchParameterMap.get(getPatientParamName()) != null) { + patientParams.addAll(theSearchParameterMap.get(getPatientParamName())); + } + if (theSearchParameterMap.get(getSubjectParamName()) != null) { + patientParams.addAll(theSearchParameterMap.get(getSubjectParamName())); + } + + for (List nextPatientList : patientParams) { + for (IQueryParameterType nextOr : nextPatientList) { + if (nextOr instanceof ReferenceParam) { + ReferenceParam ref = (ReferenceParam) nextOr; + ResourcePersistentId pid = myIdHelperService.resolveResourcePersistentIds(requestPartitionId, ref.getResourceType(), ref.getIdPart()); + orderedSubjectReferenceMap.put(pid.getIdAsLong(), nextOr); + } else { + throw new IllegalArgumentException("Invalid token type (expecting ReferenceParam): " + nextOr.getClass()); + } + } + } + + theSearchParameterMap.remove(getSubjectParamName()); + theSearchParameterMap.remove(getPatientParamName()); + for (Long subjectPid : orderedSubjectReferenceMap.keySet()) { + theSearchParameterMap.add(getSubjectParamName(), orderedSubjectReferenceMap.get(subjectPid)); + } + } + + } + abstract protected String getEffectiveParamName(); abstract protected String getCodeParamName(); abstract protected String getSubjectParamName(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java index d653a7f580f..f389183f6d2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -32,6 +32,7 @@ import org.shadehapi.elasticsearch.search.SearchHits; import org.shadehapi.elasticsearch.search.aggregations.AggregationBuilder; import org.shadehapi.elasticsearch.search.aggregations.AggregationBuilders; import org.shadehapi.elasticsearch.search.aggregations.Aggregations; +import org.shadehapi.elasticsearch.search.aggregations.BucketOrder; import org.shadehapi.elasticsearch.search.aggregations.bucket.composite.*; import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -44,7 +45,6 @@ import org.shadehapi.elasticsearch.search.sort.SortOrder; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.TreeSet; import java.util.function.Function; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -149,9 +149,6 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { " \"type\" : \"keyword\",\n" + " \"store\" : true\n" + " },\n" + - " \"codeable_concept_text\" : {\n" + - " \"type\" : \"text\"\n" + - " },\n" + " \"codingcode\" : {\n" + " \"type\" : \"keyword\"\n" + " },\n" + @@ -163,6 +160,9 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { " },\n" + " \"codingsystem\" : {\n" + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"text\" : {\n" + + " \"type\" : \"text\"\n" + " }\n" + " }\n" + " }\n" + @@ -247,8 +247,8 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { return subjectReferenceCriteria; } - private TreeSet getReferenceValues(List referenceParams) { - TreeSet referenceList = new TreeSet<>(); + private List getReferenceValues(List referenceParams) { + List referenceList = new ArrayList<>(); for (IQueryParameterType nextOr : referenceParams) { @@ -277,12 +277,14 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { private TermsAggregationBuilder createObservationCodeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { TermsAggregationBuilder observationCodeCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE, ValueType.STRING).field("codeconceptcodingcode"); + observationCodeCodeAggregationBuilder.order(BucketOrder.key(true)); // Top Hits Aggregation observationCodeCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") .sort("effectivedtm", SortOrder.DESC) .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); observationCodeCodeAggregationBuilder.size(10000); TermsAggregationBuilder observationCodeSystemAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_SYSTEM, ValueType.STRING).field("codeconceptcodingsystem"); + observationCodeSystemAggregationBuilder.order(BucketOrder.key(true)); observationCodeSystemAggregationBuilder.subAggregation(observationCodeCodeAggregationBuilder); return observationCodeSystemAggregationBuilder; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java index 032279866e2..de140a25f32 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java @@ -27,7 +27,7 @@ public class CodeJson { @JsonProperty(value = "codeable_concept_id", required = false) private String myCodeableConceptId; - @JsonProperty(value = "codeable_concept_text", required = false) + @JsonProperty(value = "text", required = false) private String myCodeableConceptText; @JsonProperty(value = "codingcode", required = false) diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java index 624a05d38b4..85b82b4dd9c 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java @@ -15,7 +15,7 @@ import java.util.concurrent.TimeUnit; @Configuration public class TestElasticsearchConfig { - private final String elasticsearchHost = "127.0.0.1"; + private final String elasticsearchHost = "localhost"; private final String elasticsearchUserId = ""; private final String elasticsearchPassword = ""; From 22af36ae7b9e2daede77fa5cc91fc90142cf76f7 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Wed, 27 May 2020 10:40:29 -0400 Subject: [PATCH 26/31] Conditional update on Partitioned Server fails (#1870) * Fix #1869 * Remove FIXME * Test fix * Test fix --- .../fhir/jpa/api/dao/IFhirResourceDao.java | 3 + .../ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java | 4 +- .../dao/index/DaoSearchParamSynchronizer.java | 22 ++++++-- .../fhir/jpa/dao/r4/PartitioningR4Test.java | 55 +++++++++++++++++++ .../tasks/HapiFhirJpaMigrationTasks.java | 3 - .../ResourceIndexedSearchParamToken.java | 3 +- 6 files changed, 79 insertions(+), 11 deletions(-) diff --git a/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDao.java b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDao.java index 1590b2ef372..210ce1c3c9d 100644 --- a/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDao.java +++ b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDao.java @@ -204,6 +204,9 @@ public interface IFhirResourceDao extends IDao { IBundleProvider search(SearchParameterMap theParams, RequestDetails theRequestDetails, HttpServletResponse theServletResponse); + /** + * Search for IDs for processing a match URLs, etc. + */ Set searchForIds(SearchParameterMap theParams, RequestDetails theRequest); /** diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java index fbd4abdbbea..f785c69223a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java @@ -1221,9 +1221,9 @@ public abstract class BaseHapiFhirDao extends BaseStora @Override public ResourceTable updateInternal(RequestDetails theRequestDetails, T theResource, boolean thePerformIndexing, boolean theForceUpdateVersion, - IBasePersistedResource theEntity2, IIdType theResourceId, IBaseResource theOldResource, TransactionDetails theTransactionDetails) { + IBasePersistedResource theEntity, IIdType theResourceId, IBaseResource theOldResource, TransactionDetails theTransactionDetails) { - ResourceTable entity = (ResourceTable) theEntity2; + ResourceTable entity = (ResourceTable) theEntity; // We'll update the resource ID with the correct version later but for // now at least set it to something useful for the interceptors diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/DaoSearchParamSynchronizer.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/DaoSearchParamSynchronizer.java index 5610817446d..c61f90cde2a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/DaoSearchParamSynchronizer.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/DaoSearchParamSynchronizer.java @@ -35,6 +35,7 @@ import javax.persistence.PersistenceContext; import javax.persistence.PersistenceContextType; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; @Service @@ -67,13 +68,20 @@ public class DaoSearchParamSynchronizer { } private void synchronize(ResourceTable theEntity, AddRemoveCount theAddRemoveCount, Collection theNewParams, Collection theExistingParams) { - for (T next : theNewParams) { + Collection newParams = theNewParams; + for (T next : newParams) { next.setPartitionId(theEntity.getPartitionId()); next.calculateHashes(); } - List paramsToRemove = subtract(theExistingParams, theNewParams); - List paramsToAdd = subtract(theNewParams, theExistingParams); + /* + * HashCodes may have changed as a result of setting the partition ID, so + * create a new set that will reflect the new hashcodes + */ + newParams = new HashSet<>(newParams); + + List paramsToRemove = subtract(theExistingParams, newParams); + List paramsToAdd = subtract(newParams, theExistingParams); tryToReuseIndexEntities(paramsToRemove, paramsToAdd); for (T next : paramsToRemove) { @@ -127,8 +135,12 @@ public class DaoSearchParamSynchronizer { return new ArrayList<>(); } - ArrayList retVal = new ArrayList<>(theSubtractFrom); - retVal.removeAll(theToSubtract); + ArrayList retVal = new ArrayList<>(); + for (T next : theSubtractFrom) { + if (!theToSubtract.contains(next)) { + retVal.add(next); + } + } return retVal; } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningR4Test.java index 12c16199668..d87656d69e7 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningR4Test.java @@ -68,6 +68,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.function.Consumer; +import java.util.stream.Collector; import java.util.stream.Collectors; import static ca.uhn.fhir.jpa.util.TestUtil.sleepAtLeast; @@ -701,6 +702,60 @@ public class PartitioningR4Test extends BaseJpaR4SystemTest { } + @Test + public void testUpdateConditionalInPartition() { + myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.DISABLED); + createRequestId(); + + // Create a resource + addCreatePartition(myPartitionId, myPartitionDate); + addReadPartition(myPartitionId); + Patient p = new Patient(); + p.setActive(false); + p.addIdentifier().setValue("12345"); + Long patientId = myPatientDao.update(p, "Patient?identifier=12345", mySrd).getId().getIdPartAsLong(); + runInTransaction(() -> { + // HFJ_RESOURCE + assertEquals(1, myResourceTableDao.count()); + ResourceTable resourceTable = myResourceTableDao.findById(patientId).orElseThrow(IllegalArgumentException::new); + assertEquals(myPartitionId, resourceTable.getPartitionId().getPartitionId().intValue()); + assertEquals(myPartitionDate, resourceTable.getPartitionId().getPartitionDate()); + + // HFJ_SPIDX_TOKEN + ourLog.info("Tokens:\n * {}", myResourceIndexedSearchParamTokenDao.findAll().stream().map(t->t.toString()).collect(Collectors.joining("\n * "))); + assertEquals(3, myResourceIndexedSearchParamTokenDao.countForResourceId(patientId)); + }); + + // Update that resource + addReadPartition(myPartitionId); + p = new Patient(); + p.setActive(true); + p.addIdentifier().setValue("12345"); + Long patientId2 = myPatientDao.update(p, "Patient?identifier=12345", mySrd).getId().getIdPartAsLong(); + + assertEquals(patientId, patientId2); + + runInTransaction(() -> { + // HFJ_RESOURCE + assertEquals(1, myResourceTableDao.count()); + ResourceTable resourceTable = myResourceTableDao.findById(patientId).orElseThrow(IllegalArgumentException::new); + assertEquals(myPartitionId, resourceTable.getPartitionId().getPartitionId().intValue()); + assertEquals(myPartitionDate, resourceTable.getPartitionId().getPartitionDate()); + + // HFJ_SPIDX_TOKEN + ourLog.info("Tokens:\n * {}", myResourceIndexedSearchParamTokenDao.findAll().stream().map(t->t.toString()).collect(Collectors.joining("\n * "))); + assertEquals(3, myResourceIndexedSearchParamTokenDao.countForResourceId(patientId)); + + // HFJ_RES_VER + int version = 2; + ResourceHistoryTable resVer = myResourceHistoryTableDao.findForIdAndVersionAndFetchProvenance(patientId, version); + assertEquals(myPartitionId, resVer.getPartitionId().getPartitionId().intValue()); + assertEquals(myPartitionDate, resVer.getPartitionId().getPartitionDate()); + + }); + + } + @Test public void testRead_PidId_AllPartitions() { IIdType patientId1 = createPatient(withPartition(1), withActiveTrue()); diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java index ca27cc33568..eae0e28ae63 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java @@ -104,9 +104,6 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks { empiLink.addIndex("20200517.5", "IDX_EMPI_PERSON_TGT").unique(true).withColumns("PERSON_PID", "TARGET_PID"); - // TRM_CONCEPT_PROPERTY -// version.onTable("TRM_CONCEPT_PROPERTY").addIndex("20200523.1", "IDX_CONCEPTPROP_CONCEPTPID").unique(false).withColumns("CONCEPT_PID"); - } protected void init500() { // 20200218 - 20200519 diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamToken.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamToken.java index 9629215e5fd..32477bc4b28 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamToken.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamToken.java @@ -59,7 +59,7 @@ import static org.apache.commons.lang3.StringUtils.trim; @Index(name = "IDX_SP_TOKEN_HASH_S", columnList = "HASH_SYS"), @Index(name = "IDX_SP_TOKEN_HASH_SV", columnList = "HASH_SYS_AND_VALUE"), // TODO PERF change this to: - // @Index(name = "IDX_SP_TOKEN_HASH_V", columnList = "HASH_VALUE,RES_ID"), + // @Index(name = "IDX_SP_TOKEN_HASH_V", columnList = "HASH_VALUE,RES_ID"), @Index(name = "IDX_SP_TOKEN_HASH_V", columnList = "HASH_VALUE"), @Index(name = "IDX_SP_TOKEN_UPDATED", columnList = "SP_UPDATED"), @@ -232,6 +232,7 @@ public class ResourceIndexedSearchParamToken extends BaseResourceIndexedSearchPa b.append(getHashValue()); b.append(getHashSystem()); b.append(getHashSystemAndValue()); + return b.toHashCode(); } From ed1e1037513678d39c5e7b1cbd61ebbaaa4a9496 Mon Sep 17 00:00:00 2001 From: Diederik Muylwyk Date: Wed, 27 May 2020 12:39:17 -0400 Subject: [PATCH 27/31] Duplicate maven-enforcer-plugin removed, executions consolidated. --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index d482ce46a7b..54b4f30d5a4 100644 --- a/pom.xml +++ b/pom.xml @@ -2041,12 +2041,6 @@ - - - - org.apache.maven.plugins - maven-enforcer-plugin - enforce-property From 93a1229789fe920632c002bd51b4cd582f2900ac Mon Sep 17 00:00:00 2001 From: Diederik Muylwyk Date: Wed, 27 May 2020 12:57:11 -0400 Subject: [PATCH 28/31] Removing redundant maven-enforcer-plugin execution. --- pom.xml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pom.xml b/pom.xml index 54b4f30d5a4..c750bb46b09 100644 --- a/pom.xml +++ b/pom.xml @@ -2041,20 +2041,6 @@ - - enforce-property - - enforce - - - - - 11 - - - true - - From 97a1bd40a1b93819dae43d4e070aab211557700f Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Wed, 27 May 2020 19:17:01 -0400 Subject: [PATCH 29/31] Added TLS parameters to Java Mail Sender (#1666) * Added TLS parameters to Java Mail Sender * changelog --- .../5_1_0/1666-add-tls-to-email-sender.yaml | 4 +++ .../deliver/email/JavaMailEmailSender.java | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1666-add-tls-to-email-sender.yaml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1666-add-tls-to-email-sender.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1666-add-tls-to-email-sender.yaml new file mode 100644 index 00000000000..5a8242b2d61 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1666-add-tls-to-email-sender.yaml @@ -0,0 +1,4 @@ +--- +type: add +issue: 1666 +title: "The email sender used by email subscriptions can now be configured with TLS parameters." diff --git a/hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/match/deliver/email/JavaMailEmailSender.java b/hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/match/deliver/email/JavaMailEmailSender.java index feb670436ba..edf2df80854 100644 --- a/hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/match/deliver/email/JavaMailEmailSender.java +++ b/hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/match/deliver/email/JavaMailEmailSender.java @@ -41,6 +41,7 @@ import javax.mail.internet.MimeMessage; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Properties; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.trim; @@ -53,6 +54,7 @@ public class JavaMailEmailSender implements IEmailSender { private JavaMailSenderImpl mySender; private String mySmtpServerUsername; private String mySmtpServerPassword; + private final Properties myJavaMailProperties = new Properties(); public String getSmtpServerHostname() { return mySmtpServerHostname; @@ -92,6 +94,38 @@ public class JavaMailEmailSender implements IEmailSender { mySmtpServerUsername = theSmtpServerUsername; } + /** + * Set the "mail.smtp.auth" Java Mail Property + */ + + public void setAuth(Boolean theAuth) { + myJavaMailProperties.setProperty("mail.smtp.auth", theAuth.toString()); + } + + /** + * Set the "mail.smtp.starttls.enable" Java Mail Property + */ + + public void setStartTlsEnable(Boolean theStartTlsEnable) { + myJavaMailProperties.setProperty("mail.smtp.starttls.enable", theStartTlsEnable.toString()); + } + + /** + * Set the "mail.smtp.starttls.required" Java Mail Property + */ + + public void setStartTlsRequired(Boolean theStartTlsRequired) { + myJavaMailProperties.setProperty("mail.smtp.starttls.required", theStartTlsRequired.toString()); + } + + /** + * Set the "mail.smtp.quitwait" Java Mail Property + */ + + public void setQuitWait(Boolean theQuitWait) { + myJavaMailProperties.setProperty("mail.smtp.quitwait", theQuitWait.toString()); + } + @Override public void send(EmailDetails theDetails) { String subscriptionId = theDetails.getSubscription().toUnqualifiedVersionless().getValue(); @@ -144,6 +178,7 @@ public class JavaMailEmailSender implements IEmailSender { mySender.setUsername(getSmtpServerUsername()); mySender.setPassword(getSmtpServerPassword()); mySender.setDefaultEncoding(Constants.CHARSET_UTF8.name()); + mySender.setJavaMailProperties(myJavaMailProperties); } private static String toTrimmedCommaSeparatedString(List theTo) { From 7f72305f578516227b5e1dcbb0c7955e88038411 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Thu, 28 May 2020 08:56:43 -0400 Subject: [PATCH 30/31] Improve Period range indexing (#1873) * Fix #1871 - Handle period range searches better * Add changelog * FIx tests * Add test logging * Address review comments --- .../1871-improve-date-range-indexing.yaml | 7 + .../uhn/hapi/fhir/docs/server_jpa/schema.md | 69 ++++ .../FhirResourceDaoDstu2SearchNoFtTest.java | 139 ++++++++ .../dao/dstu2/FhirResourceDaoDstu2Test.java | 137 ------- .../FhirResourceDaoDstu3SearchNoFtTest.java | 139 ++++++++ .../dao/dstu3/FhirResourceDaoDstu3Test.java | 137 ------- .../r4/FhirResourceDaoR4SearchNoFtTest.java | 334 +++++++++++++++++- .../FhirResourceDaoR4SearchOptimizedTest.java | 25 +- .../jpa/dao/r4/FhirResourceDaoR4Test.java | 137 ------- .../fhir/jpa/model/entity/ModelConfig.java | 146 +++++++- .../extractor/BaseSearchParamExtractor.java | 10 + 11 files changed, 853 insertions(+), 427 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1871-improve-date-range-indexing.yaml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1871-improve-date-range-indexing.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1871-improve-date-range-indexing.yaml new file mode 100644 index 00000000000..07b3afcc786 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1871-improve-date-range-indexing.yaml @@ -0,0 +1,7 @@ +--- +type: add +issue: 1871 +title: "In the JPA server, when indexing Date SearchParameters where the value being indexed is a + FHIR Period that is missing either a lower bound or an upper bound, a default value representing an + extreme 'beginning of time' or 'end of time' is now used. This allows range searches to return more + accurate results." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/schema.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/schema.md index 3514d0fbd0d..af83007725e 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/schema.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/schema.md @@ -522,3 +522,72 @@ The following columns are common to **all HFJ_SPIDX_xxx tables**. +# HFJ_SPIDX_DATE: Date Search Parameters + +For any FHIR Search Parameter of type *date* that generates a database index, a row in the *HFJ_SPIDX_DATE* table will be created. + +## Columns + +The following columns are common to **all HFJ_SPIDX_xxx tables**. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameRelationshipsDatatypeNullableDescription
SP_VALUE_LOWTimestampNullable + This is the lower bound of the date in question. +
    +
  • For a point in time date to millisecond precision (such as an Instant with a value of 2020-05-26T15:00:00.000) this represents the exact value.
    • +
  • For an instant value with lower precision, this represents the start of the possible range denoted by the value. For example, for a value of 2020-05-26 this represents 2020-05-26T00:00:00.000.
    • +
  • For a Period with a lower (start) value present, this column contains that value.
    • +
  • For a Period with no lower (start) value present, this column contains a timestamp representing the "start of time".
    • +
+
SP_VALUE_HIGHTimestampNullable + This is the upper bound of the date in question. +
    +
  • For a point in time date to millisecond precision (such as an Instant with a value of 2020-05-26T15:00:00.000) this represents the exact value.
    • +
  • For an instant value with lower precision, this represents the start of the possible range denoted by the value. For example, for a value of 2020-05-26 this represents 2020-05-26T23:59:59.999.
    • +
  • For a Period with an upper (end) value present, this column contains that value.
    • +
  • For a Period with no upper (end) value present, this column contains a timestamp representing the "end of time".
    • +
+
SP_VALUE_LOW_DATE_ORDINALIntegerNullable + This column contains the same Timestamp as SP_VALUE_LOW, but truncated to Date precision and formatted as an integer in the format "YYYYMMDD". +
SP_VALUE_HIGH_DATE_ORDINALIntegerNullable + This column contains the same Timestamp as SP_VALUE_HIGH, but truncated to Date precision and formatted as an integer in the format "YYYYMMDD". +
diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchNoFtTest.java index 5a16a2bb67c..b23035d9650 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchNoFtTest.java @@ -432,6 +432,145 @@ public class FhirResourceDaoDstu2SearchNoFtTest extends BaseJpaDstu2Test { } + @Test + public void testDatePeriodParamEndOnly() { + { + Encounter enc = new Encounter(); + enc.addIdentifier().setSystem("testDatePeriodParam").setValue("02"); + enc.getPeriod().getEndElement().setValueAsString("2001-01-02"); + myEncounterDao.create(enc, mySrd); + } + SearchParameterMap params; + List encs; + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); + // encs = toList(ourEncounterDao.search(params)); + // assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + } + + @Test + public void testDatePeriodParamStartAndEnd() { + { + Encounter enc = new Encounter(); + enc.addIdentifier().setSystem("testDatePeriodParam").setValue("03"); + enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); + enc.getPeriod().getEndElement().setValueAsString("2001-01-03"); + myEncounterDao.create(enc, mySrd); + } + + SearchParameterMap params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); + List encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-02", "2001-01-06")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-05")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-05", null)); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + } + + @Test + public void testDatePeriodParamStartOnly() { + { + Encounter enc = new Encounter(); + enc.addIdentifier().setSystem("testDatePeriodParam").setValue("01"); + enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); + myEncounterDao.create(enc, mySrd); + } + + SearchParameterMap params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); + List encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); + params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + } + + @Test public void testSearchCompositeParam() { Observation o1 = new Observation(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java index 2c6fd0baf92..4daf828056a 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java @@ -584,143 +584,6 @@ public class FhirResourceDaoDstu2Test extends BaseJpaDstu2Test { } } - @Test - public void testDatePeriodParamEndOnly() { - { - Encounter enc = new Encounter(); - enc.addIdentifier().setSystem("testDatePeriodParam").setValue("02"); - enc.getPeriod().getEndElement().setValueAsString("2001-01-02"); - myEncounterDao.create(enc, mySrd); - } - SearchParameterMap params; - List encs; - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); - // encs = toList(ourEncounterDao.search(params)); - // assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - } - - @Test - public void testDatePeriodParamStartAndEnd() { - { - Encounter enc = new Encounter(); - enc.addIdentifier().setSystem("testDatePeriodParam").setValue("03"); - enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); - enc.getPeriod().getEndElement().setValueAsString("2001-01-03"); - myEncounterDao.create(enc, mySrd); - } - - SearchParameterMap params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); - List encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-02", "2001-01-06")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-05")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-05", null)); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - } - - @Test - public void testDatePeriodParamStartOnly() { - { - Encounter enc = new Encounter(); - enc.addIdentifier().setSystem("testDatePeriodParam").setValue("01"); - enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); - myEncounterDao.create(enc, mySrd); - } - - SearchParameterMap params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); - List encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); - params.add(Encounter.SP_IDENTIFIER, new IdentifierDt("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - } @Test public void testDeleteFailsIfIncomingLinks() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java index 7cf1475a656..dcb2521d6e6 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java @@ -959,6 +959,145 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test { } + @Test + public void testDatePeriodParamEndOnly() { + { + Encounter enc = new Encounter(); + enc.addIdentifier().setSystem("testDatePeriodParam").setValue("02"); + enc.getPeriod().getEndElement().setValueAsString("2001-01-02"); + myEncounterDao.create(enc, mySrd); + } + SearchParameterMap params; + List encs; + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + // encs = toList(ourEncounterDao.search(params)); + // assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + } + + @Test + public void testDatePeriodParamStartAndEnd() { + { + Encounter enc = new Encounter(); + enc.addIdentifier().setSystem("testDatePeriodParam").setValue("03"); + enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); + enc.getPeriod().getEndElement().setValueAsString("2001-01-03"); + myEncounterDao.create(enc, mySrd); + } + + SearchParameterMap params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + List encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-02", "2001-01-06")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-05")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-05", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + } + + @Test + public void testDatePeriodParamStartOnly() { + { + Encounter enc = new Encounter(); + enc.addIdentifier().setSystem("testDatePeriodParam").setValue("01"); + enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); + myEncounterDao.create(enc, mySrd); + } + + SearchParameterMap params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + List encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + } + + /** * #222 */ diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3Test.java index ad30a24831f..af4fbc697c6 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3Test.java @@ -870,143 +870,6 @@ public class FhirResourceDaoDstu3Test extends BaseJpaDstu3Test { } } - @Test - public void testDatePeriodParamEndOnly() { - { - Encounter enc = new Encounter(); - enc.addIdentifier().setSystem("testDatePeriodParam").setValue("02"); - enc.getPeriod().getEndElement().setValueAsString("2001-01-02"); - myEncounterDao.create(enc, mySrd); - } - SearchParameterMap params; - List encs; - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - // encs = toList(ourEncounterDao.search(params)); - // assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - } - - @Test - public void testDatePeriodParamStartAndEnd() { - { - Encounter enc = new Encounter(); - enc.addIdentifier().setSystem("testDatePeriodParam").setValue("03"); - enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); - enc.getPeriod().getEndElement().setValueAsString("2001-01-03"); - myEncounterDao.create(enc, mySrd); - } - - SearchParameterMap params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - List encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-02", "2001-01-06")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-05")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-05", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - } - - @Test - public void testDatePeriodParamStartOnly() { - { - Encounter enc = new Encounter(); - enc.addIdentifier().setSystem("testDatePeriodParam").setValue("01"); - enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); - myEncounterDao.create(enc, mySrd); - } - - SearchParameterMap params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - List encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - } @Test public void testDeleteFailsIfIncomingLinks() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java index c80a75cb55d..e35bc418ff5 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java @@ -530,7 +530,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { myCaptureQueriesListener.clear(); map = new SearchParameterMap(); map.setLoadSynchronous(true); - map.add(DiagnosticReport.SP_PERFORMER, new ReferenceParam( "CareTeam").setChain(PARAM_TYPE)); + map.add(DiagnosticReport.SP_PERFORMER, new ReferenceParam("CareTeam").setChain(PARAM_TYPE)); results = myDiagnosticReportDao.search(map); ids = toUnqualifiedVersionlessIdValues(results); assertThat(ids.toString(), ids, contains(drId1.getValue())); @@ -1690,6 +1690,338 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { } + + @Test + public void testDateRangeOnPeriod_SearchByDateTime_NoUpperBound() { + Encounter enc = new Encounter(); + enc.getPeriod().getStartElement().setValueAsString("2020-05-26T12:00:00Z"); + String id1 = myEncounterDao.create(enc).getId().toUnqualifiedVersionless().getValue(); + + runInTransaction(() -> { + ourLog.info("Date indexes:\n * {}", myResourceIndexedSearchParamDateDao.findAll().stream().map(t -> t.toString()).collect(Collectors.joining("\n * "))); + }); + + // ge -> above the lower bound + SearchParameterMap map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("ge2020-05-26T13:00:00Z")); + myCaptureQueriesListener.clear(); + IBundleProvider results = myEncounterDao.search(map); + List ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // ge -> Below the lower bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("ge2020-05-26T11:00:00Z")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // le -> above the lower bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("le2020-05-26T13:00:00Z")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // le -> Below the lower bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("le2020-05-26T11:00:00Z")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, empty()); + } + + + @Test + public void testDateRangeOnPeriod_SearchByDate_NoUpperBound() { + Encounter enc = new Encounter(); + enc.getPeriod().getStartElement().setValueAsString("2020-05-26T12:00:00Z"); + String id1 = myEncounterDao.create(enc).getId().toUnqualifiedVersionless().getValue(); + + runInTransaction(() -> { + ourLog.info("Date indexes:\n * {}", myResourceIndexedSearchParamDateDao.findAll().stream().map(t -> t.toString()).collect(Collectors.joining("\n * "))); + }); + + // ge -> above the lower bound + SearchParameterMap map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("ge2020-05-27")); + myCaptureQueriesListener.clear(); + IBundleProvider results = myEncounterDao.search(map); + List ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // ge -> Below the lower bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("ge2020-05-25")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // le -> above the lower bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("le2020-05-27")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // le -> Below the lower bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("le2020-05-25")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, empty()); + } + + + @Test + public void testDateRangeOnPeriod_SearchByDateTime_NoLowerBound() { + Encounter enc = new Encounter(); + enc.getPeriod().getEndElement().setValueAsString("2020-05-26T12:00:00Z"); + String id1 = myEncounterDao.create(enc).getId().toUnqualifiedVersionless().getValue(); + + runInTransaction(() -> { + ourLog.info("Date indexes:\n * {}", myResourceIndexedSearchParamDateDao.findAll().stream().map(t -> t.toString()).collect(Collectors.joining("\n * "))); + }); + + // le -> above the upper bound + SearchParameterMap map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("le2020-05-26T13:00:00Z")); + myCaptureQueriesListener.clear(); + IBundleProvider results = myEncounterDao.search(map); + List ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // le -> Below the upper bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("le2020-05-26T11:00:00Z")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // ge -> above the upper bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("ge2020-05-26T13:00:00Z")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, empty()); + + // ge -> Below the upper bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("ge2020-05-26T11:00:00Z")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + } + + + @Test + public void testDateRangeOnPeriod_SearchByDate_NoLowerBound() { + Encounter enc = new Encounter(); + enc.getPeriod().getEndElement().setValueAsString("2020-05-26T12:00:00Z"); + String id1 = myEncounterDao.create(enc).getId().toUnqualifiedVersionless().getValue(); + + runInTransaction(() -> { + ourLog.info("Date indexes:\n * {}", myResourceIndexedSearchParamDateDao.findAll().stream().map(t -> t.toString()).collect(Collectors.joining("\n * "))); + }); + + // le -> above the upper bound + SearchParameterMap map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("le2020-05-27")); + myCaptureQueriesListener.clear(); + IBundleProvider results = myEncounterDao.search(map); + List ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // le -> Below the upper bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("le2020-05-25")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + + // ge -> above the upper bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("ge2020-05-27")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, empty()); + + // ge -> Below the upper bound + map = SearchParameterMap.newSynchronous(); + map.add(Encounter.SP_DATE, new DateParam("ge2020-05-25")); + myCaptureQueriesListener.clear(); + results = myEncounterDao.search(map); + ids = toUnqualifiedVersionlessIdValues(results); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertThat(ids, contains(id1)); + } + + + @Test + public void testDatePeriodParamEndOnly() { + { + Encounter enc = new Encounter(); + enc.addIdentifier().setSystem("testDatePeriodParam").setValue("02"); + enc.getPeriod().getEndElement().setValueAsString("2001-01-02"); + myEncounterDao.create(enc, mySrd); + } + SearchParameterMap params; + List encs; + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + } + + @Test + public void testDatePeriodParamStartAndEnd() { + { + Encounter enc = new Encounter(); + enc.addIdentifier().setSystem("testDatePeriodParam").setValue("03"); + enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); + enc.getPeriod().getEndElement().setValueAsString("2001-01-03"); + myEncounterDao.create(enc, mySrd); + } + + SearchParameterMap params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + List encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-02", "2001-01-06")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-05")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-05", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + } + + @Test + public void testDatePeriodParamStartOnly() { + { + Encounter enc = new Encounter(); + enc.addIdentifier().setSystem("testDatePeriodParam").setValue("01"); + enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); + myEncounterDao.create(enc, mySrd); + } + + SearchParameterMap params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + List encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(0, encs.size()); + + params = new SearchParameterMap(); + params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); + params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); + encs = toList(myEncounterDao.search(params)); + assertEquals(1, encs.size()); + + } + + /** * See #1174 */ diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java index b5832777f82..e5822137b6b 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java @@ -311,7 +311,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * 20 should be prefetched since that's the initial page size */ - await().until(()-> runInTransaction(()->{ + await().until(() -> runInTransaction(() -> { Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); return search.getNumFound() >= 200; })); @@ -371,8 +371,8 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * 20 should be prefetched since that's the initial page size */ - await().until(()->{ - return runInTransaction(()->{ + await().until(() -> { + return runInTransaction(() -> { return mySearchEntityDao .findByUuidAndFetchIncludes(uuid) .orElseThrow(() -> new InternalErrorException("")) @@ -507,8 +507,8 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * 20 should be prefetched since that's the initial page size */ - await().until(()->{ - return runInTransaction(()->{ + await().until(() -> { + return runInTransaction(() -> { Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); return search.getNumFound() >= 50; }); @@ -547,8 +547,8 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { /* * 20 should be prefetched since that's the initial page size */ - await().until(()->{ - return runInTransaction(()->{ + await().until(() -> { + return runInTransaction(() -> { Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); return search.getNumFound() == 20; }); @@ -611,7 +611,12 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * 20 should be prefetched since that's the initial page size */ - waitForSize(20, () -> runInTransaction(() -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")).getNumFound())); + waitForSize( + 20, + 10000, + () -> runInTransaction(() -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")).getNumFound()), + () -> "Wanted 20: " + runInTransaction(() -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")).toString())); + runInTransaction(() -> { Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(20, search.getNumFound()); @@ -673,7 +678,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { assertEquals("Patient/PT00000", ids.get(0)); assertEquals(1, ids.size()); - await().until(()-> runInTransaction(()-> mySearchEntityDao + await().until(() -> runInTransaction(() -> mySearchEntityDao .findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")) .getStatus() == SearchStatusEnum.FINISHED)); @@ -821,7 +826,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { search.getResources(0, 20); ourLog.info("** Done retrieving resources"); - await().until(()->myCaptureQueriesListener.countSelectQueries() == 4); + await().until(() -> myCaptureQueriesListener.countSelectQueries() == 4); myCaptureQueriesListener.logSelectQueriesForCurrentThread(); assertEquals(4, myCaptureQueriesListener.countSelectQueries()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java index 66ca051ad4c..862ad30b4a5 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java @@ -1118,143 +1118,6 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test { } } - @Test - public void testDatePeriodParamEndOnly() { - { - Encounter enc = new Encounter(); - enc.addIdentifier().setSystem("testDatePeriodParam").setValue("02"); - enc.getPeriod().getEndElement().setValueAsString("2001-01-02"); - myEncounterDao.create(enc, mySrd); - } - SearchParameterMap params; - List encs; - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - // encs = toList(ourEncounterDao.search(params)); - // assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - } - - @Test - public void testDatePeriodParamStartAndEnd() { - { - Encounter enc = new Encounter(); - enc.addIdentifier().setSystem("testDatePeriodParam").setValue("03"); - enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); - enc.getPeriod().getEndElement().setValueAsString("2001-01-03"); - myEncounterDao.create(enc, mySrd); - } - - SearchParameterMap params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - List encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-02", "2001-01-06")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-05")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-05", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "03")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - } - - @Test - public void testDatePeriodParamStartOnly() { - { - Encounter enc = new Encounter(); - enc.addIdentifier().setSystem("testDatePeriodParam").setValue("01"); - enc.getPeriod().getStartElement().setValueAsString("2001-01-02"); - myEncounterDao.create(enc, mySrd); - } - - SearchParameterMap params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - List encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(1, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-01")); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - params = new SearchParameterMap(); - params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-03", null)); - params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "01")); - encs = toList(myEncounterDao.search(params)); - assertEquals(0, encs.size()); - - } @Test public void testDeleteFailsIfIncomingLinks() { diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ModelConfig.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ModelConfig.java index 3f4c5c9fdc1..d417823174d 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ModelConfig.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ModelConfig.java @@ -20,13 +20,17 @@ package ca.uhn.fhir.jpa.model.entity; * #L% */ +import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.Validate; import org.hl7.fhir.dstu2.model.Subscription; +import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.hl7.fhir.r4.model.DateTimeType; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.HashSet; import java.util.Set; @@ -50,6 +54,21 @@ public class ModelConfig { "http://hl7.org/fhir/StructureDefinition/*"))); public static final String DEFAULT_WEBSOCKET_CONTEXT_PATH = "/websocket"; + + /* + *

+ * Note the following database documented limitations: + *

    + *
  • JDBC Timestamp Datatype Low Value -4713 and High Value 9999
    • + *
  • MySQL 8: the range for DATETIME values is '1000-01-01 00:00:00.000000' to '9999-12-31 23:59:59.999999`
    • + *
  • Postgresql 12: Timestamp [without time zone] Low Value 4713 BC and High Value 294276 AD
    • + *
  • Oracle: Timestamp Low Value 4712 BC and High Value 9999 CE
    • + *
  • H2: datetime2 Low Value -4713 and High Value 9999
    • + *
+ *

+ */ + protected static final String DEFAULT_PERIOD_INDEX_START_OF_TIME = "1001-01-01"; + protected static final String DEFAULT_PERIOD_INDEX_END_OF_TIME = "9000-01-01"; /** * update setter javadoc if default changes */ @@ -67,11 +86,15 @@ public class ModelConfig { private boolean myUseOrdinalDatesForDayPrecisionSearches = true; private boolean mySuppressStringIndexingInTokens = false; + private IPrimitiveType myPeriodIndexStartOfTime; + private IPrimitiveType myPeriodIndexEndOfTime; + /** * Constructor */ public ModelConfig() { - super(); + setPeriodIndexStartOfTime(new DateTimeType(DEFAULT_PERIOD_INDEX_START_OF_TIME)); + setPeriodIndexEndOfTime(new DateTimeType(DEFAULT_PERIOD_INDEX_END_OF_TIME)); } /** @@ -373,8 +396,8 @@ public class ModelConfig { /** *

* Should searches use the integer field {@code SP_VALUE_LOW_DATE_ORDINAL} and {@code SP_VALUE_HIGH_DATE_ORDINAL} in - * {@link ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate} when resolving searches where all predicates are using - * precision of {@link ca.uhn.fhir.model.api.TemporalPrecisionEnum#DAY}. + * {@link ResourceIndexedSearchParamDate} when resolving searches where all predicates are using + * precision of {@link TemporalPrecisionEnum#DAY}. *

* For example, if enabled, the search of {@code Observation?date=2020-02-25} will cause the date to be collapsed down to an * integer representing the ordinal date {@code 20200225}. It would then be compared against {@link ResourceIndexedSearchParamDate#getValueLowDateOrdinal()} @@ -392,8 +415,8 @@ public class ModelConfig { /** *

* Should searches use the integer field {@code SP_VALUE_LOW_DATE_ORDINAL} and {@code SP_VALUE_HIGH_DATE_ORDINAL} in - * {@link ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate} when resolving searches where all predicates are using - * precision of {@link ca.uhn.fhir.model.api.TemporalPrecisionEnum#DAY}. + * {@link ResourceIndexedSearchParamDate} when resolving searches where all predicates are using + * precision of {@link TemporalPrecisionEnum#DAY}. *

* For example, if enabled, the search of {@code Observation?date=2020-02-25} will cause the date to be collapsed down to an * ordinal {@code 20200225}. It would then be compared against {@link ResourceIndexedSearchParamDate#getValueLowDateOrdinal()} @@ -417,6 +440,7 @@ public class ModelConfig { *

  • Coding.display
    • *
  • Identifier.use.text
    • * + * * @since 5.0.0 */ public boolean isSuppressStringIndexingInTokens() { @@ -432,12 +456,124 @@ public class ModelConfig { *
  • Coding.display
    • *
  • Identifier.use.text
    • * + * * @since 5.0.0 */ public void setSuppressStringIndexingInTokens(boolean theSuppressStringIndexingInTokens) { mySuppressStringIndexingInTokens = theSuppressStringIndexingInTokens; } + /** + * When indexing a Period (e.g. Encounter.period) where the period has an upper bound + * but not a lower bound, a canned "start of time" value can be used as the lower bound + * in order to allow range searches to correctly identify all values in the range. + *

    + * The default value for this is {@link #DEFAULT_PERIOD_INDEX_START_OF_TIME} which + * is probably good enough for almost any application, but this can be changed if + * needed. + *

    + *

    + * Note the following database documented limitations: + *

      + *
    • JDBC Timestamp Datatype Low Value -4713 and High Value 9999
      • + *
    • MySQL 8: the range for DATETIME values is '1000-01-01 00:00:00.000000' to '9999-12-31 23:59:59.999999`
      • + *
    • Postgresql 12: Timestamp [without time zone] Low Value 4713 BC and High Value 294276 AD
      • + *
    • Oracle: Timestamp Low Value 4712 BC and High Value 9999 CE
      • + *
    • H2: datetime2 Low Value -4713 and High Value 9999
      • + *
    + *

    + * + * @see #getPeriodIndexEndOfTime() + * @since 5.1.0 + */ + public IPrimitiveType getPeriodIndexStartOfTime() { + return myPeriodIndexStartOfTime; + } + + /** + * When indexing a Period (e.g. Encounter.period) where the period has an upper bound + * but not a lower bound, a canned "start of time" value can be used as the lower bound + * in order to allow range searches to correctly identify all values in the range. + *

    + * The default value for this is {@link #DEFAULT_PERIOD_INDEX_START_OF_TIME} which + * is probably good enough for almost any application, but this can be changed if + * needed. + *

    + *

    + * Note the following database documented limitations: + *

      + *
    • JDBC Timestamp Datatype Low Value -4713 and High Value 9999
      • + *
    • MySQL 8: the range for DATETIME values is '1000-01-01 00:00:00.000000' to '9999-12-31 23:59:59.999999`
      • + *
    • Postgresql 12: Timestamp [without time zone] Low Value 4713 BC and High Value 294276 AD
      • + *
    • Oracle: Timestamp Low Value 4712 BC and High Value 9999 CE
      • + *
    • H2: datetime2 Low Value -4713 and High Value 9999
      • + *
    + *

    + * + * @see #getPeriodIndexEndOfTime() + * @since 5.1.0 + */ + public void setPeriodIndexStartOfTime(IPrimitiveType thePeriodIndexStartOfTime) { + Validate.notNull(thePeriodIndexStartOfTime, "thePeriodIndexStartOfTime must not be null"); + myPeriodIndexStartOfTime = thePeriodIndexStartOfTime; + } + + /** + * When indexing a Period (e.g. Encounter.period) where the period has a lower bound + * but not an upper bound, a canned "end of time" value can be used as the upper bound + * in order to allow range searches to correctly identify all values in the range. + *

    + * The default value for this is {@link #DEFAULT_PERIOD_INDEX_START_OF_TIME} which + * is probably good enough for almost any application, but this can be changed if + * needed. + *

    + *

    + * Note the following database documented limitations: + *

      + *
    • JDBC Timestamp Datatype Low Value -4713 and High Value 9999
      • + *
    • MySQL 8: the range for DATETIME values is '1000-01-01 00:00:00.000000' to '9999-12-31 23:59:59.999999`
      • + *
    • Postgresql 12: Timestamp [without time zone] Low Value 4713 BC and High Value 294276 AD
      • + *
    • Oracle: Timestamp Low Value 4712 BC and High Value 9999 CE
      • + *
    • H2: datetime2 Low Value -4713 and High Value 9999
      • + *
    + *

    + * + * @see #getPeriodIndexStartOfTime() + * @since 5.1.0 + */ + public IPrimitiveType getPeriodIndexEndOfTime() { + return myPeriodIndexEndOfTime; + } + + /** + * When indexing a Period (e.g. Encounter.period) where the period has an upper bound + * but not a lower bound, a canned "start of time" value can be used as the lower bound + * in order to allow range searches to correctly identify all values in the range. + *

    + * The default value for this is {@link #DEFAULT_PERIOD_INDEX_START_OF_TIME} which + * is probably good enough for almost any application, but this can be changed if + * needed. + *

    + *

    + * Note the following database documented limitations: + *

      + *
    • JDBC Timestamp Datatype Low Value -4713 and High Value 9999
      • + *
    • MySQL 8: the range for DATETIME values is '1000-01-01 00:00:00.000000' to '9999-12-31 23:59:59.999999`
      • + *
    • Postgresql 12: Timestamp [without time zone] Low Value 4713 BC and High Value 294276 AD
      • + *
    • Oracle: Timestamp Low Value 4712 BC and High Value 9999 CE
      • + *
    • H2: datetime2 Low Value -4713 and High Value 9999
      • + *
    + *

    + * + * @see #getPeriodIndexStartOfTime() + * @since 5.1.0 + */ + public void setPeriodIndexEndOfTime(IPrimitiveType thePeriodIndexEndOfTime) { + Validate.notNull(thePeriodIndexEndOfTime, "thePeriodIndexEndOfTime must not be null"); + myPeriodIndexEndOfTime = thePeriodIndexEndOfTime; + } + + private static void validateTreatBaseUrlsAsLocal(String theUrl) { Validate.notBlank(theUrl, "Base URL must not be null or empty"); diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java index 9c42925f4cf..9ba482d4d36 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java @@ -677,6 +677,16 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor String endAsString = extractValueAsString(myPeriodEndValueChild, theValue); if (start != null || end != null) { + + if (start == null) { + start = myModelConfig.getPeriodIndexStartOfTime().getValue(); + startAsString = myModelConfig.getPeriodIndexStartOfTime().getValueAsString(); + } + if (end == null) { + end = myModelConfig.getPeriodIndexEndOfTime().getValue(); + endAsString = myModelConfig.getPeriodIndexEndOfTime().getValueAsString(); + } + ResourceIndexedSearchParamDate nextEntity = new ResourceIndexedSearchParamDate(myPartitionSettings, theResourceType, theSearchParam.getName(), start, startAsString, end, endAsString, startAsString); theParams.add(nextEntity); } From a4811f150844720f516cf2675e017c507188d0a1 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Fri, 29 May 2020 23:47:44 -0400 Subject: [PATCH 31/31] Changes per code review. --- .../api/dao/IFhirResourceDaoObservation.java | 9 + .../BaseHapiFhirResourceDaoObservation.java | 25 ++ .../dao/ObservationLastNIndexPersistSvc.java | 114 +++++---- .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 8 +- ...vationIndexedCodeCodingSearchParamDao.java | 4 +- ...ObservationIndexedSearchParamLastNDao.java | 2 +- .../FhirResourceDaoObservationDstu3.java | 18 +- .../dao/r4/FhirResourceDaoObservationR4.java | 18 +- .../dao/r5/FhirResourceDaoObservationR5.java | 18 +- .../search/lastn/ElasticsearchSvcImpl.java | 126 +++------ .../jpa/search/lastn/IElasticsearchSvc.java | 8 + .../fhir/jpa/search/lastn/json/CodeJson.java | 2 +- .../search/lastn/json/ObservationJson.java | 239 ++++++++---------- .../jpa/search/lastn/util/CodeSystemHash.java | 33 --- .../lastn/ObservationCodeIndexSchema.json | 26 ++ .../search/lastn/ObservationIndexSchema.json | 50 ++++ .../FhirResourceDaoR4SearchLastNAsyncIT.java | 6 +- .../r4/FhirResourceDaoR4SearchLastNIT.java | 4 +- ...bservationIndexedSearchParamLastNR4IT.java | 24 +- ...lasticsearchSvcMultipleObservationsIT.java | 12 +- ...tNElasticsearchSvcSingleObservationIT.java | 73 ++---- .../lastn/config/TestElasticsearchConfig.java | 5 - ...ationIndexedCodeCodeableConceptEntity.java | 9 - .../ObservationIndexedCodeCodingEntity.java | 43 ++-- .../util/LastNParameterHelper.java | 59 +++-- 25 files changed, 428 insertions(+), 507 deletions(-) delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java create mode 100644 hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json create mode 100644 hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json diff --git a/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDaoObservation.java index fa723120ce6..7f01da8bba2 100644 --- a/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDaoObservation.java +++ b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/dao/IFhirResourceDaoObservation.java @@ -29,6 +29,15 @@ import javax.servlet.http.HttpServletResponse; public interface IFhirResourceDaoObservation extends IFhirResourceDao { + /** + * Returns a BundleProvider which can be used to implement the $lastn operation. + * @param paramMap Parameters supported include Observation.subject, Observation.patient, Observation.code, + * Observation.category, and max (the maximum number of Observations to return per specified subjects/patients, + * codes, and/or categories. + * @param theRequestDetails + * @param theServletResponse + * @return + */ IBundleProvider observationsLastN(SearchParameterMap paramMap, RequestDetails theRequestDetails, HttpServletResponse theServletResponse); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java index d5251ad51b8..98c12a49e11 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java @@ -22,25 +22,50 @@ package ca.uhn.fhir.jpa.dao; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation; +import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.IQueryParameterType; import ca.uhn.fhir.rest.api.*; import ca.uhn.fhir.rest.api.server.*; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; import ca.uhn.fhir.rest.param.ReferenceParam; import org.hl7.fhir.instance.model.api.*; import org.springframework.beans.factory.annotation.Autowired; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.TreeMap; public abstract class BaseHapiFhirResourceDaoObservation extends BaseHapiFhirResourceDao implements IFhirResourceDaoObservation { + @Autowired + ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; + @Autowired private IRequestPartitionHelperSvc myRequestPartitionHelperService; + protected ResourceTable updateObservationEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, + Date theDeletedTimestampOrNull, boolean thePerformIndexing, boolean theUpdateVersion, + TransactionDetails theTransactionDetails, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { + ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, + theTransactionDetails, theForceUpdate, theCreateNewHistoryEntry); + + if (!retVal.isUnchangedInCurrentOperation()) { + if (retVal.getDeleted() == null) { + // Update indexes here for LastN operation. + myObservationLastNIndexPersistSvc.indexObservation(theResource); + } else { + myObservationLastNIndexPersistSvc.deleteObservationIndex(theEntity); + } + } + + return retVal; + } + protected void updateSearchParamsForLastn(SearchParameterMap theSearchParameterMap, RequestDetails theRequestDetails) { if (!isPagingProviderDatabaseBacked(theRequestDetails)) { theSearchParameterMap.setLoadSynchronous(true); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/ObservationLastNIndexPersistSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/ObservationLastNIndexPersistSvc.java index 488041b90e6..087489d8836 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/ObservationLastNIndexPersistSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/ObservationLastNIndexPersistSvc.java @@ -39,40 +39,46 @@ public class ObservationLastNIndexPersistSvc { public void indexObservation(IBaseResource theResource) { - String subjectId = null; List subjectReferenceElement = mySearchParameterExtractor.extractValues("Observation.subject", theResource); - if (subjectReferenceElement.size() == 1) { - PathAndRef subjectPathAndRef = mySearchParameterExtractor.extractReferenceLinkFromResource(subjectReferenceElement.get(0), "Observation.subject"); - if (subjectPathAndRef != null) { - IBaseReference subjectReference = subjectPathAndRef.getRef(); - if (subjectReference != null) { - subjectId = subjectReference.getReferenceElement().getValue(); - } - } - } + String subjectId = subjectReferenceElement.stream() + .map(refElement -> mySearchParameterExtractor.extractReferenceLinkFromResource(refElement, "Observation.subject")) + .filter(Objects::nonNull) + .map(PathAndRef::getRef) + .filter(Objects::nonNull) + .map(subjectRef -> subjectRef.getReferenceElement().getValue()) + .filter(Objects::nonNull) + .findFirst().orElse(null); Date effectiveDtm = null; List effectiveDateElement = mySearchParameterExtractor.extractValues("Observation.effective", theResource); - if (effectiveDateElement.size() == 1) { + if (effectiveDateElement.size() > 0) { effectiveDtm = mySearchParameterExtractor.extractDateFromResource(effectiveDateElement.get(0), "Observation.effective"); } - // Build CodeableConcept entity for Observation.Code. List observationCodeCodeableConcepts = mySearchParameterExtractor.extractValues("Observation.code", theResource); - - // Only index for lastn if Observation has a subject, effective date/time and code - if (subjectId == null || effectiveDtm == null || observationCodeCodeableConcepts.size() == 0) { + // Only index for lastn if Observation has a code + if (observationCodeCodeableConcepts.size() == 0) { return; } + List observationCategoryCodeableConcepts = mySearchParameterExtractor.extractValues("Observation.category", theResource); + String resourcePID = theResource.getIdElement().getIdPart(); + createOrUpdateIndexedObservation(resourcePID, effectiveDtm, subjectId, observationCodeCodeableConcepts, observationCategoryCodeableConcepts); + + } + + private void createOrUpdateIndexedObservation(String resourcePID, Date theEffectiveDtm, String theSubjectId, + List theObservationCodeCodeableConcepts, + List theObservationCategoryCodeableConcepts) { + // Determine if an index already exists for Observation: boolean observationIndexUpdate = false; ObservationIndexedSearchParamLastNEntity indexedObservation = null; if (resourcePID != null) { - indexedObservation = myResourceIndexedObservationLastNDao.findForIdentifier(resourcePID); + indexedObservation = myResourceIndexedObservationLastNDao.findByIdentifier(resourcePID); } if (indexedObservation == null) { indexedObservation = new ObservationIndexedSearchParamLastNEntity(); @@ -80,27 +86,46 @@ public class ObservationLastNIndexPersistSvc { observationIndexUpdate = true; } - indexedObservation.setEffectiveDtm(effectiveDtm); + indexedObservation.setEffectiveDtm(theEffectiveDtm); indexedObservation.setIdentifier(resourcePID); - indexedObservation.setSubject(subjectId); + indexedObservation.setSubject(theSubjectId); + addCodeToObservationIndex(theObservationCodeCodeableConcepts, indexedObservation); + addCategoriesToObservationIndex(theObservationCategoryCodeableConcepts, indexedObservation); + + if (observationIndexUpdate) { + myEntityManager.merge(indexedObservation); + } else { + myEntityManager.persist(indexedObservation); + } + + } + + private void addCodeToObservationIndex(List theObservationCodeCodeableConcepts, + ObservationIndexedSearchParamLastNEntity theIndexedObservation) { // Determine if a Normalized ID was created previously for Observation Code - boolean observationCodeUpdate = false; - String observationCodeNormalizedId = getCodeCodeableConceptIdIfExists(observationCodeCodeableConcepts.get(0)); - if (observationCodeNormalizedId != null) { - observationCodeUpdate = true; - } - // Generate a new a normalized ID if necessary - if (observationCodeNormalizedId == null) { - observationCodeNormalizedId = UUID.randomUUID().toString(); - } + Optional existingObservationCodeNormalizedId = getCodeCodeableConceptIdIfExists(theObservationCodeCodeableConcepts.get(0)); // Create/update normalized Observation Code index record - ObservationIndexedCodeCodeableConceptEntity codeableConceptField = getCodeCodeableConcept(observationCodeCodeableConcepts.get(0), observationCodeNormalizedId); + ObservationIndexedCodeCodeableConceptEntity codeableConceptField = + getCodeCodeableConcept(theObservationCodeCodeableConcepts.get(0), + existingObservationCodeNormalizedId.orElse(UUID.randomUUID().toString())); + if (existingObservationCodeNormalizedId.isPresent()) { + myEntityManager.merge(codeableConceptField); + } else { + myEntityManager.persist(codeableConceptField); + } + + theIndexedObservation.setObservationCode(codeableConceptField); + theIndexedObservation.setCodeNormalizedId(codeableConceptField.getCodeableConceptId()); + + } + + private void addCategoriesToObservationIndex(List observationCategoryCodeableConcepts, + ObservationIndexedSearchParamLastNEntity indexedObservation) { // Build CodeableConcept entities for Observation.Category - List observationCategoryCodeableConcepts = mySearchParameterExtractor.extractValues("Observation.category", theResource); Set categoryCodeableConceptEntities = new HashSet<>(); for (IBase categoryCodeableConcept : observationCategoryCodeableConcepts) { // Build CodeableConcept entities for each category CodeableConcept @@ -108,20 +133,6 @@ public class ObservationLastNIndexPersistSvc { } indexedObservation.setCategoryCodeableConcepts(categoryCodeableConceptEntities); - if (observationCodeUpdate) { - myEntityManager.merge(codeableConceptField); - } else { - myEntityManager.persist(codeableConceptField); - } - - indexedObservation.setObservationCode(codeableConceptField); - indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); - if (observationIndexUpdate) { - myEntityManager.merge(indexedObservation); - } else { - myEntityManager.persist(indexedObservation); - } - } private ObservationIndexedCategoryCodeableConceptEntity getCategoryCodeableConceptEntities(IBase theValue) { @@ -151,29 +162,32 @@ public class ObservationLastNIndexPersistSvc { return codeCodeableConcept; } - private String getCodeCodeableConceptIdIfExists(IBase theValue) { + private Optional getCodeCodeableConceptIdIfExists(IBase theValue) { List codings = mySearchParameterExtractor.getCodingsFromCodeableConcept(theValue); String codeCodeableConceptId = null; + Optional codeCodeableConceptIdOptional = Optional.empty(); + for (IBase nextCoding : codings) { ResourceIndexedSearchParamToken param = mySearchParameterExtractor.createSearchParamForCoding("Observation", - new RuntimeSearchParam(null, null, "code", null, null, null, null, null, null, null), + new RuntimeSearchParam(null, null, "code", null, null, null, + null, null, null, null), nextCoding); if (param != null) { String system = param.getSystem(); String code = param.getValue(); String text = mySearchParameterExtractor.getDisplayTextForCoding(nextCoding); if (code != null && system != null) { - codeCodeableConceptId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(code, system); + codeCodeableConceptIdOptional = Optional.ofNullable(myObservationIndexedCodeCodingSearchParamDao.findByCodeAndSystem(code, system)); } else { - codeCodeableConceptId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(text); + codeCodeableConceptIdOptional = Optional.ofNullable(myObservationIndexedCodeCodingSearchParamDao.findByDisplay(text)); } - if (codeCodeableConceptId != null) { + if (codeCodeableConceptIdOptional.isPresent()) { break; } } } - return codeCodeableConceptId; + return codeCodeableConceptIdOptional; } private ObservationIndexedCategoryCodingEntity getCategoryCoding(IBase theValue) { @@ -205,7 +219,7 @@ public class ObservationLastNIndexPersistSvc { } public void deleteObservationIndex(IBasePersistedResource theEntity) { - ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findForIdentifier(theEntity.getIdDt().getIdPart()); + ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findByIdentifier(theEntity.getIdDt().getIdPart()); if (deletedObservationLastNEntity != null) { myEntityManager.remove(deletedObservationLastNEntity); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 5bc3325d53d..ad463873cbe 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -131,7 +131,7 @@ public class SearchBuilder implements ISearchBuilder { // NB: keep public public static final int MAXIMUM_PAGE_SIZE = 800; public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50; - public static boolean myIsTest = false; + public static boolean myUseMaxPageSize50ForTest = false; private static final List EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<>()); private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class); @@ -184,15 +184,15 @@ public class SearchBuilder implements ISearchBuilder { } public static int getMaximumPageSize() { - if (myIsTest) { + if (myUseMaxPageSize50ForTest) { return MAXIMUM_PAGE_SIZE_FOR_TESTING; } else { return MAXIMUM_PAGE_SIZE; } } - public static void setIsTest(boolean theIsTest) { - myIsTest = theIsTest; + public static void setMaxPageSize50ForTest(boolean theIsTest) { + myUseMaxPageSize50ForTest = theIsTest; } @Override diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java index c4ca00b856a..a437b7a7624 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java @@ -14,13 +14,13 @@ public interface IObservationIndexedCodeCodingSearchParamDao extends JpaReposito "WHERE t.myCode = :code " + "AND t.mySystem = :system " + "") - String findForCodeAndSystem(@Param("code") String theCode, @Param("system") String theSystem); + String findByCodeAndSystem(@Param("code") String theCode, @Param("system") String theSystem); @Query("" + "SELECT t.myCodeableConceptId FROM ObservationIndexedCodeCodingEntity t " + "WHERE t.myDisplay = :display" + "") - String findForDisplay(@Param("display") String theDisplay); + String findByDisplay(@Param("display") String theDisplay); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java index 5e393b0909d..9781b7b14f9 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java @@ -12,6 +12,6 @@ public interface IObservationIndexedSearchParamLastNDao extends JpaRepository executeLastN(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, Integer theMaxResultsToFetch) { - String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier"; String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; return buildAndExecuteSearch(theSearchParameterMap, theFhirContext, topHitsInclude, ObservationJson::getIdentifier, theMaxResultsToFetch); @@ -557,7 +499,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc { @VisibleForTesting List queryAllIndexedObservationCodes() throws IOException { - SearchRequest codeSearchRequest = new SearchRequest(CODE_INDEX); + SearchRequest codeSearchRequest = new SearchRequest(OBSERVATION_CODE_INDEX); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // Query searchSourceBuilder.query(QueryBuilders.matchAllQuery()); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java index f994dc7cf18..6a94e3d2a0c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -6,5 +6,13 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import java.util.List; public interface IElasticsearchSvc { + + /** + * Returns identifiers for the last most recent N observations that meet the specified criteria. + * @param theSearchParameterMap SearchParameterMap containing search parameters used for filtering the last N observations. Supported parameters include Subject, Patient, Code, Category and Max (the parameter used to determine N). + * @param theFhirContext Current FhirContext. + * @param theMaxResultsToFetch The maximum number of results to return for the purpose of paging. + * @return + */ List executeLastN(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, Integer theMaxResultsToFetch); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java index de140a25f32..46b3180c1b1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java @@ -10,7 +10,7 @@ package ca.uhn.fhir.jpa.search.lastn.json; * #L% */ -import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import ca.uhn.fhir.jpa.model.util.CodeSystemHash; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java index 4f393e8d98f..928e0c5da18 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java @@ -10,7 +10,7 @@ package ca.uhn.fhir.jpa.search.lastn.json; * #L% */ -import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import ca.uhn.fhir.jpa.model.util.CodeSystemHash; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -25,148 +25,117 @@ import java.util.List; @JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) public class ObservationJson { - @JsonProperty(value = "identifier", required = true) - private String myIdentifier; + @JsonProperty(value = "identifier", required = true) + private String myIdentifier; - @JsonProperty(value = "subject", required = true) - private String mySubject; + @JsonProperty(value = "subject", required = true) + private String mySubject; - @JsonProperty(value = "categoryconcepttext", required = false) - private List myCategory_concept_text = new ArrayList<>(); + @JsonProperty(value = "categoryconcepttext", required = false) + private List myCategory_concept_text = new ArrayList<>(); - @JsonProperty(value = "categoryconceptcodingcode", required = false) - private List> myCategory_coding_code = new ArrayList<>(); + @JsonProperty(value = "categoryconceptcodingcode", required = false) + private List> myCategory_coding_code = new ArrayList<>(); - @JsonProperty(value = "categoryconceptcodingcode_system_hash", required = false) - private List> myCategory_coding_code_system_hash = new ArrayList<>(); + @JsonProperty(value = "categoryconceptcodingcode_system_hash", required = false) + private List> myCategory_coding_code_system_hash = new ArrayList<>(); - @JsonProperty(value = "categoryconceptcodingdisplay", required = false) - private List> myCategory_coding_display = new ArrayList<>(); + @JsonProperty(value = "categoryconceptcodingdisplay", required = false) + private List> myCategory_coding_display = new ArrayList<>(); - @JsonProperty(value = "categoryconceptcodingsystem", required = false) - private List> myCategory_coding_system = new ArrayList<>(); + @JsonProperty(value = "categoryconceptcodingsystem", required = false) + private List> myCategory_coding_system = new ArrayList<>(); - @JsonProperty(value = "codeconceptid", required = false) - private String myCode_concept_id; + @JsonProperty(value = "codeconceptid", required = false) + private String myCode_concept_id; - @JsonProperty(value = "codeconcepttext", required = false) - private String myCode_concept_text; + @JsonProperty(value = "codeconcepttext", required = false) + private String myCode_concept_text; - @JsonProperty(value = "codeconceptcodingcode", required = false) - // TODO: Temporary change until sort out how to deal with multiple observation code codings -// private List myCode_coding_code = new ArrayList<>(); - private String myCode_coding_code; + @JsonProperty(value = "codeconceptcodingcode", required = false) + private String myCode_coding_code; - @JsonProperty(value = "codeconceptcodingcode_system_hash", required = false) - // TODO: Temporary change until sort out how to deal with multiple observation code codings -// private List myCode_coding_code_system_hash = new ArrayList<>(); - private String myCode_coding_code_system_hash; + @JsonProperty(value = "codeconceptcodingcode_system_hash", required = false) + private String myCode_coding_code_system_hash; - @JsonProperty(value = "codeconceptcodingdisplay", required = false) - // TODO: Temporary change until sort out how to deal with multiple observation code codings -// private List myCode_coding_display = new ArrayList<>(); - private String myCode_coding_display; + @JsonProperty(value = "codeconceptcodingdisplay", required = false) + private String myCode_coding_display; - @JsonProperty(value = "codeconceptcodingsystem", required = false) - // TODO: Temporary change until sort out how to deal with multiple observation code codings -// private List myCode_coding_system = new ArrayList<>(); - private String myCode_coding_system; + @JsonProperty(value = "codeconceptcodingsystem", required = false) + private String myCode_coding_system; - @JsonProperty(value = "effectivedtm", required = true) - private Date myEffectiveDtm; + @JsonProperty(value = "effectivedtm", required = true) + private Date myEffectiveDtm; - public ObservationJson() {} + public ObservationJson() { + } - public void setIdentifier(String theIdentifier) { - myIdentifier = theIdentifier; - } + public void setIdentifier(String theIdentifier) { + myIdentifier = theIdentifier; + } - public void setSubject(String theSubject) { - mySubject = theSubject; - } + public void setSubject(String theSubject) { + mySubject = theSubject; + } - public void setCategories(List theCategories) { - for (CodeableConcept theConcept : theCategories) { - myCategory_concept_text.add(theConcept.getText()); - List coding_code_system_hashes = new ArrayList<>(); - List coding_codes = new ArrayList<>(); - List coding_displays = new ArrayList<>(); - List coding_systems = new ArrayList<>(); - for (Coding theCategoryCoding : theConcept.getCoding()) { - coding_code_system_hashes.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCategoryCoding.getSystem(), theCategoryCoding.getCode()))); - coding_codes.add(theCategoryCoding.getCode()); - coding_displays.add(theCategoryCoding.getDisplay()); - coding_systems.add(theCategoryCoding.getSystem()); - } - myCategory_coding_code_system_hash.add(coding_code_system_hashes); - myCategory_coding_code.add(coding_codes); - myCategory_coding_display.add(coding_displays); - myCategory_coding_system.add(coding_systems); - } - } + public void setCategories(List theCategories) { + for (CodeableConcept theConcept : theCategories) { + myCategory_concept_text.add(theConcept.getText()); + List coding_code_system_hashes = new ArrayList<>(); + List coding_codes = new ArrayList<>(); + List coding_displays = new ArrayList<>(); + List coding_systems = new ArrayList<>(); + for (Coding theCategoryCoding : theConcept.getCoding()) { + coding_code_system_hashes.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCategoryCoding.getSystem(), theCategoryCoding.getCode()))); + coding_codes.add(theCategoryCoding.getCode()); + coding_displays.add(theCategoryCoding.getDisplay()); + coding_systems.add(theCategoryCoding.getSystem()); + } + myCategory_coding_code_system_hash.add(coding_code_system_hashes); + myCategory_coding_code.add(coding_codes); + myCategory_coding_display.add(coding_displays); + myCategory_coding_system.add(coding_systems); + } + } - public List getCategory_concept_text() { - return myCategory_concept_text; - } + public List getCategory_concept_text() { + return myCategory_concept_text; + } - public List> getCategory_coding_code_system_hash() { - return myCategory_coding_code_system_hash; - } + public List> getCategory_coding_code_system_hash() { + return myCategory_coding_code_system_hash; + } - public List> getCategory_coding_code() { - return myCategory_coding_code; - } + public List> getCategory_coding_code() { + return myCategory_coding_code; + } - public List> getCategory_coding_display() { - return myCategory_coding_display; - } + public List> getCategory_coding_display() { + return myCategory_coding_display; + } - public List> getCategory_coding_system() { - return myCategory_coding_system; - } + public List> getCategory_coding_system() { + return myCategory_coding_system; + } - public void setCode(CodeableConcept theCode) { - myCode_concept_text = theCode.getText(); - for(Coding theCodeCoding : theCode.getCoding()) { - // TODO: Temporary change until address how to manage multiple Observation Code codings. -/* myCode_coding_code_system_hash.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCodeCoding.getSystem(), theCodeCoding.getCode()))); - myCode_coding_code.add(theCodeCoding.getCode()); - myCode_coding_display.add(theCodeCoding.getDisplay()); - myCode_coding_system.add(theCodeCoding.getSystem()); - */ - myCode_coding_code_system_hash = String.valueOf(CodeSystemHash.hashCodeSystem(theCodeCoding.getSystem(), theCodeCoding.getCode())); - myCode_coding_code = theCodeCoding.getCode(); - myCode_coding_display = theCodeCoding.getDisplay(); - myCode_coding_system = theCodeCoding.getSystem(); - } + public void setCode(CodeableConcept theCode) { + myCode_concept_text = theCode.getText(); + for (Coding theCodeCoding : theCode.getCoding()) { + myCode_coding_code_system_hash = String.valueOf(CodeSystemHash.hashCodeSystem(theCodeCoding.getSystem(), theCodeCoding.getCode())); + myCode_coding_code = theCodeCoding.getCode(); + myCode_coding_display = theCodeCoding.getDisplay(); + myCode_coding_system = theCodeCoding.getSystem(); + } - } + } - public String getCode_concept_text() { - return myCode_concept_text; - } + public String getCode_concept_text() { + return myCode_concept_text; + } - // TODO: Temporary changes until resolve problem of how to manage Observation Code with multiple codings -/* - public List getCode_coding_code_system_hash() { - return myCode_coding_code_system_hash; - } - - public List getCode_coding_code() { - return myCode_coding_code; - } - - public List getCode_coding_display() { - return myCode_coding_display; - } - - public List getCode_coding_system() { - return myCode_coding_system; - } -*/ - public String getCode_coding_code_system_hash() { - return myCode_coding_code_system_hash; - } + public String getCode_coding_code_system_hash() { + return myCode_coding_code_system_hash; + } public String getCode_coding_code() { return myCode_coding_code; @@ -180,28 +149,28 @@ public class ObservationJson { return myCode_coding_system; } - public void setCode_concept_id(String theCodeId) { - myCode_concept_id = theCodeId; - } + public void setCode_concept_id(String theCodeId) { + myCode_concept_id = theCodeId; + } - public String getCode_concept_id() { - return myCode_concept_id; - } + public String getCode_concept_id() { + return myCode_concept_id; + } - public void setEffectiveDtm(Date theEffectiveDtm) { - myEffectiveDtm = theEffectiveDtm; - } + public void setEffectiveDtm(Date theEffectiveDtm) { + myEffectiveDtm = theEffectiveDtm; + } - public Date getEffectiveDtm() { - return myEffectiveDtm; - } + public Date getEffectiveDtm() { + return myEffectiveDtm; + } - public String getSubject() { - return mySubject; - } + public String getSubject() { + return mySubject; + } - public String getIdentifier() { - return myIdentifier; - } + public String getIdentifier() { + return myIdentifier; + } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java deleted file mode 100644 index 9995b1d3ec0..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java +++ /dev/null @@ -1,33 +0,0 @@ -package ca.uhn.fhir.jpa.search.lastn.util; - -import ca.uhn.fhir.util.UrlUtil; -import com.google.common.base.Charsets; -import com.google.common.hash.HashCode; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; - -public class CodeSystemHash { - private static final HashFunction HASH_FUNCTION = Hashing.murmur3_128(0); - private static final byte[] DELIMITER_BYTES = "|".getBytes(Charsets.UTF_8); - - static public long hashCodeSystem( String system, String code ) { - Hasher hasher = HASH_FUNCTION.newHasher(); - addStringToHasher(hasher, system); - addStringToHasher(hasher, code); - - HashCode hashCode = hasher.hash(); - return hashCode.asLong(); - } - - static private void addStringToHasher(Hasher hasher, String next) { - if (next == null) { - hasher.putByte((byte) 0); - } else { - next = UrlUtil.escapeUrlParam(next); - byte[] bytes = next.getBytes(Charsets.UTF_8); - hasher.putBytes(bytes); - } - hasher.putBytes(DELIMITER_BYTES); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json new file mode 100644 index 00000000000..acedddf3490 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationCodeIndexSchema.json @@ -0,0 +1,26 @@ +{ + "mappings" : { + "ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity" : { + "properties" : { + "codeable_concept_id" : { + "type" : "keyword" + }, + "codingcode" : { + "type" : "keyword" + }, + "codingcode_system_hash" : { + "type" : "keyword" + }, + "codingdisplay" : { + "type" : "keyword" + }, + "codingsystem" : { + "type" : "keyword" + }, + "text" : { + "type" : "keyword" + } + } + } + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json new file mode 100644 index 00000000000..a91869e41d5 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/search/lastn/ObservationIndexSchema.json @@ -0,0 +1,50 @@ +{ + "mappings" : { + "ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity" : { + "properties" : { + "codeconceptid" : { + "type" : "keyword" + }, + "codeconcepttext" : { + "type" : "text" + }, + "codeconceptcodingcode" : { + "type" : "keyword" + }, + "codeconceptcodingsystem" : { + "type" : "keyword" + }, + "codeconceptcodingcode_system_hash" : { + "type" : "keyword" + }, + "codeconceptcodingdisplay" : { + "type" : "keyword" + }, + "categoryconcepttext" : { + "type" : "keyword" + }, + "categoryconceptcodingcode" : { + "type" : "keyword" + }, + "categoryconceptcodingsystem" : { + "type" : "keyword" + }, + "categoryconceptcodingcode_system_hash" : { + "type" : "keyword" + }, + "categoryconceptcodingdisplay" : { + "type" : "keyword" + }, + "effectivedtm" : { + "type" : "date" + }, + "identifier" : { + "type" : "keyword" + }, + "subject" : { + "type" : "keyword" + } + } + } + } +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java index 7978c956e15..b7a28efb5fc 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNAsyncIT.java @@ -52,14 +52,14 @@ public class FhirResourceDaoR4SearchLastNAsyncIT extends BaseR4SearchLastN { mySmallerPreFetchThresholds.add(-1); myDaoConfig.setSearchPreFetchThresholds(mySmallerPreFetchThresholds); - SearchBuilder.setIsTest(true); + SearchBuilder.setMaxPageSize50ForTest(true); } @After public void after() { myDaoConfig.setSearchPreFetchThresholds(originalPreFetchThresholds); - SearchBuilder.setIsTest(false); + SearchBuilder.setMaxPageSize50ForTest(false); } @Test @@ -82,7 +82,7 @@ public class FhirResourceDaoR4SearchLastNAsyncIT extends BaseR4SearchLastN { when(mySrd.getParameters()).thenReturn(requestParameters); // Set chunk size to 50 - SearchBuilder.setIsTest(true); + SearchBuilder.setMaxPageSize50ForTest(true); // Expand default fetch sizes to ensure all observations are returned in first page: List myBiggerPreFetchThresholds = new ArrayList<>(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java index 8614a9e7c7e..d2314dc191e 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java @@ -23,7 +23,7 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN { @After public void resetMaximumPageSize() { - SearchBuilder.setIsTest(false); + SearchBuilder.setMaxPageSize50ForTest(false); } @Test @@ -46,7 +46,7 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN { when(mySrd.getParameters()).thenReturn(requestParameters); // Set chunk size to 50 - SearchBuilder.setIsTest(true); + SearchBuilder.setMaxPageSize50ForTest(true); myCaptureQueriesListener.clear(); List results = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null)); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java index 1f29f9bf978..aef102cd3eb 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java @@ -130,7 +130,6 @@ public class PersistObservationIndexedSearchParamLastNR4IT { myObservation.setCode(getObservationCode()); - //myObservationLastNIndexPersistR4Svc.indexObservation(myObservation); testObservationPersist.indexObservation(myObservation); } @@ -139,12 +138,12 @@ public class PersistObservationIndexedSearchParamLastNR4IT { // Add three CodeableConcepts for category List categoryConcepts = new ArrayList<>(); // Create three codings and first category CodeableConcept - List category1 = new ArrayList<>(); + List category = new ArrayList<>(); CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); - category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, "test-heart-rate display")); - category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); - category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); - categoryCodeableConcept1.setCoding(category1); + category.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, "test-heart-rate display")); + category.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + categoryCodeableConcept1.setCoding(category); categoryConcepts.add(categoryCodeableConcept1); // Create three codings and second category CodeableConcept List category2 = new ArrayList<>(); @@ -169,9 +168,6 @@ public class PersistObservationIndexedSearchParamLastNR4IT { // Create CodeableConcept for Code with three codings. CodeableConcept codeableConceptField = new CodeableConcept().setText(SINGLE_OBSERVATION_CODE_TEXT); codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, "test-code display")); - // TODO: Temporarily limit this to a single Observation Code coding until we sort out how to manage multiple codings -// codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); -// codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); return codeableConceptField; } @@ -263,8 +259,6 @@ public class PersistObservationIndexedSearchParamLastNR4IT { Date effectiveDtm = observationDate.getTime(); observation.setEffective(new DateTimeType(effectiveDtm)); -// myObservationLastNIndexPersistR4Svc.indexObservation(observation); - testObservationPersist.indexObservation(observation); } @@ -279,7 +273,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT { indexMultipleObservations(); assertEquals(100, myResourceIndexedObservationLastNDao.count()); // Check that fifth observation for fifth patient has been indexed. - ObservationIndexedSearchParamLastNEntity observation = myResourceIndexedObservationLastNDao.findForIdentifier("55"); + ObservationIndexedSearchParamLastNEntity observation = myResourceIndexedObservationLastNDao.findByIdentifier("55"); assertNotNull(observation); SearchParameterMap searchParameterMap = new SearchParameterMap(); @@ -295,12 +289,11 @@ public class PersistObservationIndexedSearchParamLastNR4IT { entity.setResourceType("Observation"); entity.setVersion(0L); -// myObservationLastNIndexPersistR4Svc.deleteObservationIndex(entity); testObservationPersist.deleteObservationIndex(entity); // Confirm that observation was deleted. assertEquals(99, myResourceIndexedObservationLastNDao.count()); - observation = myResourceIndexedObservationLastNDao.findForIdentifier("55"); + observation = myResourceIndexedObservationLastNDao.findByIdentifier("55"); assertNull(observation); observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirCtx, 200); @@ -341,10 +334,9 @@ public class PersistObservationIndexedSearchParamLastNR4IT { updatedObservation.setCategory(getCategoryCode()); updatedObservation.setCode(getObservationCode()); -// myObservationLastNIndexPersistR4Svc.indexObservation(updatedObservation); testObservationPersist.indexObservation(updatedObservation); - ObservationIndexedSearchParamLastNEntity updatedObservationEntity = myResourceIndexedObservationLastNDao.findForIdentifier(SINGLE_OBSERVATION_PID); + ObservationIndexedSearchParamLastNEntity updatedObservationEntity = myResourceIndexedObservationLastNDao.findByIdentifier(SINGLE_OBSERVATION_PID); assertEquals("1234", updatedObservationEntity.getSubject()); assertEquals(newEffectiveDtm.getValue(), updatedObservationEntity.getEffectiveDtm()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java index 4b16c31c095..e128026f737 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java @@ -54,7 +54,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT { @After public void after() throws IOException { elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.CODE_INDEX); + elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX); } @Test @@ -307,18 +307,12 @@ public class LastNElasticsearchSvcMultipleObservationsIT { String codeableConceptId1 = UUID.randomUUID().toString(); CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); - // TODO: uncomment the following once there is a solution to supporting multiple codings for Observation Code -// codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); -// codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1); String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1); String codeableConceptId2 = UUID.randomUUID().toString(); CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); - // TODO: uncomment the following once there is a solution to supporting multiple codings for Observation Code -// codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); -// codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2); String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2); @@ -357,12 +351,12 @@ public class LastNElasticsearchSvcMultipleObservationsIT { observationJson.setCategories(categoryConcepts1); observationJson.setCode(codeableConceptField1); observationJson.setCode_concept_id(codeableConceptId1); - assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.CODE_INDEX, codeableConceptId1, codeJson1Document, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); + assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX, codeableConceptId1, codeJson1Document, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); } else { observationJson.setCategories(categoryConcepts2); observationJson.setCode(codeableConceptField2); observationJson.setCode_concept_id(codeableConceptId2); - assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.CODE_INDEX, codeableConceptId2, codeJson2Document, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); + assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX, codeableConceptId2, codeJson2Document, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); } Calendar observationDate = new GregorianCalendar(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java index e220bf96569..1e2a6ecb8ed 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java @@ -1,27 +1,37 @@ package ca.uhn.fhir.jpa.search.lastn; import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.model.util.CodeSystemHash; import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchConfig; import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; -import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.dstu2.resource.Observation; -import ca.uhn.fhir.rest.param.*; +import ca.uhn.fhir.rest.param.ReferenceAndListParam; +import ca.uhn.fhir.rest.param.ReferenceOrListParam; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenAndListParam; +import ca.uhn.fhir.rest.param.TokenOrListParam; +import ca.uhn.fhir.rest.param.TokenParam; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import org.hl7.fhir.r4.model.CodeableConcept; import org.hl7.fhir.r4.model.Coding; -import org.junit.*; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -68,12 +78,6 @@ public class LastNElasticsearchSvcSingleObservationIT { final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; final String CODEFIRSTCODINGCODE = "test-code"; final String CODEFIRSTCODINGDISPLAY = "test-code display"; -// final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; -// final String CODESECONDCODINGCODE = "test-alt-code"; -// final String CODESECONDCODINGDISPLAY = "test-alt-code display"; -// final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; -// final String CODETHIRDCODINGCODE = "test-second-alt-code"; -// final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; final FhirContext myFhirContext = FhirContext.forR4(); @@ -89,7 +93,7 @@ public class LastNElasticsearchSvcSingleObservationIT { @After public void after() throws IOException { elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_INDEX); - elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.CODE_INDEX); + elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX); } @Test @@ -209,32 +213,6 @@ public class LastNElasticsearchSvcSingleObservationIT { String code_concept_text_values = observation.getCode_concept_text(); assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. - /* - List code_coding_systems = observation.getCode_coding_system(); - assertEquals(3,code_coding_systems.size()); - assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems.get(0)); - assertEquals(CODESECONDCODINGSYSTEM, code_coding_systems.get(1)); - assertEquals(CODETHIRDCODINGSYSTEM, code_coding_systems.get(2)); - - List code_coding_codes = observation.getCode_coding_code(); - assertEquals(3, code_coding_codes.size()); - assertEquals(CODEFIRSTCODINGCODE, code_coding_codes.get(0)); - assertEquals(CODESECONDCODINGCODE, code_coding_codes.get(1)); - assertEquals(CODETHIRDCODINGCODE, code_coding_codes.get(2)); - - List code_coding_display = observation.getCode_coding_display(); - assertEquals(3, code_coding_display.size()); - assertEquals(CODEFIRSTCODINGDISPLAY, code_coding_display.get(0)); - assertEquals(CODESECONDCODINGDISPLAY, code_coding_display.get(1)); - assertEquals(CODETHIRDCODINGDISPLAY, code_coding_display.get(2)); - - List code_coding_code_system_hash = observation.getCode_coding_code_system_hash(); - assertEquals(3, code_coding_code_system_hash.size()); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash.get(0)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), code_coding_code_system_hash.get(1)); - assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), code_coding_code_system_hash.get(2)); -*/ String code_coding_systems = observation.getCode_coding_system(); assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems); @@ -258,36 +236,20 @@ public class LastNElasticsearchSvcSingleObservationIT { assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText); List persistedCodeCodingSystems = persistedObservationCode.getCoding_system(); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. -// assertEquals(3,persistedCodeCodingSystems.size()); assertEquals(1, persistedCodeCodingSystems.size()); assertEquals(CODEFIRSTCODINGSYSTEM, persistedCodeCodingSystems.get(0)); -// assertEquals(CODESECONDCODINGSYSTEM, persistedCodeCodingSystems.get(1)); -// assertEquals(CODETHIRDCODINGSYSTEM, persistedCodeCodingSystems.get(2)); List persistedCodeCodingCodes = persistedObservationCode.getCoding_code(); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. -// assertEquals(3, persistedCodeCodingCodes.size()); assertEquals(1, persistedCodeCodingCodes.size()); assertEquals(CODEFIRSTCODINGCODE, persistedCodeCodingCodes.get(0)); -// assertEquals(CODESECONDCODINGCODE, persistedCodeCodingCodes.get(1)); -// assertEquals(CODETHIRDCODINGCODE, persistedCodeCodingCodes.get(2)); List persistedCodeCodingDisplays = persistedObservationCode.getCoding_display(); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. -// assertEquals(3, persistedCodeCodingDisplays.size()); assertEquals(1, persistedCodeCodingDisplays.size()); assertEquals(CODEFIRSTCODINGDISPLAY, persistedCodeCodingDisplays.get(0)); -// assertEquals(CODESECONDCODINGDISPLAY, persistedCodeCodingDisplays.get(1)); -// assertEquals(CODETHIRDCODINGDISPLAY, persistedCodeCodingDisplays.get(2)); List persistedCodeCodingCodeSystemHashes = persistedObservationCode.getCoding_code_system_hash(); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. -// assertEquals(3, persistedCodeCodingCodeSystemHashes.size()); assertEquals(1, persistedCodeCodingCodeSystemHashes.size()); assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(0)); -// assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(1)); -// assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(2)); } @@ -330,9 +292,6 @@ public class LastNElasticsearchSvcSingleObservationIT { indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID); CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT); codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY)); - // TODO: Temporary changes until find a solution for addressing Observation Code with multiple codings. -// codeableConceptField.addCoding(new Coding(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE, CODESECONDCODINGDISPLAY)); -// codeableConceptField.addCoding(new Coding(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE, CODETHIRDCODINGDISPLAY)); indexedObservation.setCode(codeableConceptField); String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); @@ -340,7 +299,7 @@ public class LastNElasticsearchSvcSingleObservationIT { CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); - assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); + assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE)); try { Thread.sleep(1000L); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java index 85b82b4dd9c..85a088e60b2 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java @@ -47,9 +47,4 @@ public class TestElasticsearchConfig { return embeddedElastic; } -// @PreDestroy -// public void stop() { -// embeddedElasticSearch().stop(); -// } - } diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java index 93d1a916086..d2ec25ff4d0 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodeableConceptEntity.java @@ -24,12 +24,9 @@ public class ObservationIndexedCodeCodeableConceptEntity { @Column(name = "CODEABLE_CONCEPT_TEXT", nullable = true, length = MAX_LENGTH) private String myCodeableConceptText; - // TODO: Make coding a Collection. Need to first figure out how to maintain this over time. @IndexedEmbedded(depth=2, prefix = "coding") -// @OneToMany(mappedBy = "myCodeableConceptId", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) @JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_CONCEPT_CODE")) @OneToOne(cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) -// private Set myObservationIndexedCodeCodingEntitySet; private ObservationIndexedCodeCodingEntity myObservationIndexedCodeCodingEntity; public ObservationIndexedCodeCodeableConceptEntity() { @@ -42,10 +39,6 @@ public class ObservationIndexedCodeCodeableConceptEntity { } public void addCoding(ObservationIndexedCodeCodingEntity theObservationIndexedCodeCodingEntity) { -// if (myObservationIndexedCodeCodingEntitySet == null) { -// myObservationIndexedCodeCodingEntitySet = new HashSet<>(); -// } -// myObservationIndexedCodeCodingEntitySet.add(theObservationIndexedCodeCodingEntity); myObservationIndexedCodeCodingEntity = theObservationIndexedCodeCodingEntity; } @@ -65,6 +58,4 @@ public class ObservationIndexedCodeCodeableConceptEntity { myCodeableConceptText = theCodeableConceptText; } - - } diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java index 7373f04bc91..edae1da7d88 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java @@ -13,36 +13,31 @@ public class ObservationIndexedCodeCodingEntity { public static final int MAX_LENGTH = 200; - // TODO: Fix this to allow multiple codings for observation code -// @Id -// @SequenceGenerator(name = "SEQ_CODING_FIELD", sequenceName = "SEQ_CODING_FIELD") -// @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CODING_FIELD") -// private Long myId; - @Id - @Column(name="CODEABLE_CONCEPT_ID", length = MAX_LENGTH) - private String myCodeableConceptId; + @Column(name = "CODEABLE_CONCEPT_ID", length = MAX_LENGTH) + private String myCodeableConceptId; - @Field (name = "code", analyze = Analyze.NO) - private String myCode; + @Field(name = "code", analyze = Analyze.NO) + private String myCode; - @Field (name = "system", analyze = Analyze.NO) - private String mySystem; + @Field(name = "system", analyze = Analyze.NO) + private String mySystem; - @Field (name = "code_system_hash", analyze = Analyze.NO) - private String myCodeSystemHash; + @Field(name = "code_system_hash", analyze = Analyze.NO) + private String myCodeSystemHash; - @Field (name = "display") - private String myDisplay; + @Field(name = "display") + private String myDisplay; - public ObservationIndexedCodeCodingEntity() {} + public ObservationIndexedCodeCodingEntity() { + } - public ObservationIndexedCodeCodingEntity(String theSystem, String theCode, String theDisplay, String theCodeableConceptId) { - myCode = theCode; - mySystem = theSystem; - myCodeSystemHash = String.valueOf(CodeSystemHash.hashCodeSystem(theSystem, theCode)); - myDisplay = theDisplay; - myCodeableConceptId = theCodeableConceptId; - } + public ObservationIndexedCodeCodingEntity(String theSystem, String theCode, String theDisplay, String theCodeableConceptId) { + myCode = theCode; + mySystem = theSystem; + myCodeSystemHash = String.valueOf(CodeSystemHash.hashCodeSystem(theSystem, theCode)); + myDisplay = theDisplay; + myCodeableConceptId = theCodeableConceptId; + } } diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java index 9792226d71b..abc1a3850b4 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java @@ -10,32 +10,47 @@ public class LastNParameterHelper { if (theParamName == null) { return false; } - if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) { - if (theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_PATIENT) - || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)) { - return true; - } else { - return false; - } - } else if (theContext.getVersion().getVersion() == FhirVersionEnum.R4) { - if (theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_PATIENT) - || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CODE)) { - return true; - } else { - return false; - } - } else if (theContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) { - if (theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_PATIENT) - || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CODE)) { - return true; - } else { - return false; - } + + FhirVersionEnum version = theContext.getVersion().getVersion(); + + if (isR5(version) && isLastNParameterR5(theParamName)) { + return true; + } else if (isR4(version) && isLastNParameterR4(theParamName)) { + return true; + } else if (isDstu3(version) && isLastNParameterDstu3(theParamName)) { + return true; } else { - throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString()); + return false; } } + private static boolean isDstu3(FhirVersionEnum theVersion) { + return (theVersion == FhirVersionEnum.DSTU3); + } + + private static boolean isR4(FhirVersionEnum theVersion) { + return (theVersion == FhirVersionEnum.R4); + } + + private static boolean isR5(FhirVersionEnum theVersion) { + return (theVersion == FhirVersionEnum.R5); + } + + private static boolean isLastNParameterDstu3(String theParamName) { + return (theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_PATIENT) + || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)); + } + + private static boolean isLastNParameterR4(String theParamName) { + return (theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_PATIENT) + || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CODE)); + } + + private static boolean isLastNParameterR5(String theParamName) { + return (theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_PATIENT) + || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)); + } + public static String getSubjectParamName(FhirContext theContext) { if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) { return org.hl7.fhir.r5.model.Observation.SP_SUBJECT;