diff --git a/hapi-deployable-pom/pom.xml b/hapi-deployable-pom/pom.xml index 599b8fe0ee4..f33e54912df 100644 --- a/hapi-deployable-pom/pom.xml +++ b/hapi-deployable-pom/pom.xml @@ -155,6 +155,8 @@ javac.bat about.html changelog.xml + .*/favicon.ico$ + Log4j-charsets.properties diff --git a/hapi-fhir-elasticsearch-6/pom.xml b/hapi-fhir-elasticsearch-6/pom.xml new file mode 100644 index 00000000000..c63846bc972 --- /dev/null +++ b/hapi-fhir-elasticsearch-6/pom.xml @@ -0,0 +1,183 @@ + + + + 4.0.0 + + ca.uhn.hapi.fhir + hapi-fhir-elasticsearch-6 + 1.0-SNAPSHOT + + hapi-fhir-elasticsearch-6 + + http://www.example.com + + + UTF-8 + 1.7 + 1.7 + + + + + junit + junit + 4.12 + test + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.5.4 + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.dataformat + * + + + com.github.spullara.mustache.java + compiler + + + com.tdunning + t-digest + + + commons-codec + commons-codec + + + commons-logging + commons-logging + + + net.bytebuddy + byte-buddy + + + net.sf.jopt-simple + jopt-simple + + + org.apache.httpcomponents + * + + + org.apache.lucene + lucene-analyzers-common + + + org.apache.lucene + lucene-backward-codecs + + + org.apache.lucene + lucene-sandbox + + + org.elasticsearch + jna + + + org.hdrhistogram + HdrHistogram + + + org.yaml + snakeyaml + + + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + + maven-shade-plugin + 3.2.1 + + + package + + shade + + + true + shaded6 + + + com.carrotsearch.hppc + com.shadehapi.carrotsearch.hppc + + + org.apache.logging.log4j + org.shadehapi.apache.logging.log4j + + + org.apache.lucene + org.shadehapi.apache.lucene + + + org.elasticsearch + org.shadehapi.elasticsearch + + + org.joda + org.shadehapi.joda + + + + + + + + + diff --git a/hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java b/hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java new file mode 100644 index 00000000000..9b658e5bf48 --- /dev/null +++ b/hapi-fhir-elasticsearch-6/src/main/java/ca/uhn/fhir/fhir/App.java @@ -0,0 +1,13 @@ +package ca.uhn.fhir.fhir; + +/** + * Hello world! + * + */ +public class App +{ + public static void main( String[] args ) + { + System.out.println( "Hello World!" ); + } +} diff --git a/hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java b/hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java new file mode 100644 index 00000000000..83fcc8b98cb --- /dev/null +++ b/hapi-fhir-elasticsearch-6/src/test/java/ca/uhn/fhir/fhir/AppTest.java @@ -0,0 +1,20 @@ +package ca.uhn.fhir.fhir; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Unit test for simple App. + */ +public class AppTest +{ + /** + * Rigorous Test :-) + */ + @Test + public void shouldAnswerWithTrue() + { + assertTrue( true ); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java new file mode 100644 index 00000000000..ee3ecde5d84 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodeableConceptSearchParamDao.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.jpa.dao.data; + +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface IObservationIndexedCodeCodeableConceptSearchParamDao extends JpaRepository { +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java new file mode 100644 index 00000000000..d5d46f47e16 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java @@ -0,0 +1,26 @@ +package ca.uhn.fhir.jpa.dao.data; + +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodingEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +@Repository +public interface IObservationIndexedCodeCodingSearchParamDao extends JpaRepository { + + @Query("" + + "SELECT t.myCodeableConceptId FROM ObservationIndexedCodeCodingEntity t " + + "WHERE t.myCode = :code " + + "AND t.mySystem = :system " + + "") + String findForCodeAndSystem(@Param("code") String theCode, @Param("system") String theSystem); + + + @Query("" + + "SELECT t.myCodeableConceptId FROM ObservationIndexedCodeCodingEntity t " + + "WHERE t.myDisplay = :display" + + "") + String findForDisplay(@Param("display") String theDisplay); + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java new file mode 100644 index 00000000000..7a33dec68cd --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedSearchParamLastNDao.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.jpa.dao.data; + +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface IObservationIndexedSearchParamLastNDao extends JpaRepository{ +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java new file mode 100644 index 00000000000..69caae6ff39 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/ObservationLastNIndexPersistSvc.java @@ -0,0 +1,80 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao; +import ca.uhn.fhir.jpa.dao.lastn.entity.*; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Observation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.*; + +@Component +public class ObservationLastNIndexPersistSvc { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myObservationIndexedCodeableConceptSearchParamDao; + + @Autowired + IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao; + + public void indexObservation(Observation theObservation) { + ObservationIndexedSearchParamLastNEntity indexedObservation = new ObservationIndexedSearchParamLastNEntity(); + String resourcePID = theObservation.getId(); + indexedObservation.setIdentifier(resourcePID); + // TODO: Need to improve this as there are multiple use cases involving subject. + String subjectId = "Patient/" + theObservation.getSubject().getReference(); + indexedObservation.setSubject(subjectId); + Date effectiveDtm = theObservation.getEffectiveDateTimeType().getValue(); + indexedObservation.setEffectiveDtm(effectiveDtm); + + // Build CodeableConcept entities for Observation.Category + Set categoryConcepts = new HashSet<>(); + for(CodeableConcept categoryCodeableConcept : theObservation.getCategory()) { + // Build Coding entities for each category CodeableConcept + Set categoryCodingEntities = new HashSet<>(); + ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConceptEntity = new ObservationIndexedCategoryCodeableConceptEntity(categoryCodeableConcept.getText()); + for(Coding categoryCoding : categoryCodeableConcept.getCoding()){ + categoryCodingEntities.add(new ObservationIndexedCategoryCodingEntity(categoryCoding.getSystem(), categoryCoding.getCode(), categoryCoding.getDisplay())); + } + categoryCodeableConceptEntity.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities); + categoryConcepts.add(categoryCodeableConceptEntity); + } + indexedObservation.setCategoryCodeableConcepts(categoryConcepts); + + // Build CodeableConcept entity for Observation.Code. + CodeableConcept codeCodeableConcept = theObservation.getCode(); + String observationCodeNormalizedId = null; + + // Determine if a Normalized ID was created previously for Observation Code + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + if (codeCoding.hasCode() && codeCoding.hasSystem()) { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForCodeAndSystem(codeCoding.getCode(), codeCoding.getSystem()); + } else { + observationCodeNormalizedId = myObservationIndexedCodeCodingSearchParamDao.findForDisplay(codeCoding.getDisplay()); + } + } + // Generate a new a normalized ID if necessary + if (observationCodeNormalizedId == null) { + observationCodeNormalizedId = UUID.randomUUID().toString(); + } + + // Create/update normalized Observation Code index record + ObservationIndexedCodeCodeableConceptEntity codeableConceptField = new ObservationIndexedCodeCodeableConceptEntity(codeCodeableConcept.getText(), observationCodeNormalizedId); + for (Coding codeCoding : codeCodeableConcept.getCoding()) { + codeableConceptField.addCoding(new ObservationIndexedCodeCodingEntity(codeCoding.getSystem(), codeCoding.getCode(), codeCoding.getDisplay(), observationCodeNormalizedId)); + } + myObservationIndexedCodeableConceptSearchParamDao.save(codeableConceptField); + + indexedObservation.setObservationCode(codeableConceptField); + indexedObservation.setCodeNormalizedId(observationCodeNormalizedId); + myResourceIndexedObservationLastNDao.save(indexedObservation); + + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java new file mode 100644 index 00000000000..7d5d98525af --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodeableConceptEntity.java @@ -0,0 +1,40 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import org.hibernate.search.annotations.DocumentId; +import org.hibernate.search.annotations.Field; +import org.hibernate.search.annotations.IndexedEmbedded; + +import javax.persistence.*; +import java.util.Set; + +@Embeddable +public class ObservationIndexedCategoryCodeableConceptEntity { + + @Id + @DocumentId(name = "category_concept_id") + @SequenceGenerator(name = "SEQ_CATEGORY_CONCEPT", sequenceName = "SEQ_CATEGORY_CONCEPT") + @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CATEGORY_CONCEPT") + @Column(name = "CATEGORY_CONCEPT_ID") + private Long myId; + + @Field(name = "text") + @Column(name = "CODEABLE_CONCEPT_TEXT", nullable = true) + private String myCodeableConceptText; + + @IndexedEmbedded(depth=2, prefix = "coding") + @OneToMany(mappedBy = "myCodeableConceptId", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) + private Set myObservationIndexedCategoryCodingEntitySet; + + public ObservationIndexedCategoryCodeableConceptEntity(String theCodeableConceptText) { + setCodeableConceptText(theCodeableConceptText); + } + + public void setObservationIndexedCategoryCodingEntitySet(Set theObservationIndexedCategoryCodingEntitySet) { + myObservationIndexedCategoryCodingEntitySet = theObservationIndexedCategoryCodingEntitySet; + } + + public void setCodeableConceptText(String theCodeableConceptText) { + myCodeableConceptText = theCodeableConceptText; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java new file mode 100644 index 00000000000..9bc95ed657c --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCategoryCodingEntity.java @@ -0,0 +1,28 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import ca.uhn.fhir.jpa.dao.lastn.util.CodeSystemHash; +import org.hibernate.search.annotations.Analyze; +import org.hibernate.search.annotations.Field; + +import javax.persistence.*; + +@Embeddable +public class ObservationIndexedCategoryCodingEntity { + + @Field (name = "code", analyze = Analyze.NO) + private String myCode; + @Field (name = "system", analyze = Analyze.NO) + private String mySystem; + @Field (name = "code_system_hash", analyze = Analyze.NO) + private String myCodeSystemHash; + @Field (name = "display") + private String myDisplay; + + public ObservationIndexedCategoryCodingEntity(String theSystem, String theCode, String theDisplay) { + myCode = theCode; + mySystem = theSystem; + myCodeSystemHash = String.valueOf(CodeSystemHash.hashCodeSystem(theSystem, theCode)); + myDisplay = theDisplay; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java new file mode 100644 index 00000000000..6f9b559a476 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodeableConceptEntity.java @@ -0,0 +1,65 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import org.hibernate.search.annotations.DocumentId; +import org.hibernate.search.annotations.Field; +import org.hibernate.search.annotations.Indexed; +import org.hibernate.search.annotations.IndexedEmbedded; + +import javax.persistence.*; +import java.util.HashSet; +import java.util.Set; + +@Entity +@Indexed(index = "code_index") +@Embeddable +@Table(name = "HFJ_SPIDX_LASTN_CODEABLE_CONCEPT") +public class ObservationIndexedCodeCodeableConceptEntity { + + @Id + @DocumentId(name = "codeable_concept_id") + @Column(name="CODEABLE_CONCEPT_ID") + private String myCodeableConceptId; + + @Field(name = "text") + @Column(name = "CODEABLE_CONCEPT_TEXT", nullable = true) + private String myCodeableConceptText; + + @IndexedEmbedded(depth=2, prefix = "coding") + @OneToMany(mappedBy = "myCodeableConceptId", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) + private Set myObservationIndexedCodeCodingEntitySet; + + public ObservationIndexedCodeCodeableConceptEntity() { + + } + + public ObservationIndexedCodeCodeableConceptEntity(String theCodeableConceptText, String theCodeableConceptId) { + setCodeableConceptText(theCodeableConceptText); + setCodeableConceptId(theCodeableConceptId); + } + + public void addCoding(ObservationIndexedCodeCodingEntity theObservationIndexedCodeCodingEntity) { + if (myObservationIndexedCodeCodingEntitySet == null) { + myObservationIndexedCodeCodingEntitySet = new HashSet<>(); + } + myObservationIndexedCodeCodingEntitySet.add(theObservationIndexedCodeCodingEntity); + } + + public String getCodeableConceptId() { + return myCodeableConceptId; + } + + public void setCodeableConceptId(String theCodeableConceptId) { + myCodeableConceptId = theCodeableConceptId; + } + + public String getCodeableConceptText() { + return myCodeableConceptText; + } + + public void setCodeableConceptText(String theCodeableConceptText) { + myCodeableConceptText = theCodeableConceptText; + } + + + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java new file mode 100644 index 00000000000..69183faf393 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedCodeCodingEntity.java @@ -0,0 +1,44 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import ca.uhn.fhir.jpa.dao.lastn.util.CodeSystemHash; +import org.hibernate.search.annotations.Analyze; +import org.hibernate.search.annotations.Field; + +import javax.persistence.*; + +@Entity +@Embeddable +@Table(name = "HFJ_SPIDX_LASTN_CODING") +public class ObservationIndexedCodeCodingEntity { + + @Id + @SequenceGenerator(name = "SEQ_CODING_FIELD", sequenceName = "SEQ_CODING_FIELD") + @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CODING_FIELD") + private Long myId; + + @Column(name="CODEABLE_CONCEPT_ID") + private String myCodeableConceptId; + + @Field (name = "code", analyze = Analyze.NO) + private String myCode; + + @Field (name = "system", analyze = Analyze.NO) + private String mySystem; + + @Field (name = "code_system_hash", analyze = Analyze.NO) + private String myCodeSystemHash; + + @Field (name = "display") + private String myDisplay; + + public ObservationIndexedCodeCodingEntity() {} + + public ObservationIndexedCodeCodingEntity(String theSystem, String theCode, String theDisplay, String theCodeableConceptId) { + myCode = theCode; + mySystem = theSystem; + myCodeSystemHash = String.valueOf(CodeSystemHash.hashCodeSystem(theSystem, theCode)); + myDisplay = theDisplay; + myCodeableConceptId = theCodeableConceptId; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java new file mode 100644 index 00000000000..70f8996089a --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java @@ -0,0 +1,88 @@ +package ca.uhn.fhir.jpa.dao.lastn.entity; + +import org.hibernate.search.annotations.*; + +import javax.persistence.*; +import java.util.*; + +@Entity +@Table(name = "HFJ_LASTN_OBSERVATION") +@Indexed(index = "observation_index") +public class ObservationIndexedSearchParamLastNEntity { + + @Id + @SequenceGenerator(name = "SEQ_LASTN", sequenceName = "SEQ_LASTN") + @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_LASTN") + @Column(name = "LASTN_ID") + private Long myId; + + @Field(name = "subject", analyze = Analyze.NO) + @Column(name = "LASTN_SUBJECT_ID", nullable = false) + private String mySubject; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_OBSERVATION_CODE_FK")) + @IndexedEmbedded(depth = 2, prefix = "codeconcept") + private ObservationIndexedCodeCodeableConceptEntity myObservationCode; + + @Field(name = "codeconceptid", analyze = Analyze.NO) + @Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false) + private String myCodeNormalizedId; + + @IndexedEmbedded(depth = 2, prefix = "categoryconcept") + @Transient + private Set myCategoryCodeableConcepts; + + @Field(name = "effectivedtm", analyze = Analyze.NO) + @Temporal(TemporalType.TIMESTAMP) + @Column(name = "LASTN_EFFECTIVE_DATETIME", nullable = true) + private Date myEffectiveDtm; + + @DocumentId(name = "identifier") + @Column(name = "RESOURCE_IDENTIFIER", nullable = false) + private String myIdentifier; + + public ObservationIndexedSearchParamLastNEntity() {} + + public String getSubject() { + return mySubject; + } + + public void setSubject(String theSubject) { + mySubject = theSubject; + } + + public String getIdentifier() { + return myIdentifier; + } + + public void setIdentifier(String theIdentifier) { + myIdentifier = theIdentifier; + } + + public void setEffectiveDtm(Date theEffectiveDtm) { + myEffectiveDtm = theEffectiveDtm; + } + + public Date getEffectiveDtm() { + return myEffectiveDtm; + } + + public void setCodeNormalizedId(String theCodeNormalizedId) { + myCodeNormalizedId = theCodeNormalizedId; + } + + public String getCodeNormalizedId() { + return myCodeNormalizedId; + } + + + public void setObservationCode(ObservationIndexedCodeCodeableConceptEntity theObservationCode) { + myObservationCode = theObservationCode; + } + + public void setCategoryCodeableConcepts(Set theCategoryCodeableConcepts) { + myCategoryCodeableConcepts = theCategoryCodeableConcepts; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java new file mode 100644 index 00000000000..6cac1cd3627 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/util/CodeSystemHash.java @@ -0,0 +1,33 @@ +package ca.uhn.fhir.jpa.dao.lastn.util; + +import ca.uhn.fhir.util.UrlUtil; +import com.google.common.base.Charsets; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; + +public class CodeSystemHash { + private static final HashFunction HASH_FUNCTION = Hashing.murmur3_128(0); + private static final byte[] DELIMITER_BYTES = "|".getBytes(Charsets.UTF_8); + + static public long hashCodeSystem( String system, String code ) { + Hasher hasher = HASH_FUNCTION.newHasher(); + addStringToHasher(hasher, system); + addStringToHasher(hasher, code); + + HashCode hashCode = hasher.hash(); + return hashCode.asLong(); + } + + static private void addStringToHasher(Hasher hasher, String next) { + if (next == null) { + hasher.putByte((byte) 0); + } else { + next = UrlUtil.escapeUrlParam(next); + byte[] bytes = next.getBytes(Charsets.UTF_8); + hasher.putBytes(bytes); + } + hasher.putBytes(DELIMITER_BYTES); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java new file mode 100644 index 00000000000..91bc3679bf7 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java @@ -0,0 +1,92 @@ +package ca.uhn.fhir.jpa.dao.r4; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; +import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; +import ca.uhn.fhir.model.api.IResource; +import ca.uhn.fhir.rest.api.CacheControlDirective; +import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.param.DateRangeParam; +import ca.uhn.fhir.rest.param.StringAndListParam; +import ca.uhn.fhir.rest.param.StringParam; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.hl7.fhir.r4.model.Observation; + +import javax.servlet.http.HttpServletRequest; +import java.util.Collections; +import java.util.Date; + +public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDao { + + private IBundleProvider doLastNOperation(IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequest) { + SearchParameterMap paramMap = new SearchParameterMap(); + if (theCount != null) { + paramMap.setCount(theCount.getValue()); + } + if (theContent != null) { + paramMap.add(Constants.PARAM_CONTENT, theContent); + } + if (theNarrative != null) { + paramMap.add(Constants.PARAM_TEXT, theNarrative); + } + paramMap.setIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive())); + paramMap.setEverythingMode(theId != null ? EverythingModeEnum.PATIENT_INSTANCE : EverythingModeEnum.PATIENT_TYPE); + paramMap.setSort(theSort); + paramMap.setLastUpdated(theLastUpdated); + if (theId != null) { + paramMap.add("_id", new StringParam(theId.getIdPart())); + } + + if (!isPagingProviderDatabaseBacked(theRequest)) { + paramMap.setLoadSynchronous(true); + } + + return mySearchCoordinatorSvc.registerSearch(this, paramMap, getResourceName(), new CacheControlDirective().parse(theRequest.getHeaders(Constants.HEADER_CACHE_CONTROL)), theRequest); + } + + public IBundleProvider observationsLastN(HttpServletRequest theServletRequest, IIdType theId, IPrimitiveType theCount, DateRangeParam theLastUpdated, SortSpec theSort, StringAndListParam theContent, StringAndListParam theNarrative, StringAndListParam theFilter, RequestDetails theRequestDetails) { + return doLastNOperation(theId, theCount, theLastUpdated, theSort, theContent, theNarrative, theFilter, theRequestDetails); + } + + @Override + public ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, IBasePersistedResource theEntity, Date theDeletedTimestampOrNull, boolean thePerformIndexing, + boolean theUpdateVersion, Date theUpdateTime, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { + ResourceTable retVal = super.updateEntity(theRequest, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theUpdateTime, theForceUpdate, theCreateNewHistoryEntry); + + if(thePerformIndexing) { + // Update indexes here for LastN operation. + Observation observation = (Observation)theResource; + + } + + return retVal; + } + + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java new file mode 100644 index 00000000000..2444195c631 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchBulkIndexSvcImpl.java @@ -0,0 +1,188 @@ +package ca.uhn.fhir.jpa.search.lastn; +/* +import org.shadehapi.elasticsearch.action.DocWriteRequest; +import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.shadehapi.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.shadehapi.elasticsearch.action.bulk.BulkItemResponse; +import org.shadehapi.elasticsearch.action.bulk.BulkRequest; +import org.shadehapi.elasticsearch.action.bulk.BulkResponse; +import org.shadehapi.elasticsearch.action.index.IndexRequest; +import org.shadehapi.elasticsearch.client.RequestOptions; +import org.shadehapi.elasticsearch.client.RestHighLevelClient; +import org.shadehapi.elasticsearch.common.xcontent.XContentType; +*/ +import java.io.IOException; + +public class ElasticsearchBulkIndexSvcImpl { + +// RestHighLevelClient myRestHighLevelClient; + +// BulkRequest myBulkRequest = null; + + public ElasticsearchBulkIndexSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { + +// myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername,thePassword); + + try { + createObservationIndexIfMissing(); + createCodeIndexIfMissing(); + } catch (IOException theE) { + throw new RuntimeException("Failed to create document index", theE); + } + } + + public void createObservationIndexIfMissing() throws IOException { + if(indexExists(IndexConstants.OBSERVATION_INDEX)) { + return; + } + String observationMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeconceptid\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"norms\" : true\n" + + " },\n" + + " \"codeconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codeconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"effectivedtm\" : {\n" + + " \"type\" : \"date\"\n" + + " },\n" + + " \"identifier\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"subject\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if(!createIndex(IndexConstants.OBSERVATION_INDEX, observationMapping)) { + throw new RuntimeException("Failed to create observation index"); + } + + } + + public void createCodeIndexIfMissing() throws IOException { + if(indexExists(IndexConstants.CODE_INDEX)) { + return; + } + String codeMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeable_concept_id\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"codeable_concept_text\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if (!createIndex(IndexConstants.CODE_INDEX, codeMapping)) { + throw new RuntimeException("Failed to create code index"); + } + + } + + public boolean createIndex(String theIndexName, String theMapping) throws IOException { +/* CreateIndexRequest request = new CreateIndexRequest(theIndexName); + request.source(theMapping, XContentType.JSON); + CreateIndexResponse createIndexResponse = myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT); + return createIndexResponse.isAcknowledged(); + */ + return false; + } + + public boolean indexExists(String theIndexName) throws IOException { +/* GetIndexRequest request = new GetIndexRequest(); + request.indices(theIndexName); + return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); + + */ + return false; + } + + public void addToBulkIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { +/* IndexRequest request = new IndexRequest(theIndexName); + request.id(theDocumentId); + request.type(theDocumentType); + + request.source(theObservationDocument, XContentType.JSON); + + if (myBulkRequest == null) { + myBulkRequest = new BulkRequest(); + } + myBulkRequest.add(request); */ + } + + public void executeBulkIndex() throws IOException { +/* if (myBulkRequest == null) { + throw new RuntimeException(("No index requests have been added to the bulk request")); + } + BulkResponse bulkResponse = myRestHighLevelClient.bulk(myBulkRequest, RequestOptions.DEFAULT); + for (BulkItemResponse bulkItemResponse : bulkResponse) { + if (bulkItemResponse.getOpType() != DocWriteRequest.OpType.CREATE && bulkItemResponse.getOpType() != DocWriteRequest.OpType.INDEX) { + throw new RuntimeException("Unexpected response for bulk index request: " + bulkItemResponse.getOpType()); + } + } + myBulkRequest = null; */ + } + + public boolean bulkRequestPending() { +// return (myBulkRequest != null); + return false; + } + + public void closeClient() throws IOException { +// myRestHighLevelClient.close(); + } + + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchRestClientFactory.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchRestClientFactory.java new file mode 100644 index 00000000000..38c2c5c4f6d --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchRestClientFactory.java @@ -0,0 +1,33 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.shadehapi.elasticsearch.client.RestClient; +import org.shadehapi.elasticsearch.client.RestClientBuilder; +import org.shadehapi.elasticsearch.client.RestHighLevelClient; + +public class ElasticsearchRestClientFactory { + + static public RestHighLevelClient createElasticsearchHighLevelRestClient(String theHostname, int thePort, String theUsername, String thePassword) { + final CredentialsProvider credentialsProvider = + new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(theUsername, thePassword)); + + RestClientBuilder clientBuilder = RestClient.builder( + new HttpHost(theHostname, thePort)) + .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider)); + + Header[] defaultHeaders = new Header[]{new BasicHeader("Content-Type", "application/json")}; + clientBuilder.setDefaultHeaders(defaultHeaders); + + return new RestHighLevelClient(clientBuilder); + + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java new file mode 100644 index 00000000000..62cc30fd6a1 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java @@ -0,0 +1,550 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.api.IQueryParameterType; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +//import org.assertj.core.util.VisibleForTesting; +import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.shadehapi.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.shadehapi.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.shadehapi.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.shadehapi.elasticsearch.action.DocWriteResponse; +import org.shadehapi.elasticsearch.action.index.IndexRequest; +import org.shadehapi.elasticsearch.action.index.IndexResponse; +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.shadehapi.elasticsearch.action.support.master.AcknowledgedResponse; +import org.shadehapi.elasticsearch.client.RequestOptions; +import org.shadehapi.elasticsearch.client.RestHighLevelClient; +import org.shadehapi.elasticsearch.common.xcontent.XContentType; +import org.shadehapi.elasticsearch.index.query.BoolQueryBuilder; +import org.shadehapi.elasticsearch.index.query.QueryBuilders; +import org.shadehapi.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.shadehapi.elasticsearch.search.SearchHit; +import org.shadehapi.elasticsearch.search.SearchHits; +import org.shadehapi.elasticsearch.search.aggregations.AggregationBuilder; +import org.shadehapi.elasticsearch.search.aggregations.AggregationBuilders; +import org.shadehapi.elasticsearch.search.aggregations.Aggregations; +import org.shadehapi.elasticsearch.search.aggregations.bucket.composite.*; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.shadehapi.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits; +import org.shadehapi.elasticsearch.search.aggregations.support.ValueType; +import org.shadehapi.elasticsearch.search.builder.SearchSourceBuilder; +import org.shadehapi.elasticsearch.search.sort.SortOrder; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.commons.lang3.StringUtils.isBlank; + +//@Component +public class ElasticsearchSvcImpl implements IElasticsearchSvc { + + RestHighLevelClient myRestHighLevelClient; + + ObjectMapper objectMapper = new ObjectMapper(); + + + public ElasticsearchSvcImpl(String theHostname, int thePort, String theUsername, String thePassword) { + + myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theHostname, thePort, theUsername,thePassword); + + try { + createObservationIndexIfMissing(); + createCodeIndexIfMissing(); + } catch (IOException theE) { + throw new RuntimeException("Failed to create document index", theE); + } + } + + public void createObservationIndexIfMissing() throws IOException { + if(indexExists(IndexConstants.OBSERVATION_INDEX)) { + return; + } + String observationMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeconceptid\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"norms\" : true\n" + + " },\n" + + " \"codeconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codeconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codeconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconcepttext\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"categoryconceptcodingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"categoryconceptcodingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"effectivedtm\" : {\n" + + " \"type\" : \"date\"\n" + + " },\n" + + " \"identifier\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"subject\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if(!createIndex(IndexConstants.OBSERVATION_INDEX, observationMapping)) { + throw new RuntimeException("Failed to create observation index"); + } + + } + + public void createCodeIndexIfMissing() throws IOException { + if(indexExists(IndexConstants.CODE_INDEX)) { + return; + } + String codeMapping = "{\n" + + " \"mappings\" : {\n" + + " \"ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity\" : {\n" + + " \"properties\" : {\n" + + " \"codeable_concept_id\" : {\n" + + " \"type\" : \"keyword\",\n" + + " \"store\" : true\n" + + " },\n" + + " \"codeable_concept_text\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingcode\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingcode_system_hash\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"codingdisplay\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"codingsystem\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + if (!createIndex(IndexConstants.CODE_INDEX, codeMapping)) { + throw new RuntimeException("Failed to create code index"); + } + + } + + public boolean createIndex(String theIndexName, String theMapping) throws IOException { + CreateIndexRequest request = new CreateIndexRequest(theIndexName); + request.source(theMapping, XContentType.JSON); + CreateIndexResponse createIndexResponse = myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT); + return createIndexResponse.isAcknowledged(); + + } + + public boolean performIndex(String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) throws IOException { + + IndexResponse indexResponse = myRestHighLevelClient.index(createIndexRequest(theIndexName, theDocumentId,theIndexDocument,theDocumentType), + RequestOptions.DEFAULT); + + return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); + } + + private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { + IndexRequest request = new IndexRequest(theIndexName); + request.id(theDocumentId); + request.type(theDocumentType); + + request.source(theObservationDocument, XContentType.JSON); + return request; + } + + public boolean indexExists(String theIndexName) throws IOException { + GetIndexRequest request = new GetIndexRequest(); + request.indices(theIndexName); + return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); + } + + SearchRequest buildObservationCodesSearchRequest(int theMaxResultSetSize) { + SearchRequest searchRequest = new SearchRequest(IndexConstants.CODE_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + // Query + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.size(theMaxResultSetSize); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + private CompositeAggregationBuilder createCompositeAggregationBuilder(int theMaximumResultSetSize, int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { + TermsAggregationBuilder observationCodeAggregationBuilder = new TermsAggregationBuilder("group_by_code", ValueType.STRING).field("codeconceptid"); + // Top Hits Aggregation + observationCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + .sort("effectivedtm", SortOrder.DESC) + .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); + observationCodeAggregationBuilder.size(theMaximumResultSetSize); + CompositeValuesSourceBuilder subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject"); + List> compositeAggSubjectSources = new ArrayList(); + compositeAggSubjectSources.add(subjectValuesBuilder); + CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder("group_by_subject", compositeAggSubjectSources); + compositeAggregationSubjectBuilder.subAggregation(observationCodeAggregationBuilder); + + return compositeAggregationSubjectBuilder; + } + + private TermsAggregationBuilder createTermsAggregationBuilder(int theMaximumResultSetSize, int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { + TermsAggregationBuilder observationCodeAggregationBuilder = new TermsAggregationBuilder("group_by_code", ValueType.STRING).field("codeconceptid"); + // Top Hits Aggregation + observationCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + .sort("effectivedtm", SortOrder.DESC).fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode)); + observationCodeAggregationBuilder.size(theMaximumResultSetSize); + TermsAggregationBuilder subjectsBuilder = new TermsAggregationBuilder("group_by_subject", ValueType.STRING).field("subject"); + subjectsBuilder.subAggregation(observationCodeAggregationBuilder); + subjectsBuilder.size(theMaximumResultSetSize); + return subjectsBuilder; + } + + public SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { + return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + } + + public List buildObservationCompositeResults(SearchResponse theSearchResponse) throws IOException { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedComposite aggregatedSubjects = responseAggregations.get("group_by_subject"); + List subjectBuckets = aggregatedSubjects.getBuckets(); + List codes = new ArrayList<>(); + for(ParsedComposite.ParsedBucket subjectBucket : subjectBuckets) { + Aggregations observationCodeAggregations = subjectBucket.getAggregations(); + ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); + List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); + for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { + Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + SearchHit[] topHits = parsedTopHits.getHits().getHits(); + for (SearchHit topHit : topHits) { + String sources = topHit.getSourceAsString(); + ObservationJson code = objectMapper.readValue(sources, ObservationJson.class); + codes.add(code); + } + } + } + return codes; + } + + public List buildObservationTermsResults(SearchResponse theSearchResponse) throws IOException { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedTerms aggregatedSubjects = responseAggregations.get("group_by_subject"); + List subjectBuckets = aggregatedSubjects.getBuckets(); + List codes = new ArrayList<>(); + for (Terms.Bucket subjectBucket : subjectBuckets) { + Aggregations observationCodeAggregations = subjectBucket.getAggregations(); + ParsedTerms aggregatedObservationCodes = observationCodeAggregations.get("group_by_code"); + List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); + for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { + Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + SearchHit[] topHits = parsedTopHits.getHits().getHits(); + for (SearchHit topHit : topHits) { + String sources = topHit.getSourceAsString(); + ObservationJson code = objectMapper.readValue(sources,ObservationJson.class); + codes.add(code); + } + } + } + return codes; + } + + public List buildCodeResult(SearchResponse theSearchResponse) throws JsonProcessingException { + SearchHits codeHits = theSearchResponse.getHits(); + List codes = new ArrayList<>(); + for (SearchHit codeHit : codeHits) { + CodeJson code = objectMapper.readValue(codeHit.getSourceAsString(), CodeJson.class); + codes.add(code); + } + return codes; + } + +// @VisibleForTesting + public SearchRequest buildObservationAllFieldsCompositeSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { + + return buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, null)); + } + + public SearchRequest buildObservationCompositeSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { + // Return only identifiers + String[] topHitsInclude = {"identifier","subject","effectivedtm","codeconceptid"}; + return buildObservationsSearchRequest(theSearchParameterMap, createCompositeAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, topHitsInclude)); + } + +// @VisibleForTesting + public SearchRequest buildObservationAllFieldsTermsSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { + return buildObservationsSearchRequest(theSearchParameterMap, createTermsAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, null)); + } + + public SearchRequest buildObservationTermsSearchRequest(int maxResultSetSize, SearchParameterMap theSearchParameterMap, int theMaxObservationsPerCode) { + // Return only identifiers + String[] topHitsInclude = {"identifier","subject","effectivedtm","codeconceptid"}; + return buildObservationsSearchRequest(theSearchParameterMap, createTermsAggregationBuilder(maxResultSetSize, theMaxObservationsPerCode, topHitsInclude)); + } + + private SearchRequest buildObservationsSearchRequest(SearchParameterMap theSearchParameterMap, AggregationBuilder theAggregationBuilder) { + SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + // Query + if(!searchParamsHaveLastNCriteria(theSearchParameterMap)) { + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + } else { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + addSubjectsCriteria(boolQueryBuilder,theSearchParameterMap); + addCategoriesCriteria(boolQueryBuilder,theSearchParameterMap); + addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap); + searchSourceBuilder.query(boolQueryBuilder); + } + searchSourceBuilder.size(0); + + // Aggregation by order codes + searchSourceBuilder.aggregation(theAggregationBuilder); + searchRequest.source(searchSourceBuilder); + + return searchRequest; + } + + private Boolean searchParamsHaveLastNCriteria(SearchParameterMap theSearchParameterMap) { + return theSearchParameterMap != null && + (theSearchParameterMap.containsKey("patient") || theSearchParameterMap.containsKey("subject") || + theSearchParameterMap.containsKey("category") || theSearchParameterMap.containsKey("code")); + } + + private void addSubjectsCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.containsKey("patient") || theSearchParameterMap.containsKey("subject")) { + ArrayList subjectReferenceCriteria = new ArrayList<>(); + List> andOrParams = new ArrayList<>(); + if (theSearchParameterMap.get("patient") != null) { + andOrParams.addAll(theSearchParameterMap.get("patient")); + } + if (theSearchParameterMap.get("subject") != null) { + andOrParams.addAll(theSearchParameterMap.get("subject")); + } + for (List nextAnd : andOrParams) { + subjectReferenceCriteria.addAll(getReferenceValues(nextAnd)); + } + if (subjectReferenceCriteria.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("subject", subjectReferenceCriteria)); + } + } + + } + + private List getReferenceValues(List referenceParams) { + ArrayList referenceList = new ArrayList<>(); + + for (IQueryParameterType nextOr : referenceParams) { + + if (nextOr instanceof ReferenceParam) { + ReferenceParam ref = (ReferenceParam) nextOr; + if (isBlank(ref.getChain())) { + if (ref.getResourceType() != null) { + referenceList.add(ref.getResourceType() + "/" + ref.getValue()); + } else { + referenceList.add(ref.getValue()); + } + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting ReferenceParam): " + nextOr.getClass()); + } + } + return referenceList; + } + + private void addCategoriesCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.containsKey("category")) { + ArrayList codeSystemHashList = new ArrayList<>(); + ArrayList codeOnlyList = new ArrayList<>(); + ArrayList systemOnlyList = new ArrayList<>(); + ArrayList textOnlyList = new ArrayList<>(); + List> andOrParams = theSearchParameterMap.get("category"); + for (List nextAnd : andOrParams) { + codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); + codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); + systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); + textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); + } + if (codeSystemHashList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode_system_hash", codeSystemHashList)); + } + if (codeOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingcode", codeOnlyList)); + } + if (systemOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("categoryconceptcodingsystem", systemOnlyList)); + } + if (textOnlyList.size() > 0) { + BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); + for (String textOnlyParam : textOnlyList) { + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("categoryconcepttext", textOnlyParam)); + } + theBoolQueryBuilder.must(myTextBoolQueryBuilder); + } + } + + } + + private List getCodingCodeSystemValues(List codeParams) { + ArrayList codeSystemHashList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams ) { + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.getSystem() != null && ref.getValue() != null) { + codeSystemHashList.add(String.valueOf(CodeSystemHash.hashCodeSystem(ref.getSystem(), ref.getValue()))); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return codeSystemHashList; + } + + private List getCodingCodeOnlyValues(List codeParams ) { + ArrayList codeOnlyList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams) { + + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.getValue() != null && ref.getSystem() == null && !ref.isText()) { + codeOnlyList.add(ref.getValue()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return codeOnlyList; + } + + private List getCodingSystemOnlyValues(List codeParams ) { + ArrayList systemOnlyList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams) { + + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.getValue() == null && ref.getSystem() != null) { + systemOnlyList.add(ref.getSystem()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return systemOnlyList; + } + + private List getCodingTextOnlyValues(List codeParams ) { + ArrayList textOnlyList = new ArrayList<>(); + for (IQueryParameterType nextOr : codeParams ) { + + if (nextOr instanceof TokenParam) { + TokenParam ref = (TokenParam) nextOr; + if (ref.isText() && ref.getValue() != null) { + textOnlyList.add(ref.getValue()); + } + } else { + throw new IllegalArgumentException("Invalid token type (expecting TokenParam): " + nextOr.getClass()); + } + } + return textOnlyList; + } + + private void addObservationCodeCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap) { + if (theSearchParameterMap.containsKey("code")) { + ArrayList codeSystemHashList = new ArrayList<>(); + ArrayList codeOnlyList = new ArrayList<>(); + ArrayList systemOnlyList = new ArrayList<>(); + ArrayList textOnlyList = new ArrayList<>(); + List> andOrParams = theSearchParameterMap.get("code"); + for (List nextAnd : andOrParams) { + codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); + codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); + systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); + textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); + } + if (codeSystemHashList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode_system_hash", codeSystemHashList)); + } + if (codeOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingcode", codeOnlyList)); + } + if (systemOnlyList.size() > 0) { + theBoolQueryBuilder.must(QueryBuilders.termsQuery("codeconceptcodingsystem", systemOnlyList)); + } + if (textOnlyList.size() > 0) { + BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); + for (String textOnlyParam : textOnlyList) { + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconceptcodingdisplay", textOnlyParam)); + myTextBoolQueryBuilder.should(QueryBuilders.matchPhraseQuery("codeconcepttext", textOnlyParam)); + } + theBoolQueryBuilder.must(myTextBoolQueryBuilder); + } + } + + } + +// @VisibleForTesting + public boolean deleteIndex(String theIndexName) throws IOException { + DeleteIndexRequest request = new DeleteIndexRequest(theIndexName); + AcknowledgedResponse deleteIndexResponse = myRestHighLevelClient.indices().delete(request, RequestOptions.DEFAULT); + + return deleteIndexResponse.isAcknowledged(); + } + + + +// @VisibleForTesting + public void deleteAllDocuments(String theIndexName) throws IOException { + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(theIndexName); + deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); + myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); + } + +/* @Override + public IBundleProvider lastN(HttpServletRequest theServletRequest, RequestDetails theRequestDetails) { + + return null; + } + + @Override + public IBundleProvider uniqueCodes(HttpServletRequest theServletRequest, RequestDetails theRequestDetails) { + return null; + } + + */ +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java new file mode 100644 index 00000000000..4c2be1cb2b5 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java @@ -0,0 +1,12 @@ +package ca.uhn.fhir.jpa.search.lastn; + +//import ca.uhn.fhir.rest.api.server.IBundleProvider; + +public interface IElasticsearchSvc { + + +// IBundleProvider lastN(javax.servlet.http.HttpServletRequest theServletRequest, ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails); + +// IBundleProvider uniqueCodes(javax.servlet.http.HttpServletRequest theServletRequest, ca.uhn.fhir.rest.api.server.RequestDetails theRequestDetails); + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java new file mode 100644 index 00000000000..94276924896 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.jpa.search.lastn; + +public class IndexConstants { + + public static final String OBSERVATION_INDEX = "observation_index"; + public static final String CODE_INDEX = "code_index"; + public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; + public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity"; +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java new file mode 100644 index 00000000000..10c0453431e --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/MulticodeObservationsIntoElasticSearch.java @@ -0,0 +1,122 @@ +package ca.uhn.fhir.jpa.search.lastn.cli; + +import ca.uhn.fhir.jpa.search.lastn.json.IdJson; +import ca.uhn.fhir.jpa.search.lastn.json.IndexJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +public class MulticodeObservationsIntoElasticSearch { + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(MulticodeObservationsIntoElasticSearch.class); + + private static final ObjectMapper ourMapperNonPrettyPrint; + + static { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + public static void main(String[] theArgs) { + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subject = "Patient/"+UUID.randomUUID().toString(); + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + String nextResourceId = UUID.randomUUID().toString(); + + IdJson id = new IdJson(nextResourceId); + IndexJson documentIndex = new IndexJson(id); + + ObservationJson observationDocument = new ObservationJson(); + observationDocument.setIdentifier(nextResourceId); + observationDocument.setSubject(subject); + // Add three CodeableConcepts for category + List category = new ArrayList<>(); + // Create three codings and first category CodeableConcept + Coding categoryCoding1_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test heart-rate"); + Coding categoryCoding1_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test alternate heart-rate"); + Coding categoryCoding1_3 = new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test second alternate heart-rate"); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept(); + categoryCodeableConcept1.getCoding().add(categoryCoding1_1); + categoryCodeableConcept1.getCoding().add(categoryCoding1_2); + categoryCodeableConcept1.getCoding().add(categoryCoding1_3); + categoryCodeableConcept1.setText("Heart Rate Codeable Concept"); + category.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + Coding categoryCoding2_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test vital signs"); + Coding categoryCoding2_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test alternate vital signs"); + Coding categoryCoding2_3 = new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test second alternate vital signs"); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept(); + categoryCodeableConcept2.getCoding().add(categoryCoding2_1); + categoryCodeableConcept2.getCoding().add(categoryCoding2_2); + categoryCodeableConcept2.getCoding().add(categoryCoding2_3); + categoryCodeableConcept2.setText("Vital Signs Codeable Concept"); + category.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + Coding categoryCoding3_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test vital signs panel"); + Coding categoryCoding3_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test alternate vital signs panel"); + Coding categoryCoding3_3 = new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test second alternate vital signs panel"); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept(); + categoryCodeableConcept3.getCoding().add(categoryCoding3_1); + categoryCodeableConcept3.getCoding().add(categoryCoding3_2); + categoryCodeableConcept3.getCoding().add(categoryCoding3_3); + categoryCodeableConcept3.setText("Vital Signs Panel Codeable Concept"); + category.add(categoryCodeableConcept3); + observationDocument.setCategories(category); + + Coding codeCoding1 = new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test observation code"); + Coding codeCoding2 = new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test observation code"); + Coding codeCoding3 = new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test observation code"); + CodeableConcept code = new CodeableConcept(); + code.getCoding().add(codeCoding1); + code.getCoding().add(codeCoding2); + code.getCoding().add(codeCoding3); + code.setText("Observation code CodeableConcept"); + observationDocument.setCode(code); + observationDocument.setCode_concept_id("multicode_test_normalized_code"); + + Date effectiveDtm = new Date(); + observationDocument.setEffectiveDtm(effectiveDtm); + + StringWriter stringWriter = new StringWriter(); + + File outputFile = new File("Observations_multiplecodes.json"); + try { + FileOutputStream outputStream = new FileOutputStream(outputFile, true); + ourMapperNonPrettyPrint.writeValue(stringWriter, documentIndex); + stringWriter.append('\n'); + ourMapperNonPrettyPrint.writeValue(stringWriter, observationDocument); + ourMapperNonPrettyPrint.writeValue(outputStream, documentIndex); + outputStream.write('\n'); + ourMapperNonPrettyPrint.writeValue(outputStream, observationDocument); + outputStream.write('\n'); + outputStream.flush(); + outputStream.close(); + } catch (IOException theE) { + theE.printStackTrace(); + } + } + + } + + ourLog.info("Upload complete"); + + } + +} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java new file mode 100644 index 00000000000..300fbba9315 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneMillionPatientsIntoElasticSearch.java @@ -0,0 +1,128 @@ +package ca.uhn.fhir.jpa.search.lastn.cli; + +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchBulkIndexSvcImpl; +import ca.uhn.fhir.jpa.search.lastn.IndexConstants; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.SimpleStopWatch; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.commons.lang3.RandomStringUtils; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.io.IOException; +import java.util.*; + +public class OneMillionPatientsIntoElasticSearch { +// private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(OneMillionPatientsIntoElasticSearch.class); + + private static final ObjectMapper ourMapperNonPrettyPrint; + + static { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + public static void main(String[] theArgs) throws IOException { + + ElasticsearchBulkIndexSvcImpl elasticsearchSvc = new ElasticsearchBulkIndexSvcImpl("localhost",9301, "elastic", "changeme"); + try { + + SimpleStopWatch stopwatch = new SimpleStopWatch(); + + List observationCodeIds = new ArrayList<>(); + List observationCodes = new ArrayList<>(); + + for (int codeCount = 0; codeCount < 1000; codeCount++) { + String code = RandomStringUtils.random(10,true,true); + Coding codeCoding = new Coding("http://mycodes.org/fhir/observation-code", code, "test observation code " + code); + CodeableConcept codeConcept = new CodeableConcept(); + codeConcept.getCoding().add(codeCoding); + codeConcept.setText("test observation code concept " + code); + String codeableConceptId = UUID.randomUUID().toString(); + observationCodeIds.add(codeableConceptId); + observationCodes.add(codeConcept); + CodeJson codeDocument = new CodeJson(codeConcept, codeableConceptId); + String printedObservationDocument = ourMapperNonPrettyPrint.writeValueAsString(codeDocument); + System.out.println(printedObservationDocument); + System.out.println(codeableConceptId); + elasticsearchSvc.addToBulkIndexRequest("code_index", codeableConceptId, printedObservationDocument, IndexConstants.CODE_DOCUMENT_TYPE); + if ((codeCount+1)%250 == 0) { + elasticsearchSvc.executeBulkIndex(); + long elapsedTime = stopwatch.getElapsedTime(); + stopwatch.restart(); + System.out.println("Elapsed processing time = " + elapsedTime/1000 + "s"); + System.out.println("Average processing time/code = " + elapsedTime/5000 + "ms"); + } + } + + for (int patientCount = 0; patientCount < 1000000 ; patientCount++) { + String subject = "Patient/"+UUID.randomUUID().toString(); + ArrayList observationCodesSubSet = new ArrayList<>(); + ArrayList observationCodeIdsSubSet = new ArrayList<>(); + for (int observationCount = 0; observationCount < 15 ; observationCount++) { + int codeIndex = (int) (1000 * Math.random()); + observationCodesSubSet.add(observationCodes.get(codeIndex)); + observationCodeIdsSubSet.add(observationCodeIds.get(codeIndex)); + } + int repeatedCodeIndex = (int) (1000 * Math.random()); + CodeableConcept repeatedCoding = observationCodes.get(repeatedCodeIndex); + for (int observationCount = 0; observationCount < 10 ; observationCount++ ) { + observationCodesSubSet.add(repeatedCoding); + observationCodeIdsSubSet.add(observationCodeIds.get(repeatedCodeIndex)); + } + int entryCount = 0; + for (int codingCount=0; codingCount category = new ArrayList<>(); + Coding categoryCoding = new Coding("http://mycodes.org/fhir/category-code", "test-category-code", "test category display"); + CodeableConcept categoryCodeableConcept = new CodeableConcept(); + categoryCodeableConcept.getCoding().add(categoryCoding); + categoryCodeableConcept.setText("Test Category CodeableConcept Text"); + category.add(categoryCodeableConcept); + observationDocument.setCategories(category); + observationDocument.setCode(observationCodesSubSet.get(codingCount)); + observationDocument.setCode_concept_id(observationCodeIdsSubSet.get(codingCount)); + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + entryCount++; + Date effectiveDtm = observationDate.getTime(); + observationDocument.setEffectiveDtm(effectiveDtm); + + String printedObservationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationDocument); + elasticsearchSvc.addToBulkIndexRequest("observation_index", nextResourceId, printedObservationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE ); + } + if ((patientCount+1)%100 == 0) { + System.out.println("Entries created = " + (patientCount+1)*25); + } + if ((patientCount+1)%250 == 0) { + elasticsearchSvc.executeBulkIndex(); + long elapsedTime = stopwatch.getElapsedTime(); + stopwatch.restart(); + System.out.println("Elapsed processing time = " + elapsedTime/1000 + "s"); + System.out.println("Average processing time/observation = " + elapsedTime/5000 + "ms"); + } + + + } + + if (elasticsearchSvc.bulkRequestPending()) { + elasticsearchSvc.executeBulkIndex(); + } + + // ourLog.info("Upload complete"); + } finally { + elasticsearchSvc.closeClient(); + } + + } + +} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java new file mode 100644 index 00000000000..3a6d2218a3a --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/OneThousandObservationsIntoElasticSearch.java @@ -0,0 +1,116 @@ +package ca.uhn.fhir.jpa.search.lastn.cli; + +import ca.uhn.fhir.jpa.search.lastn.json.IdJson; +import ca.uhn.fhir.jpa.search.lastn.json.IndexJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +public class OneThousandObservationsIntoElasticSearch { + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(OneThousandObservationsIntoElasticSearch.class); + + private static final ObjectMapper ourMapperNonPrettyPrint; + + static { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + public static void main(String[] theArgs) { + + for (int patientCount = 0; patientCount < 3 ; patientCount++) { + + String subject = "Patient/"+UUID.randomUUID().toString(); + + for ( int entryCount = 0; entryCount < 1100 ; entryCount++ ) { + String nextResourceId = UUID.randomUUID().toString(); + + IdJson id = new IdJson(nextResourceId); + IndexJson documentIndex = new IndexJson(id); + + ObservationJson observationDocument = new ObservationJson(); + observationDocument.setIdentifier(nextResourceId); + observationDocument.setSubject(subject); + // Add three CodeableConcepts for category + List category = new ArrayList<>(); + // Create three codings and first category CodeableConcept + Coding categoryCoding1_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test heart-rate"); + Coding categoryCoding1_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test alternate heart-rate"); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept(); + categoryCodeableConcept1.getCoding().add(categoryCoding1_1); + categoryCodeableConcept1.getCoding().add(categoryCoding1_2); + categoryCodeableConcept1.setText("Heart Rate CodeableConcept"); + category.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + Coding categoryCoding2_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test vital signs"); + Coding categoryCoding2_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test alternate vital signs"); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept(); + categoryCodeableConcept2.getCoding().add(categoryCoding2_1); + categoryCodeableConcept2.getCoding().add(categoryCoding2_2); + categoryCodeableConcept2.setText("Vital Signs CodeableConcept"); + category.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + Coding categoryCoding3_1 = new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test vital signs panel"); + Coding categoryCoding3_2 = new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test alternate vital signs panel"); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept(); + categoryCodeableConcept3.getCoding().add(categoryCoding3_1); + categoryCodeableConcept3.getCoding().add(categoryCoding3_2); + categoryCodeableConcept3.setText("Vital Signs Panel CodeableConcept"); + category.add(categoryCodeableConcept3); + observationDocument.setCategories(category); + + Coding codeCoding1 = new Coding("http://mycodes.org/fhir/observation-code", "test-code_" + entryCount, "test observation code"); + Coding codeCoding2 = new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code_" + entryCount, "test observation code"); + Coding codeCoding3 = new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code" + entryCount, "test observation code"); + CodeableConcept code = new CodeableConcept(); + code.getCoding().add(codeCoding1); + code.getCoding().add(codeCoding2); + code.getCoding().add(codeCoding3); + code.setText("Observation code CodeableConcept " + entryCount); + observationDocument.setCode(code); + observationDocument.setCode_concept_id("multicode_test_normalized_code_" + entryCount); + + Date effectiveDtm = new Date(); + observationDocument.setEffectiveDtm(effectiveDtm); + + StringWriter stringWriter = new StringWriter(); + + File outputFile = new File("one_thousand_observations.json"); + try { + FileOutputStream outputStream = new FileOutputStream(outputFile, true); + ourMapperNonPrettyPrint.writeValue(stringWriter, documentIndex); + stringWriter.append('\n'); + ourMapperNonPrettyPrint.writeValue(stringWriter, observationDocument); + ourMapperNonPrettyPrint.writeValue(outputStream, documentIndex); + outputStream.write('\n'); + ourMapperNonPrettyPrint.writeValue(outputStream, observationDocument); + outputStream.write('\n'); + outputStream.flush(); + outputStream.close(); + } catch (IOException theE) { + theE.printStackTrace(); + } + } + + } + + ourLog.info("Upload complete"); + + } + +} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java new file mode 100644 index 00000000000..bfe2e82bd8f --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/cli/UploadSampleDatasetIntoElasticSearch.java @@ -0,0 +1,153 @@ +package ca.uhn.fhir.jpa.search.lastn.cli; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.parser.IParser; +import ca.uhn.fhir.parser.LenientErrorHandler; +import ca.uhn.fhir.jpa.search.lastn.json.IdJson; +import ca.uhn.fhir.jpa.search.lastn.json.IndexJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.base.Charsets; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.io.IOUtils; +import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; + +import java.io.*; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +public class UploadSampleDatasetIntoElasticSearch { + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(UploadSampleDatasetIntoElasticSearch.class); + + private static final ObjectMapper ourMapperNonPrettyPrint; + + static { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + public static void main(String[] theArgs) { + + FhirContext myFhirCtx = FhirContext.forR4(); + myFhirCtx.getRestfulClientFactory().setSocketTimeout(120000); + + PathMatchingResourcePatternResolver provider = new PathMatchingResourcePatternResolver(); + final Resource[] bundleResources; + try { + bundleResources = provider.getResources("*.json.bz2"); + } catch (IOException e) { + throw new RuntimeException("Unexpected error during transmission: " + e.toString(), e); + } + + AtomicInteger index = new AtomicInteger(); + + Arrays.stream(bundleResources).forEach( + resource -> { + index.incrementAndGet(); + + InputStream resIs = null; + String nextBundleString; + try { + resIs = resource.getInputStream(); + resIs = new BZip2CompressorInputStream(resIs); + nextBundleString = IOUtils.toString(resIs, Charsets.UTF_8); + } catch (IOException e) { + ourLog.error("Failure reading: {}", resource.getFilename(), e); + return; + } finally { + try { + if (resIs != null) { + resIs.close(); + } + } catch (final IOException ioe) { + // ignore + } + } + + ourLog.info("Uploading {}/{} - {} ({} bytes)", index, bundleResources.length, resource.getFilename(), nextBundleString.length()); + + /* + * SMART demo apps rely on the use of LOINC 3141-9 (Body Weight Measured) + * instead of LOINC 29463-7 (Body Weight) + */ + nextBundleString = nextBundleString.replace("\"29463-7\"", "\"3141-9\""); + + IParser parser = myFhirCtx.newJsonParser(); + parser.setParserErrorHandler(new LenientErrorHandler(false)); + Bundle bundle = parser.parseResource(Bundle.class, nextBundleString); + + for (BundleEntryComponent nextEntry : bundle.getEntry()) { + + /* + * Synthea gives resources UUIDs with urn:uuid: prefix, which is only + * used for placeholders. We're going to use these as the actual resource + * IDs, so we strip the prefix. + */ + String nextResourceId = nextEntry.getFullUrl(); + if (nextResourceId == null) { + nextResourceId = UUID.randomUUID().toString(); + } + + nextResourceId = nextResourceId.replace("urn:uuid:", ""); + nextEntry.getResource().setId(nextResourceId); + nextEntry.setFullUrl(nextResourceId); + + if (nextEntry.getResource().getResourceType().equals(ResourceType.Observation)) { + + IdJson id = new IdJson(nextResourceId); + IndexJson documentIndex = new IndexJson(id); + + org.hl7.fhir.r4.model.Observation observation = (Observation) nextEntry.getResource(); + ObservationJson observationDocument = new ObservationJson(); + observationDocument.setIdentifier(nextResourceId); + String subject = "Patient/"+observation.getSubject().getReference(); + observationDocument.setSubject(subject); + List category = observation.getCategory(); + observationDocument.setCategories(category); + observationDocument.setCode_concept_id(category.get(0).getCodingFirstRep().getSystem() + "/" + category.get(0).getCodingFirstRep().getCode()); + CodeableConcept code = observation.getCode(); + observationDocument.setCode(code); + Date effectiveDtm = observation.getEffectiveDateTimeType().getValue(); + observationDocument.setEffectiveDtm(effectiveDtm); + + StringWriter stringWriter = new StringWriter(); + File outputFile = new File("Observations.json"); + try { + FileOutputStream outputStream = new FileOutputStream(outputFile, true); + ourMapperNonPrettyPrint.writeValue(stringWriter, documentIndex); + stringWriter.append('\n'); + ourMapperNonPrettyPrint.writeValue(stringWriter, observationDocument); + ourMapperNonPrettyPrint.writeValue(outputStream, documentIndex); + outputStream.write('\n'); + ourMapperNonPrettyPrint.writeValue(outputStream, observationDocument); + outputStream.write('\n'); + outputStream.flush(); + outputStream.close(); + } catch (IOException theE) { + theE.printStackTrace(); + } + System.out.println(stringWriter.toString()); + + } + + } + + } + ); + + ourLog.info("Upload complete"); + + } + +} \ No newline at end of file diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java new file mode 100644 index 00000000000..3ede8b1d67f --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/config/ElasticsearchConfig.java @@ -0,0 +1,26 @@ +package ca.uhn.fhir.jpa.search.lastn.config; + +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import java.io.IOException; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") +@EnableTransactionManagement +public class ElasticsearchConfig { + + private final String elasticsearchHost = "127.0.0.1"; + private final Integer elasticsearchPort = 9301; + private final String elasticsearchUserId = "elastic"; + private final String elasticsearchPassword = "changeme"; + + @Bean() + public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java new file mode 100644 index 00000000000..032279866e2 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java @@ -0,0 +1,82 @@ +package ca.uhn.fhir.jpa.search.lastn.json; + +/* + * #%L + * Smile CDR - CDR + * %% + * Copyright (C) 2016 - 2019 Simpatico Intelligent Systems Inc + * %% + * All rights reserved. + * #L% + */ + +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.util.ArrayList; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class CodeJson { + + @JsonProperty(value = "codeable_concept_id", required = false) + private String myCodeableConceptId; + + @JsonProperty(value = "codeable_concept_text", required = false) + private String myCodeableConceptText; + + @JsonProperty(value = "codingcode", required = false) + private List myCoding_code = new ArrayList<>(); + + @JsonProperty(value = "codingcode_system_hash", required = true) + private List myCoding_code_system_hash = new ArrayList<>(); + + @JsonProperty(value = "codingdisplay", required = false) + private List myCoding_display = new ArrayList<>(); + + @JsonProperty(value = "codingsystem", required = false) + private List myCoding_system = new ArrayList<>(); + + public CodeJson(){ + } + + public CodeJson(CodeableConcept theCodeableConcept, String theCodeableConceptId) { + myCodeableConceptText = theCodeableConcept.getText(); + myCodeableConceptId = theCodeableConceptId; + for (Coding theCoding : theCodeableConcept.getCoding()) { + myCoding_code.add(theCoding.getCode()); + myCoding_system.add(theCoding.getSystem()); + myCoding_display.add(theCoding.getDisplay()); + myCoding_code_system_hash.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCoding.getSystem(), theCoding.getCode()))); + } + } + + public String getCodeableConceptId() { + return myCodeableConceptId; + } + + public String getCodeableConceptText() { + return myCodeableConceptText; + } + + public List getCoding_code() { + return myCoding_code; + } + + public List getCoding_code_system_hash() { + return myCoding_code_system_hash; + } + + public List getCoding_display() { + return myCoding_display; + } + + public List getCoding_system() { + return myCoding_system; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java new file mode 100644 index 00000000000..1df4410f054 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IdJson.java @@ -0,0 +1,19 @@ +package ca.uhn.fhir.jpa.search.lastn.json; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class IdJson { + + @JsonProperty(value = "_id", required = true) + private String myId; + + public IdJson(String theId) { + myId = theId; + } + + public String getId() { return myId; } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java new file mode 100644 index 00000000000..dc1880124b4 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/IndexJson.java @@ -0,0 +1,20 @@ +package ca.uhn.fhir.jpa.search.lastn.json; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class IndexJson { + + @JsonProperty(value = "index", required = true) + private IdJson myIndex; + + public IndexJson(IdJson theIndex) { + myIndex = theIndex; + } + + public IdJson getId() { return myIndex; } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java new file mode 100644 index 00000000000..959dba5c5e7 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java @@ -0,0 +1,175 @@ +package ca.uhn.fhir.jpa.search.lastn.json; + +/* + * #%L + * Smile CDR - CDR + * %% + * Copyright (C) 2016 - 2019 Simpatico Intelligent Systems Inc + * %% + * All rights reserved. + * #L% + */ + +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonAutoDetect(creatorVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class ObservationJson { + + @JsonProperty(value = "identifier", required = true) + private String myIdentifier; + + @JsonProperty(value = "subject", required = true) + private String mySubject; + + @JsonProperty(value = "categoryconcepttext", required = false) + private List myCategory_concept_text = new ArrayList<>(); + + @JsonProperty(value = "categoryconceptcodingcode", required = false) + private List> myCategory_coding_code = new ArrayList<>(); + + @JsonProperty(value = "categoryconceptcodingcode_system_hash", required = false) + private List> myCategory_coding_code_system_hash = new ArrayList<>(); + + @JsonProperty(value = "categoryconceptcodingdisplay", required = false) + private List> myCategory_coding_display = new ArrayList<>(); + + @JsonProperty(value = "categoryconceptcodingsystem", required = false) + private List> myCategory_coding_system = new ArrayList<>(); + + @JsonProperty(value = "codeconceptid", required = false) + private String myCode_concept_id; + + @JsonProperty(value = "codeconcepttext", required = false) + private String myCode_concept_text; + + @JsonProperty(value = "codeconceptcodingcode", required = false) + private List myCode_coding_code = new ArrayList<>(); + + @JsonProperty(value = "codeconceptcodingcode_system_hash", required = false) + private List myCode_coding_code_system_hash = new ArrayList<>(); + + @JsonProperty(value = "codeconceptcodingdisplay", required = false) + private List myCode_coding_display = new ArrayList<>(); + + @JsonProperty(value = "codeconceptcodingsystem", required = false) + private List myCode_coding_system = new ArrayList<>(); + + @JsonProperty(value = "effectivedtm", required = true) + private Date myEffectiveDtm; + + public ObservationJson() {} + + public void setIdentifier(String theIdentifier) { + myIdentifier = theIdentifier; + } + + public void setSubject(String theSubject) { + mySubject = theSubject; + } + + public void setCategories(List theCategories) { + for (CodeableConcept theConcept : theCategories) { + myCategory_concept_text.add(theConcept.getText()); + List coding_code_system_hashes = new ArrayList<>(); + List coding_codes = new ArrayList<>(); + List coding_displays = new ArrayList<>(); + List coding_systems = new ArrayList<>(); + for (Coding theCategoryCoding : theConcept.getCoding()) { + coding_code_system_hashes.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCategoryCoding.getSystem(), theCategoryCoding.getCode()))); + coding_codes.add(theCategoryCoding.getCode()); + coding_displays.add(theCategoryCoding.getDisplay()); + coding_systems.add(theCategoryCoding.getSystem()); + } + myCategory_coding_code_system_hash.add(coding_code_system_hashes); + myCategory_coding_code.add(coding_codes); + myCategory_coding_display.add(coding_displays); + myCategory_coding_system.add(coding_systems); + } + } + + public List getCategory_concept_text() { + return myCategory_concept_text; + } + + public List> getCategory_coding_code_system_hash() { + return myCategory_coding_code_system_hash; + } + + public List> getCategory_coding_code() { + return myCategory_coding_code; + } + + public List> getCategory_coding_display() { + return myCategory_coding_display; + } + + public List> getCategory_coding_system() { + return myCategory_coding_system; + } + + public void setCode(CodeableConcept theCode) { + myCode_concept_text = theCode.getText(); + for(Coding theCodeCoding : theCode.getCoding()) { + myCode_coding_code_system_hash.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCodeCoding.getSystem(), theCodeCoding.getCode()))); + myCode_coding_code.add(theCodeCoding.getCode()); + myCode_coding_display.add(theCodeCoding.getDisplay()); + myCode_coding_system.add(theCodeCoding.getSystem()); + } + + } + + public String getCode_concept_text() { + return myCode_concept_text; + } + + public List getCode_coding_code_system_hash() { + return myCode_coding_code_system_hash; + } + + public List getCode_coding_code() { + return myCode_coding_code; + } + + public List getCode_coding_display() { + return myCode_coding_display; + } + + public List getCode_coding_system() { + return myCode_coding_system; + } + + public void setCode_concept_id(String theCodeId) { + myCode_concept_id = theCodeId; + } + + public String getCode_concept_id() { + return myCode_concept_id; + } + + public void setEffectiveDtm(Date theEffectiveDtm) { + myEffectiveDtm = theEffectiveDtm; + } + + public Date getEffectiveDtm() { + return myEffectiveDtm; + } + + public String getSubject() { + return mySubject; + } + + public String getIdentifier() { + return myIdentifier; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java new file mode 100644 index 00000000000..9995b1d3ec0 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/CodeSystemHash.java @@ -0,0 +1,33 @@ +package ca.uhn.fhir.jpa.search.lastn.util; + +import ca.uhn.fhir.util.UrlUtil; +import com.google.common.base.Charsets; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; + +public class CodeSystemHash { + private static final HashFunction HASH_FUNCTION = Hashing.murmur3_128(0); + private static final byte[] DELIMITER_BYTES = "|".getBytes(Charsets.UTF_8); + + static public long hashCodeSystem( String system, String code ) { + Hasher hasher = HASH_FUNCTION.newHasher(); + addStringToHasher(hasher, system); + addStringToHasher(hasher, code); + + HashCode hashCode = hasher.hash(); + return hashCode.asLong(); + } + + static private void addStringToHasher(Hasher hasher, String next) { + if (next == null) { + hasher.putByte((byte) 0); + } else { + next = UrlUtil.escapeUrlParam(next); + byte[] bytes = next.getBytes(Charsets.UTF_8); + hasher.putBytes(bytes); + } + hasher.putBytes(DELIMITER_BYTES); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java new file mode 100644 index 00000000000..20381a5e2fe --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/util/SimpleStopWatch.java @@ -0,0 +1,18 @@ +package ca.uhn.fhir.jpa.search.lastn.util; + +public class SimpleStopWatch { + private long myStarted = System.currentTimeMillis(); + + public SimpleStopWatch() { + + } + + public long getElapsedTime() { + return System.currentTimeMillis() - myStarted; + } + + public void restart() { + myStarted = System.currentTimeMillis(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java new file mode 100644 index 00000000000..32a1324bea5 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/IntegratedObservationIndexedSearchParamLastNTest.java @@ -0,0 +1,214 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.jpa.dao.lastn.config.TestIntegratedObservationIndexSearchConfig; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.hl7.fhir.r4.model.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestIntegratedObservationIndexSearchConfig.class }) +public class IntegratedObservationIndexedSearchParamLastNTest { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; + + @Autowired + ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; + + @Autowired + private ElasticsearchSvcImpl elasticsearchSvc; + + final String RESOURCEPID = "123"; + final String SUBJECTID = "4567"; + final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; + final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; + + final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; + final String CODEFIRSTCODINGCODE = "test-code"; + + @Before + public void before() throws IOException { + + myResourceIndexedObservationLastNDao.deleteAll(); + myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); + + } + + @After + public void after() { + + myResourceIndexedObservationLastNDao.deleteAll(); + myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); + + } + + @Test + public void testIndexObservationSingle() throws IOException { + + Observation myObservation = new Observation(); + String resourcePID = "123"; + myObservation.setId(resourcePID); + Reference subjectId = new Reference("4567"); + myObservation.setSubject(subjectId); + Date effectiveDtm = new Date(); + myObservation.setEffective(new DateTimeType(effectiveDtm)); + + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for for second category"); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText("Test Codeable Concept Field for third category"); + category3.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test-vitals-panel display")); + category3.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test-alt-vitals-panel display")); + category3.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test-2nd-alt-vitals-panel display")); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + myObservation.setCategory(categoryConcepts); + + // Create CodeableConcept for Code with three codings. + String observationCodeText = "Test Codeable Concept Field for Code"; + CodeableConcept codeableConceptField = new CodeableConcept().setText(observationCodeText); + codeableConceptField.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test-code display")); + codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); + codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); + myObservation.setCode(codeableConceptField); + + myObservationLastNIndexPersistSvc.indexObservation(myObservation); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); + searchParameterMap.add("code", codeParam); + + // execute Observation ID search - Terms Aggregation + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(1, observationIdsOnly.size()); + ObservationJson observationIdOnly = observationIdsOnly.get(0); + assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + // execute Observation ID search - Composite Aggregation +// searchRequestIdsOnly = elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); +// responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); +// observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + +// assertEquals(1, observationIdsOnly.size()); +// observationIdOnly = observationIdsOnly.get(0); +// assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + } + + +// @Test + public void testIndexObservationMultiple() { + + // Create two CodeableConcept values each for a Code with three codings. + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + + // Create two CodeableConcept entities for category, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subjectId = String.valueOf(patientCount); + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + + Observation observation = new Observation(); + observation.setId(String.valueOf(entryCount + patientCount*10)); + Reference subject = new Reference(subjectId); + observation.setSubject(subject); + + if (entryCount%2 == 1) { + observation.setCategory(categoryConcepts1); + observation.setCode(codeableConceptField1); + } else { + observation.setCategory(categoryConcepts2); + observation.setCode(codeableConceptField2); + } + + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observation.setEffective(new DateTimeType(effectiveDtm)); + + myObservationLastNIndexPersistSvc.indexObservation(observation); + + } + + } + + assertEquals(100, myResourceIndexedObservationLastNDao.count()); + assertEquals(2, myCodeableConceptIndexedSearchParamNormalizedDao.count()); + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java new file mode 100644 index 00000000000..e044a9b9e57 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/PersistObservationIndexedSearchParamLastNTest.java @@ -0,0 +1,185 @@ +package ca.uhn.fhir.jpa.dao.lastn; + +import ca.uhn.fhir.jpa.dao.lastn.config.TestObservationIndexSearchConfig; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao; +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedCodeCodeableConceptEntity; +import ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity; +import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao; +import org.hl7.fhir.r4.model.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.*; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestObservationIndexSearchConfig.class }) +public class PersistObservationIndexedSearchParamLastNTest { + + @Autowired + IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao; + + @Autowired + IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao; + + @Autowired + ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc; + + @Before + public void before() { + + myResourceIndexedObservationLastNDao.deleteAll(); + myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); + + } + + @After + public void after() { + + myResourceIndexedObservationLastNDao.deleteAll(); + myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll(); + + } + + @Test + public void testIndexObservationSingle() { + + Observation myObservation = new Observation(); + String resourcePID = "123"; + myObservation.setId(resourcePID); + Reference subjectId = new Reference("4567"); + myObservation.setSubject(subjectId); + Date effectiveDtm = new Date(); + myObservation.setEffective(new DateTimeType(effectiveDtm)); + + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for for second category"); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText("Test Codeable Concept Field for third category"); + category3.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vitals-panel", "test-vitals-panel display")); + category3.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals-panel", "test-alt-vitals-panel display")); + category3.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals-panel", "test-2nd-alt-vitals-panel display")); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + myObservation.setCategory(categoryConcepts); + + // Create CodeableConcept for Code with three codings. + String observationCodeText = "Test Codeable Concept Field for Code"; + CodeableConcept codeableConceptField = new CodeableConcept().setText(observationCodeText); + codeableConceptField.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code", "test-code display")); + codeableConceptField.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code", "test-alt-code display")); + codeableConceptField.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code", "test-second-alt-code display")); + myObservation.setCode(codeableConceptField); + + myObservationLastNIndexPersistSvc.indexObservation(myObservation); + + List persistedObservationEntities = myResourceIndexedObservationLastNDao.findAll(); + assertEquals(1, persistedObservationEntities.size()); + ObservationIndexedSearchParamLastNEntity persistedObservationEntity = persistedObservationEntities.get(0); + assertEquals("Patient/"+subjectId.getReference(), persistedObservationEntity.getSubject()); + assertEquals(resourcePID, persistedObservationEntity.getIdentifier()); + assertEquals(effectiveDtm, persistedObservationEntity.getEffectiveDtm()); + + String observationCodeNormalizedId = persistedObservationEntity.getCodeNormalizedId(); + + List persistedObservationCodes = myCodeableConceptIndexedSearchParamNormalizedDao.findAll(); + assertEquals(1, persistedObservationCodes.size()); + ObservationIndexedCodeCodeableConceptEntity persistedObservationCode = persistedObservationCodes.get(0); + assertEquals(observationCodeNormalizedId, persistedObservationCode.getCodeableConceptId()); + assertEquals(observationCodeText, persistedObservationCode.getCodeableConceptText()); + + } + + @Test + public void testIndexObservationMultiple() { + + // Create two CodeableConcept values each for a Code with three codings. + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + + // Create two CodeableConcept entities for category, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subjectId = String.valueOf(patientCount); + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + + Observation observation = new Observation(); + observation.setId(String.valueOf(entryCount + patientCount*10)); + Reference subject = new Reference(subjectId); + observation.setSubject(subject); + + if (entryCount%2 == 1) { + observation.setCategory(categoryConcepts1); + observation.setCode(codeableConceptField1); + } else { + observation.setCategory(categoryConcepts2); + observation.setCode(codeableConceptField2); + } + + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observation.setEffective(new DateTimeType(effectiveDtm)); + + myObservationLastNIndexPersistSvc.indexObservation(observation); + + } + + } + + assertEquals(100, myResourceIndexedObservationLastNDao.count()); + assertEquals(2, myCodeableConceptIndexedSearchParamNormalizedDao.count()); + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java new file mode 100644 index 00000000000..0aec82de5f1 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestIntegratedObservationIndexSearchConfig.java @@ -0,0 +1,23 @@ +package ca.uhn.fhir.jpa.dao.lastn.config; + +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import java.io.IOException; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory", + basePackages = {"ca.uhn.fhir.jpa.dao"}) +@EnableTransactionManagement +public class TestIntegratedObservationIndexSearchConfig extends TestObservationIndexSearchConfig { + + @Bean() + public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java new file mode 100644 index 00000000000..ed8dc143e26 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/lastn/config/TestObservationIndexSearchConfig.java @@ -0,0 +1,88 @@ +package ca.uhn.fhir.jpa.dao.lastn.config; + +import ca.uhn.fhir.context.ConfigurationException; +import ca.uhn.fhir.jpa.config.TestR4Config; +import ca.uhn.fhir.jpa.config.r4.BaseR4Config; +import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory; +import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; +import org.hibernate.cfg.AvailableSettings; +import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus; +import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; +import pl.allegro.tech.embeddedelasticsearch.PopularProperties; + +import java.io.IOException; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory", + basePackages = {"ca.uhn.fhir.jpa.dao"}) +@EnableTransactionManagement +public class TestObservationIndexSearchConfig extends TestR4Config { + + final String elasticsearchHost = "127.0.0.1"; + final String elasticsearchUserId = ""; + final String elasticsearchPassword = ""; + + private static final String ELASTIC_VERSION = "6.5.4"; + + @Override + public Properties jpaProperties() { + + Properties extraProperties = new Properties(); + extraProperties.put("hibernate.dialect", org.hibernate.dialect.H2Dialect.class); + extraProperties.put("hibernate.format_sql", "false"); + extraProperties.put("hibernate.show_sql", "false"); + extraProperties.put(AvailableSettings.HBM2DDL_AUTO, "update"); + extraProperties.put("hibernate.jdbc.batch_size", "5000"); + extraProperties.put("hibernate.cache.use_query_cache", "false"); + extraProperties.put("hibernate.cache.use_second_level_cache", "false"); + extraProperties.put("hibernate.cache.use_structured_entries", "false"); + extraProperties.put("hibernate.cache.use_minimal_puts", "false"); + extraProperties.put("hibernate.search.model_mapping", LuceneSearchMappingFactory.class.getName()); + extraProperties.put("hibernate.search.lucene_version", "LUCENE_CURRENT"); + extraProperties.put("hibernate.search.default.worker.execution", "sync"); + + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); + new ElasticsearchHibernatePropertiesBuilder() + .setDebugRefreshAfterWrite(true) + .setDebugPrettyPrintJsonLog(true) + .setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE) + .setIndexManagementWaitTimeoutMillis(10000) + .setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW) + .setRestUrl("http://" + elasticsearchHost + ":" + elasticsearchPort) + .setUsername(elasticsearchUserId) + .setPassword(elasticsearchPassword) + .apply(extraProperties); + + extraProperties.setProperty("hibernate.search.default.elasticsearch.refresh_after_write", "true"); + return extraProperties; + } + + @Bean + public EmbeddedElastic embeddedElasticSearch() { + EmbeddedElastic embeddedElastic; + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } + + return embeddedElastic; + } + + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java new file mode 100644 index 00000000000..2e4c751093b --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchPerformanceTests.java @@ -0,0 +1,195 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.SimpleStopWatch; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import com.fasterxml.jackson.databind.ObjectMapper; +/* +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.shadehapi.elasticsearch.index.query.QueryBuilders; +import org.shadehapi.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.shadehapi.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; +import org.shadehapi.elasticsearch.search.SearchHit; +import org.shadehapi.elasticsearch.search.aggregations.AggregationBuilders; +import org.shadehapi.elasticsearch.search.aggregations.Aggregations; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.shadehapi.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.shadehapi.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits; +import org.shadehapi.elasticsearch.search.aggregations.support.ValueType; +import org.shadehapi.elasticsearch.search.builder.SearchSourceBuilder; +import org.shadehapi.elasticsearch.search.sort.SortOrder; + */ +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +@Ignore +public class ElasticsearchPerformanceTests { + + private ElasticsearchSvcImpl elasticsearchSvc = new ElasticsearchSvcImpl("localhost", 9301, "elastic", "changeme"); + private List patientIds = new ArrayList<>(); + + @Before + public void before() throws IOException { +/* SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + RandomScoreFunctionBuilder randomScoreFunctionBuilder = new RandomScoreFunctionBuilder(); + Date now = new Date(); + randomScoreFunctionBuilder.seed(now.getTime()); + FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(randomScoreFunctionBuilder); + searchSourceBuilder.query(functionScoreQueryBuilder); + searchSourceBuilder.size(0); + + // Aggregation by order codes + TermsAggregationBuilder observationCodeValuesBuilder = new TermsAggregationBuilder("group_by_patient", ValueType.STRING).field("subject"); + observationCodeValuesBuilder.size(1000); + // Top Hits Aggregation + String[] topHitsInclude = {"subject"}; + observationCodeValuesBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + .sort("effectivedtm", SortOrder.DESC) + .fetchSource(topHitsInclude, null).size(1)); + + searchSourceBuilder.aggregation(observationCodeValuesBuilder); + searchRequest.source(searchSourceBuilder); + + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + patientIds = buildResults(response); +*/ + } + +// @Test + public void testObservationCodesQueryPerformance() throws IOException { +// SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(10000); + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(int i=0; i<1000; i++) { + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List codes = elasticsearchSvc.buildCodeResult(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); +// assertEquals(1000L, codes.size()); + } + System.out.println("Codes query Average elapsed time = " + totalElapsedTimes/1000 + " ms"); + System.out.println("Codes query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/1000 + " ms"); + } + + @Test + public void testObservationsQueryPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List observations = elasticsearchSvc.buildObservationTermsResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); +// assertEquals(25, observations.size()); + } + System.out.println("Observations query Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("Observations query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + +// @Test + public void testSingleObservationsQuery() throws IOException { + long totalElapsedTimes = 0L; + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "5b2091a5-a50e-447b-aff4-c34b69eb43d5"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 100); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List observations = elasticsearchSvc.buildObservationCompositeResults(response); +// assertEquals(25, observations.size()); + System.out.println("Average elapsed time = " + totalElapsedTimes/1000 + " ms"); + } + +// @Test + public void testLastNObservationsQueryTermsPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 5); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// elasticsearchSvc.buildObservationTermsResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); + } + System.out.println("LastN Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("LastN Average elapsed search and Build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + +// @Test + public void testLastNObservationsQueryCompPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 5); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// elasticsearchSvc.buildObservationCompositeResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); + } + System.out.println("LastN Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("LastN Average elapsed search and Build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + +/* private List buildResults(SearchResponse theSearchResponse) throws IOException { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedTerms aggregatedObservationCodes = responseAggregations.get("group_by_patient"); + aggregatedObservationCodes.getBuckets(); + List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); + ObjectMapper objectMapper = new ObjectMapper(); + List identifiers = new ArrayList<>(10000); + for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { + Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + SearchHit[] topHits = parsedTopHits.getHits().getHits(); + for (SearchHit topHit : topHits) { + String sources = topHit.getSourceAsString(); + ObservationJson code = objectMapper.readValue(sources,ObservationJson.class); + identifiers.add(code.getSubject().substring(8)); + } + } + return identifiers; + } +*/ + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java new file mode 100644 index 00000000000..ea9308d1896 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchV5PerformanceTests.java @@ -0,0 +1,153 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.SimpleStopWatch; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.tophits.ParsedTopHits; +import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +@Ignore +public class ElasticsearchV5PerformanceTests { + + private ElasticsearchV5SvcImpl elasticsearchSvc = new ElasticsearchV5SvcImpl("localhost", 9301, "elastic", "changeme"); + private List patientIds = new ArrayList<>(); + + @Before + public void before() throws IOException { + SearchRequest searchRequest = new SearchRequest(IndexConstants.OBSERVATION_INDEX); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + RandomScoreFunctionBuilder randomScoreFunctionBuilder = new RandomScoreFunctionBuilder(); + Date now = new Date(); + randomScoreFunctionBuilder.seed(now.getTime()); + FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(randomScoreFunctionBuilder); + searchSourceBuilder.query(functionScoreQueryBuilder); + searchSourceBuilder.size(0); + + // Aggregation by order codes + TermsAggregationBuilder observationCodeValuesBuilder = new TermsAggregationBuilder("group_by_patient", ValueType.STRING).field("subject"); + observationCodeValuesBuilder.size(1000); + // Top Hits Aggregation + String[] topHitsInclude = {"subject"}; + observationCodeValuesBuilder.subAggregation(AggregationBuilders.topHits("most_recent_effective") + .sort("effectivedtm", SortOrder.DESC) + .fetchSource(topHitsInclude, null).size(1)); + + searchSourceBuilder.aggregation(observationCodeValuesBuilder); + searchRequest.source(searchSourceBuilder); + +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); +// patientIds = buildResults(response); + + } + + @Test + public void testObservationCodesQueryPerformance() throws IOException { +// SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(10000); + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(int i=0; i<1000; i++) { + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List codes = elasticsearchSvc.buildCodeResult(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); +// assertEquals(1000L, codes.size()); + } + System.out.println("Codes query Average elapsed time = " + totalElapsedTimes/1000 + " ms"); + System.out.println("Codes query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/1000 + " ms"); + } + + @Test + public void testObservationsQueryPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// List observations = elasticsearchSvc.buildObservationTermsResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); +// assertEquals(25, observations.size()); + } + System.out.println("Observations query Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("Observations query Average elapsed search and build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + + @Test + public void testLastNObservationsQueryTermsPerformance() throws IOException { + long totalElapsedTimes = 0L; + long totalElapsedSearchAndBuildTimes = 0L; + for(String patientId : patientIds) { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/category-code", "test-category-code"); + searchParameterMap.add("category", categoryParam); + +// SearchRequest searchRequest = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 5); + SimpleStopWatch stopWatch = new SimpleStopWatch(); +// SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + totalElapsedTimes += stopWatch.getElapsedTime(); +// elasticsearchSvc.buildObservationTermsResults(response); + totalElapsedSearchAndBuildTimes += stopWatch.getElapsedTime(); + } + System.out.println("LastN Average elapsed time = " + totalElapsedTimes/(patientIds.size()) + " ms"); + System.out.println("LastN Average elapsed search and Build time = " + totalElapsedSearchAndBuildTimes/(patientIds.size()) + " ms"); + } + + private List buildResults(SearchResponse theSearchResponse) throws IOException { + Aggregations responseAggregations = theSearchResponse.getAggregations(); + ParsedTerms aggregatedObservationCodes = responseAggregations.get("group_by_patient"); + aggregatedObservationCodes.getBuckets(); + List observationCodeBuckets = aggregatedObservationCodes.getBuckets(); + ObjectMapper objectMapper = new ObjectMapper(); + List identifiers = new ArrayList<>(10000); + for (Terms.Bucket observationCodeBucket : observationCodeBuckets) { + Aggregations topHitObservationCodes = observationCodeBucket.getAggregations(); + ParsedTopHits parsedTopHits = topHitObservationCodes.get("most_recent_effective"); + SearchHit[] topHits = parsedTopHits.getHits().getHits(); + for (SearchHit topHit : topHits) { + String sources = topHit.getSourceAsString(); + ObservationJson code = objectMapper.readValue(sources,ObservationJson.class); + identifiers.add(code.getSubject().substring(8)); + } + } + return identifiers; + } + + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java new file mode 100644 index 00000000000..b893a73c1a0 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsTest.java @@ -0,0 +1,323 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.param.TokenParamModifier; +import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchConfig; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.junit.*; +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.*; + +import static ca.uhn.fhir.jpa.search.lastn.IndexConstants.*; +import static org.junit.Assert.*; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestElasticsearchConfig.class } ) +public class LastNElasticsearchSvcMultipleObservationsTest { + + @Autowired + private ElasticsearchSvcImpl elasticsearchSvc; + + private static ObjectMapper ourMapperNonPrettyPrint; + + private Map>> createdPatientObservationMap = new HashMap<>(); + + + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + @Before + public void before() throws IOException { + createMultiplePatientsAndObservations(); + } + + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(CODE_INDEX); + } + + @Test + public void testLastNNoCriteriaQuery() throws IOException { + + // execute Observation ID search (Terms Aggregation) last 3 observations for each patient + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, null, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + validateQueryResponse(observationIdsOnly); + + // execute Observation ID search (Terms Aggregation) last 3 observations for each patient + searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, null, 3); + responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + + validateQueryResponse(observationIdsOnly); + + // Retrieve all Observation codes + SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + List codes = elasticsearchSvc.buildCodeResult(response); + assertEquals(2, codes.size()); + + } + + private void validateQueryResponse(List observationIdsOnly) { + assertEquals(60, observationIdsOnly.size()); + + // Observation documents should be grouped by subject, then by observation code, and then sorted by effective date/time + // within each observation code. Verify the grouping by creating a nested Map. + Map>> queriedPatientObservationMap = new HashMap<>(); + ObservationJson previousObservationJson = null; + for (ObservationJson observationJson : observationIdsOnly) { + assertNotNull(observationJson.getIdentifier()); + assertNotNull(observationJson.getSubject()); + assertNotNull(observationJson.getCode_concept_id()); + assertNotNull(observationJson.getEffectiveDtm()); + if (previousObservationJson == null) { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } else if (observationJson.getSubject().equals(previousObservationJson.getSubject())) { + if (observationJson.getCode_concept_id().equals(previousObservationJson.getCode_concept_id())) { + queriedPatientObservationMap.get(observationJson.getSubject()).get(observationJson.getCode_concept_id()). + add(observationJson.getEffectiveDtm()); + } else { + Map> codeObservationDateMap = queriedPatientObservationMap.get(observationJson.getSubject()); + // Ensure that code concept was not already retrieved out of order for this subject/patient. + assertFalse(codeObservationDateMap.containsKey(observationJson.getCode_concept_id())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + codeObservationDateMap.put(observationJson.getCode_concept_id(),observationDates); + } + } else { + // Ensure that subject/patient was not already retrieved out of order + assertFalse(queriedPatientObservationMap.containsKey(observationJson.getSubject())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } + previousObservationJson = observationJson; + } + + // Finally check that only the most recent effective date/time values were returned and in the correct order. + for(String subjectId : queriedPatientObservationMap.keySet()) { + Map> queriedObservationCodeMap = queriedPatientObservationMap.get(subjectId); + Map> createdObservationCodeMap = createdPatientObservationMap.get(subjectId); + for(String observationCode : queriedObservationCodeMap.keySet()) { + List queriedObservationDates = queriedObservationCodeMap.get(observationCode); + List createdObservationDates = createdObservationCodeMap.get(observationCode); + for (int dateIdx=0; dateIdx observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(10, observationIdsOnly.size()); + + } + + @Test + public void testLastNCodeCodeOnlyCategoryCodeOnly() throws IOException { + // Include subject and patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam ("test-heart-rate"); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("test-code-1"); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(5, observationIdsOnly.size()); + + } + + @Test + public void testLastNCodeSystemOnlyCategorySystemOnly() throws IOException { + // Include subject and patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", null); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(10, observationIdsOnly.size()); + } + + @Test + public void testLastNCodeCodeTextCategoryTextOnly() throws IOException { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("test-heart-rate display"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("test-code-1 display"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(5, observationIdsOnly.size()); + + } + + private void createMultiplePatientsAndObservations() throws IOException { + // Create CodeableConcepts for two Codes, each with three codings. + String codeableConceptId1 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1); + String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1); + + String codeableConceptId2 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2); + String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2); + + // Create CodeableConcepts for two categories, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subject = "Patient/"+patientCount; + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + + ObservationJson observationJson = new ObservationJson(); + String identifier = String.valueOf((entryCount + patientCount*10)); + observationJson.setIdentifier(identifier); + observationJson.setSubject(subject); + + if (entryCount%2 == 1) { + observationJson.setCategories(categoryConcepts1); + observationJson.setCode(codeableConceptField1); + observationJson.setCode_concept_id(codeableConceptId1); + assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId1, codeJson1Document, CODE_DOCUMENT_TYPE)); + } else { + observationJson.setCategories(categoryConcepts2); + observationJson.setCode(codeableConceptField2); + observationJson.setCode_concept_id(codeableConceptId2); + assertTrue(elasticsearchSvc.performIndex(CODE_INDEX, codeableConceptId2, codeJson2Document, CODE_DOCUMENT_TYPE)); + } + + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observationJson.setEffectiveDtm(effectiveDtm); + + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationJson); + assertTrue(elasticsearchSvc.performIndex(OBSERVATION_INDEX, identifier,observationDocument, OBSERVATION_DOCUMENT_TYPE)); + + if (createdPatientObservationMap.containsKey(subject)) { + Map> observationCodeMap = createdPatientObservationMap.get(subject); + if (observationCodeMap.containsKey(observationJson.getCode_concept_id())) { + List observationDates = observationCodeMap.get(observationJson.getCode_concept_id()); + // Want dates to be sorted in descending order + observationDates.add(0, effectiveDtm); + // Only keep the three most recent dates for later check. + if(observationDates.size() > 3) { + observationDates.remove(3); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + observationCodeMap.put(observationJson.getCode_concept_id(), observationDates); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(), observationDates); + createdPatientObservationMap.put(subject, codeObservationMap); + } + } + } + + try { + Thread.sleep(2000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java new file mode 100644 index 00000000000..167d88dba80 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationTest.java @@ -0,0 +1,357 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchConfig; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.shadehapi.elasticsearch.action.search.SearchRequest; +import org.shadehapi.elasticsearch.action.search.SearchResponse; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.junit.*; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestElasticsearchConfig.class } ) +public class LastNElasticsearchSvcSingleObservationTest { + + @Autowired + ElasticsearchSvcImpl elasticsearchSvc; + + static ObjectMapper ourMapperNonPrettyPrint; + + final String RESOURCEPID = "123"; + final String SUBJECTID = "4567"; + final String SUBJECTTYPEANDID = "Patient/4567"; + final Date EFFECTIVEDTM = new Date(); + final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category"; + final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; + final String CATEGORYSECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-category"; + final String CATEGORYTHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-category"; + final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; + final String FIRSTCATEGORYFIRSTCODINGDISPLAY = "test-heart-rate display"; + final String FIRSTCATEGORYSECONDCODINGCODE = "test-alt-heart-rate"; + final String FIRSTCATEGORYSECONDCODINGDISPLAY = "test-alt-heart-rate display"; + final String FIRSTCATEGORYTHIRDCODINGCODE = "test-2nd-alt-heart-rate"; + final String FIRSTCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-heart-rate display"; + final String SECONDCATEGORYTEXT = "Test Codeable Concept Field for for second category"; + final String SECONDCATEGORYFIRSTCODINGCODE = "test-vital-signs"; + final String SECONDCATEGORYFIRSTCODINGDISPLAY = "test-vital-signs display"; + final String SECONDCATEGORYSECONDCODINGCODE = "test-alt-vitals"; + final String SECONDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String SECONDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals"; + final String SECONDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals display"; + final String THIRDCATEGORYTEXT = "Test Codeable Concept Field for third category"; + final String THIRDCATEGORYFIRSTCODINGCODE = "test-vital-panel"; + final String THIRDCATEGORYFIRSTCODINGDISPLAY = "test-vitals-panel display"; + final String THIRDCATEGORYSECONDCODINGCODE = "test-alt-vitals-panel"; + final String THIRDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String THIRDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals-panel"; + final String THIRDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals-panel display"; + + final String OBSERVATIONSINGLECODEID = UUID.randomUUID().toString(); + final String OBSERVATIONCODETEXT = "Test Codeable Concept Field for Code"; + final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; + final String CODEFIRSTCODINGCODE = "test-code"; + final String CODEFIRSTCODINGDISPLAY = "test-code display"; + final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; + final String CODESECONDCODINGCODE = "test-alt-code"; + final String CODESECONDCODINGDISPLAY = "test-alt-code display"; + final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; + final String CODETHIRDCODINGCODE = "test-second-alt-code"; + final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; + + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + + } + + // @Before + public void before() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } + + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } + + @Test + public void testSingleObservationQuery() throws IOException { + + createSingleObservation(); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); + searchParameterMap.add("code", codeParam); + + // execute Observation ID search - Terms Aggregation + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(1, observationIdsOnly.size()); + ObservationJson observationIdOnly = observationIdsOnly.get(0); + assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + // execute Observation ID search - Composite Aggregation + searchRequestIdsOnly = elasticsearchSvc.buildObservationCompositeSearchRequest(1000, searchParameterMap, 3); + responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + observationIdsOnly = elasticsearchSvc.buildObservationCompositeResults(responseIds); + + assertEquals(1, observationIdsOnly.size()); + observationIdOnly = observationIdsOnly.get(0); + assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + // execute full Observation search - Terms Aggregation + SearchRequest searchRequestAllFields = elasticsearchSvc.buildObservationAllFieldsTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequestAllFields); + List observations = elasticsearchSvc.buildObservationTermsResults(response); + + validateFullObservationSearch(observations); + + // execute full Observation search - Composite Aggregation + searchRequestAllFields= elasticsearchSvc.buildObservationAllFieldsCompositeSearchRequest(1000, searchParameterMap, 3); + response = elasticsearchSvc.executeSearchRequest(searchRequestAllFields); + observations = elasticsearchSvc.buildObservationCompositeResults(response); + + validateFullObservationSearch(observations); + } + + private void validateFullObservationSearch(List observations) throws IOException { + + assertEquals(1, observations.size()); + ObservationJson observation = observations.get(0); + assertEquals(RESOURCEPID, observation.getIdentifier()); + + assertEquals(SUBJECTTYPEANDID, observation.getSubject()); + assertEquals(RESOURCEPID, observation.getIdentifier()); + assertEquals(EFFECTIVEDTM, observation.getEffectiveDtm()); + assertEquals(OBSERVATIONSINGLECODEID, observation.getCode_concept_id()); + + List category_concept_text_values = observation.getCategory_concept_text(); + assertEquals(3,category_concept_text_values.size()); + assertEquals(FIRSTCATEGORYTEXT, category_concept_text_values.get(0)); + assertEquals(SECONDCATEGORYTEXT, category_concept_text_values.get(1)); + assertEquals(THIRDCATEGORYTEXT, category_concept_text_values.get(2)); + + List> category_codings_systems = observation.getCategory_coding_system(); + assertEquals(3,category_codings_systems.size()); + List category_coding_systems = category_codings_systems.get(0); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_coding_systems = category_codings_systems.get(1); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_coding_systems = category_codings_systems.get(2); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + + List> category_codings_codes = observation.getCategory_coding_code(); + assertEquals(3, category_codings_codes.size()); + List category_coding_codes = category_codings_codes.get(0); + assertEquals(3, category_coding_codes.size()); + assertEquals(FIRSTCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(1); + assertEquals(3, category_coding_codes.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(2); + assertEquals(3, category_coding_codes.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + + List> category_codings_displays = observation.getCategory_coding_display(); + assertEquals(3, category_codings_displays.size()); + List category_coding_displays = category_codings_displays.get(0); + assertEquals(FIRSTCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(1); + assertEquals(3, category_coding_displays.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(2); + assertEquals(3, category_coding_displays.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + + List> category_codings_code_system_hashes = observation.getCategory_coding_code_system_hash(); + assertEquals(3, category_codings_code_system_hashes.size()); + List category_coding_code_system_hashes = category_codings_code_system_hashes.get(0); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(1); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(2); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + + String code_concept_text_values = observation.getCode_concept_text(); + assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); + + List code_coding_systems = observation.getCode_coding_system(); + assertEquals(3,code_coding_systems.size()); + assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems.get(0)); + assertEquals(CODESECONDCODINGSYSTEM, code_coding_systems.get(1)); + assertEquals(CODETHIRDCODINGSYSTEM, code_coding_systems.get(2)); + + List code_coding_codes = observation.getCode_coding_code(); + assertEquals(3, code_coding_codes.size()); + assertEquals(CODEFIRSTCODINGCODE, code_coding_codes.get(0)); + assertEquals(CODESECONDCODINGCODE, code_coding_codes.get(1)); + assertEquals(CODETHIRDCODINGCODE, code_coding_codes.get(2)); + + List code_coding_display = observation.getCode_coding_display(); + assertEquals(3, code_coding_display.size()); + assertEquals(CODEFIRSTCODINGDISPLAY, code_coding_display.get(0)); + assertEquals(CODESECONDCODINGDISPLAY, code_coding_display.get(1)); + assertEquals(CODETHIRDCODINGDISPLAY, code_coding_display.get(2)); + + List code_coding_code_system_hash = observation.getCode_coding_code_system_hash(); + assertEquals(3, code_coding_code_system_hash.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), code_coding_code_system_hash.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), code_coding_code_system_hash.get(2)); + + // Retrieve all Observation codes + SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + List codes = elasticsearchSvc.buildCodeResult(response); + assertEquals(1, codes.size()); + CodeJson persistedObservationCode = codes.get(0); + + String persistedCodeConceptID = persistedObservationCode.getCodeableConceptId(); + assertEquals(OBSERVATIONSINGLECODEID, persistedCodeConceptID); + String persistedCodeConceptText = persistedObservationCode.getCodeableConceptText(); + assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText); + + List persistedCodeCodingSystems = persistedObservationCode.getCoding_system(); + assertEquals(3,persistedCodeCodingSystems.size()); + assertEquals(CODEFIRSTCODINGSYSTEM, persistedCodeCodingSystems.get(0)); + assertEquals(CODESECONDCODINGSYSTEM, persistedCodeCodingSystems.get(1)); + assertEquals(CODETHIRDCODINGSYSTEM, persistedCodeCodingSystems.get(2)); + + List persistedCodeCodingCodes = persistedObservationCode.getCoding_code(); + assertEquals(3, persistedCodeCodingCodes.size()); + assertEquals(CODEFIRSTCODINGCODE, persistedCodeCodingCodes.get(0)); + assertEquals(CODESECONDCODINGCODE, persistedCodeCodingCodes.get(1)); + assertEquals(CODETHIRDCODINGCODE, persistedCodeCodingCodes.get(2)); + + List persistedCodeCodingDisplays = persistedObservationCode.getCoding_display(); + assertEquals(3, persistedCodeCodingDisplays.size()); + assertEquals(CODEFIRSTCODINGDISPLAY, persistedCodeCodingDisplays.get(0)); + assertEquals(CODESECONDCODINGDISPLAY, persistedCodeCodingDisplays.get(1)); + assertEquals(CODETHIRDCODINGDISPLAY, persistedCodeCodingDisplays.get(2)); + + List persistedCodeCodingCodeSystemHashes = persistedObservationCode.getCoding_code_system_hash(); + assertEquals(3, persistedCodeCodingCodeSystemHashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(2)); + + + } + + private void createSingleObservation() throws IOException { + ObservationJson indexedObservation = new ObservationJson(); + indexedObservation.setIdentifier(RESOURCEPID); + indexedObservation.setSubject(SUBJECTTYPEANDID); + indexedObservation.setEffectiveDtm(EFFECTIVEDTM); + + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText(FIRSTCATEGORYTEXT); + category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY)); + category1.add(new Coding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY)); + category1.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText(SECONDCATEGORYTEXT); + category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY)); + category2.add(new Coding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY)); + category2.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText(THIRDCATEGORYTEXT); + category3.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY)); + category3.add(new Coding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY)); + category3.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + indexedObservation.setCategories(categoryConcepts); + + // Create CodeableConcept for Code with three codings. + indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID); + CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT); + codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY)); + codeableConceptField.addCoding(new Coding(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE, CODESECONDCODINGDISPLAY)); + codeableConceptField.addCoding(new Coding(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE, CODETHIRDCODINGDISPLAY)); + indexedObservation.setCode(codeableConceptField); + + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); + + CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); + String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); + + try { + Thread.sleep(1000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java new file mode 100644 index 00000000000..08f731e0f15 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcMultipleObservationsTest.java @@ -0,0 +1,312 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchV5Config; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.param.TokenParamModifier; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.*; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestElasticsearchV5Config.class } ) +public class LastNElasticsearchV5SvcMultipleObservationsTest { + + @Autowired + private ElasticsearchV5SvcImpl elasticsearchSvc; + + private static ObjectMapper ourMapperNonPrettyPrint; + + private Map>> createdPatientObservationMap = new HashMap<>(); + + + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + } + + @Before + public void before() throws IOException { +// createMultiplePatientsAndObservations(); + } + + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } +/* + @Test + public void testLastNNoCriteriaQuery() throws IOException { + + // execute Observation ID search (Terms Aggregation) last 3 observations for each patient + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, null, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + validateQueryResponse(observationIdsOnly); + + } + + private void validateQueryResponse(List observationIdsOnly) { + assertEquals(60, observationIdsOnly.size()); + + // Observation documents should be grouped by subject, then by observation code, and then sorted by effective date/time + // within each observation code. Verify the grouping by creating a nested Map. + Map>> queriedPatientObservationMap = new HashMap<>(); + ObservationJson previousObservationJson = null; + for (ObservationJson observationJson : observationIdsOnly) { + assertNotNull(observationJson.getIdentifier()); + assertNotNull(observationJson.getSubject()); + assertNotNull(observationJson.getCode_concept_id()); + assertNotNull(observationJson.getEffectiveDtm()); + if (previousObservationJson == null) { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } else if (observationJson.getSubject().equals(previousObservationJson.getSubject())) { + if (observationJson.getCode_concept_id().equals(previousObservationJson.getCode_concept_id())) { + queriedPatientObservationMap.get(observationJson.getSubject()).get(observationJson.getCode_concept_id()). + add(observationJson.getEffectiveDtm()); + } else { + Map> codeObservationDateMap = queriedPatientObservationMap.get(observationJson.getSubject()); + // Ensure that code concept was not already retrieved out of order for this subject/patient. + assertFalse(codeObservationDateMap.containsKey(observationJson.getCode_concept_id())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + codeObservationDateMap.put(observationJson.getCode_concept_id(),observationDates); + } + } else { + // Ensure that subject/patient was not already retrieved out of order + assertFalse(queriedPatientObservationMap.containsKey(observationJson.getSubject())); + ArrayList observationDates = new ArrayList<>(); + observationDates.add(observationJson.getEffectiveDtm()); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(),observationDates); + queriedPatientObservationMap.put(observationJson.getSubject(), codeObservationMap); + } + previousObservationJson = observationJson; + } + + // Finally check that only the most recent effective date/time values were returned and in the correct order. + for(String subjectId : queriedPatientObservationMap.keySet()) { + Map> queriedObservationCodeMap = queriedPatientObservationMap.get(subjectId); + Map> createdObservationCodeMap = createdPatientObservationMap.get(subjectId); + for(String observationCode : queriedObservationCodeMap.keySet()) { + List queriedObservationDates = queriedObservationCodeMap.get(observationCode); + List createdObservationDates = createdObservationCodeMap.get(observationCode); + for (int dateIdx=0; dateIdx observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(10, observationIdsOnly.size()); + + } + + @Test + public void testLastNCodeCodeOnlyCategoryCodeOnly() throws IOException { + // Include subject and patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam ("test-heart-rate"); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("test-code-1"); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(5, observationIdsOnly.size()); + + } + + @Test + public void testLastNCodeSystemOnlyCategorySystemOnly() throws IOException { + // Include subject and patient + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("http://mycodes.org/fhir/observation-category", null); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("http://mycodes.org/fhir/observation-code", null); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(10, observationIdsOnly.size()); + } + + @Test + public void testLastNCodeCodeTextCategoryTextOnly() throws IOException { + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3"); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam("test-heart-rate display"); + categoryParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam("test-code-1 display"); + codeParam.setModifier(TokenParamModifier.TEXT); + searchParameterMap.add("code", codeParam); + + SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 100); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(5, observationIdsOnly.size()); + + } + + private void createMultiplePatientsAndObservations() throws IOException { + // Create CodeableConcepts for two Codes, each with three codings. + String codeableConceptId1 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code"); + codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-1", "test-alt-code-1 display")); + codeableConceptField1.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-1", "test-second-alt-code-1 display")); + CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1); + String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1); + + String codeableConceptId2 = UUID.randomUUID().toString(); + CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code"); + codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://myalternatecodes.org/fhir/observation-code", "test-alt-code-2", "test-alt-code-2 display")); + codeableConceptField2.addCoding(new Coding("http://mysecondaltcodes.org/fhir/observation-code", "test-second-alt-code-2", "test-second-alt-code-2 display")); + CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2); + String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2); + + // Create CodeableConcepts for two categories, each with three codings. + List category1 = new ArrayList<>(); + // Create three codings and first category CodeableConcept + category1.add(new Coding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "test-heart-rate display")); + category1.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "test-alt-heart-rate display")); + category1.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "test-2nd-alt-heart-rate display")); + List categoryConcepts1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText("Test Codeable Concept Field for first category"); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts1.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + category2.add(new Coding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "test-vital-signs display")); + category2.add(new Coding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "test-alt-vitals display")); + category2.add(new Coding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "test-2nd-alt-vitals display")); + List categoryConcepts2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText("Test Codeable Concept Field for second category"); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts2.add(categoryCodeableConcept2); + + for (int patientCount = 0; patientCount < 10 ; patientCount++) { + + String subject = "Patient/"+patientCount; + + for ( int entryCount = 0; entryCount < 10 ; entryCount++ ) { + + ObservationJson observationJson = new ObservationJson(); + String identifier = String.valueOf((entryCount + patientCount*10)); + observationJson.setIdentifier(identifier); + observationJson.setSubject(subject); + + if (entryCount%2 == 1) { + observationJson.setCategories(categoryConcepts1); + observationJson.setCode(codeableConceptField1); + observationJson.setCode_concept_id(codeableConceptId1); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, codeableConceptId1, codeJson1Document, IndexConstants.CODE_DOCUMENT_TYPE)); + } else { + observationJson.setCategories(categoryConcepts2); + observationJson.setCode(codeableConceptField2); + observationJson.setCode_concept_id(codeableConceptId2); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, codeableConceptId2, codeJson2Document, IndexConstants.CODE_DOCUMENT_TYPE)); + } + + Calendar observationDate = new GregorianCalendar(); + observationDate.add(Calendar.HOUR, -10 + entryCount); + Date effectiveDtm = observationDate.getTime(); + observationJson.setEffectiveDtm(effectiveDtm); + + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(observationJson); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, identifier,observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); + + if (createdPatientObservationMap.containsKey(subject)) { + Map> observationCodeMap = createdPatientObservationMap.get(subject); + if (observationCodeMap.containsKey(observationJson.getCode_concept_id())) { + List observationDates = observationCodeMap.get(observationJson.getCode_concept_id()); + // Want dates to be sorted in descending order + observationDates.add(0, effectiveDtm); + // Only keep the three most recent dates for later check. + if(observationDates.size() > 3) { + observationDates.remove(3); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + observationCodeMap.put(observationJson.getCode_concept_id(), observationDates); + } + } else { + ArrayList observationDates = new ArrayList<>(); + observationDates.add(effectiveDtm); + Map> codeObservationMap = new HashMap<>(); + codeObservationMap.put(observationJson.getCode_concept_id(), observationDates); + createdPatientObservationMap.put(subject, codeObservationMap); + } + } + } + + try { + Thread.sleep(2000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } + + } +*/ +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java new file mode 100644 index 00000000000..48b42d31ec5 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchV5SvcSingleObservationTest.java @@ -0,0 +1,346 @@ +package ca.uhn.fhir.jpa.search.lastn; + +import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchV5Config; +import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; +import ca.uhn.fhir.jpa.search.lastn.util.CodeSystemHash; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.TokenParam; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { TestElasticsearchV5Config.class } ) +public class LastNElasticsearchV5SvcSingleObservationTest { + + @Autowired + ElasticsearchV5SvcImpl elasticsearchSvc; + + static ObjectMapper ourMapperNonPrettyPrint; + + final String RESOURCEPID = "123"; + final String SUBJECTID = "4567"; + final String SUBJECTTYPEANDID = "Patient/4567"; + final Date EFFECTIVEDTM = new Date(); + final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category"; + final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category"; + final String CATEGORYSECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-category"; + final String CATEGORYTHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-category"; + final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate"; + final String FIRSTCATEGORYFIRSTCODINGDISPLAY = "test-heart-rate display"; + final String FIRSTCATEGORYSECONDCODINGCODE = "test-alt-heart-rate"; + final String FIRSTCATEGORYSECONDCODINGDISPLAY = "test-alt-heart-rate display"; + final String FIRSTCATEGORYTHIRDCODINGCODE = "test-2nd-alt-heart-rate"; + final String FIRSTCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-heart-rate display"; + final String SECONDCATEGORYTEXT = "Test Codeable Concept Field for for second category"; + final String SECONDCATEGORYFIRSTCODINGCODE = "test-vital-signs"; + final String SECONDCATEGORYFIRSTCODINGDISPLAY = "test-vital-signs display"; + final String SECONDCATEGORYSECONDCODINGCODE = "test-alt-vitals"; + final String SECONDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String SECONDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals"; + final String SECONDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals display"; + final String THIRDCATEGORYTEXT = "Test Codeable Concept Field for third category"; + final String THIRDCATEGORYFIRSTCODINGCODE = "test-vital-panel"; + final String THIRDCATEGORYFIRSTCODINGDISPLAY = "test-vitals-panel display"; + final String THIRDCATEGORYSECONDCODINGCODE = "test-alt-vitals-panel"; + final String THIRDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display"; + final String THIRDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals-panel"; + final String THIRDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals-panel display"; + + final String OBSERVATIONSINGLECODEID = UUID.randomUUID().toString(); + final String OBSERVATIONCODETEXT = "Test Codeable Concept Field for Code"; + final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code"; + final String CODEFIRSTCODINGCODE = "test-code"; + final String CODEFIRSTCODINGDISPLAY = "test-code display"; + final String CODESECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-code"; + final String CODESECONDCODINGCODE = "test-alt-code"; + final String CODESECONDCODINGDISPLAY = "test-alt-code display"; + final String CODETHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-code"; + final String CODETHIRDCODINGCODE = "test-second-alt-code"; + final String CODETHIRDCODINGDISPLAY = "test-second-alt-code display"; + + @BeforeClass + public static void beforeClass() { + ourMapperNonPrettyPrint = new ObjectMapper(); + ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT); + ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + + } + + // @Before + public void before() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } + + @After + public void after() throws IOException { + elasticsearchSvc.deleteAllDocuments(IndexConstants.OBSERVATION_INDEX); + elasticsearchSvc.deleteAllDocuments(IndexConstants.CODE_INDEX); + } + @Test + public void testSingleObservationQuery() throws IOException { + + createSingleObservation(); + + SearchParameterMap searchParameterMap = new SearchParameterMap(); + ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID); + searchParameterMap.add("subject", subjectParam); + TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE); + searchParameterMap.add("category", categoryParam); + TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE); + searchParameterMap.add("code", codeParam); + + // execute Observation ID search - Terms Aggregation +/* SearchRequest searchRequestIdsOnly = elasticsearchSvc.buildObservationTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse responseIds = elasticsearchSvc.executeSearchRequest(searchRequestIdsOnly); + List observationIdsOnly = elasticsearchSvc.buildObservationTermsResults(responseIds); + + assertEquals(1, observationIdsOnly.size()); + ObservationJson observationIdOnly = observationIdsOnly.get(0); + assertEquals(RESOURCEPID, observationIdOnly.getIdentifier()); + + // execute full Observation search - Terms Aggregation + SearchRequest searchRequestAllFields = elasticsearchSvc.buildObservationAllFieldsTermsSearchRequest(1000, searchParameterMap, 3); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequestAllFields); + List observations = elasticsearchSvc.buildObservationTermsResults(response); + + validateFullObservationSearch(observations); +*/ + } + + private void validateFullObservationSearch(List observations) throws IOException { + + assertEquals(1, observations.size()); + ObservationJson observation = observations.get(0); + assertEquals(RESOURCEPID, observation.getIdentifier()); + + assertEquals(SUBJECTTYPEANDID, observation.getSubject()); + assertEquals(RESOURCEPID, observation.getIdentifier()); + assertEquals(EFFECTIVEDTM, observation.getEffectiveDtm()); + assertEquals(OBSERVATIONSINGLECODEID, observation.getCode_concept_id()); + + List category_concept_text_values = observation.getCategory_concept_text(); + assertEquals(3,category_concept_text_values.size()); + assertEquals(FIRSTCATEGORYTEXT, category_concept_text_values.get(0)); + assertEquals(SECONDCATEGORYTEXT, category_concept_text_values.get(1)); + assertEquals(THIRDCATEGORYTEXT, category_concept_text_values.get(2)); + + List> category_codings_systems = observation.getCategory_coding_system(); + assertEquals(3,category_codings_systems.size()); + List category_coding_systems = category_codings_systems.get(0); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_codings_systems.get(1); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + category_codings_systems.get(2); + assertEquals(3, category_coding_systems.size()); + assertEquals(CATEGORYFIRSTCODINGSYSTEM, category_coding_systems.get(0)); + assertEquals(CATEGORYSECONDCODINGSYSTEM, category_coding_systems.get(1)); + assertEquals(CATEGORYTHIRDCODINGSYSTEM, category_coding_systems.get(2)); + + List> category_codings_codes = observation.getCategory_coding_code(); + assertEquals(3, category_codings_codes.size()); + List category_coding_codes = category_codings_codes.get(0); + assertEquals(3, category_coding_codes.size()); + assertEquals(FIRSTCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(1); + assertEquals(3, category_coding_codes.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + category_coding_codes = category_codings_codes.get(2); + assertEquals(3, category_coding_codes.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGCODE, category_coding_codes.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGCODE, category_coding_codes.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGCODE, category_coding_codes.get(2)); + + List> category_codings_displays = observation.getCategory_coding_display(); + assertEquals(3, category_codings_displays.size()); + List category_coding_displays = category_codings_displays.get(0); + assertEquals(FIRSTCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(FIRSTCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(FIRSTCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(1); + assertEquals(3, category_coding_displays.size()); + assertEquals(SECONDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(SECONDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(SECONDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + category_coding_displays = category_codings_displays.get(2); + assertEquals(3, category_coding_displays.size()); + assertEquals(THIRDCATEGORYFIRSTCODINGDISPLAY, category_coding_displays.get(0)); + assertEquals(THIRDCATEGORYSECONDCODINGDISPLAY, category_coding_displays.get(1)); + assertEquals(THIRDCATEGORYTHIRDCODINGDISPLAY, category_coding_displays.get(2)); + + List> category_codings_code_system_hashes = observation.getCategory_coding_code_system_hash(); + assertEquals(3, category_codings_code_system_hashes.size()); + List category_coding_code_system_hashes = category_codings_code_system_hashes.get(0); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(1); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + category_coding_code_system_hashes = category_codings_code_system_hashes.get(2); + assertEquals(3, category_coding_code_system_hashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE)), category_coding_code_system_hashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE)), category_coding_code_system_hashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2)); + + String code_concept_text_values = observation.getCode_concept_text(); + assertEquals(OBSERVATIONCODETEXT, code_concept_text_values); + + List code_coding_systems = observation.getCode_coding_system(); + assertEquals(3,code_coding_systems.size()); + assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems.get(0)); + assertEquals(CODESECONDCODINGSYSTEM, code_coding_systems.get(1)); + assertEquals(CODETHIRDCODINGSYSTEM, code_coding_systems.get(2)); + + List code_coding_codes = observation.getCode_coding_code(); + assertEquals(3, code_coding_codes.size()); + assertEquals(CODEFIRSTCODINGCODE, code_coding_codes.get(0)); + assertEquals(CODESECONDCODINGCODE, code_coding_codes.get(1)); + assertEquals(CODETHIRDCODINGCODE, code_coding_codes.get(2)); + + List code_coding_display = observation.getCode_coding_display(); + assertEquals(3, code_coding_display.size()); + assertEquals(CODEFIRSTCODINGDISPLAY, code_coding_display.get(0)); + assertEquals(CODESECONDCODINGDISPLAY, code_coding_display.get(1)); + assertEquals(CODETHIRDCODINGDISPLAY, code_coding_display.get(2)); + + List code_coding_code_system_hash = observation.getCode_coding_code_system_hash(); + assertEquals(3, code_coding_code_system_hash.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), code_coding_code_system_hash.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), code_coding_code_system_hash.get(2)); + + // Retrieve all Observation codes +/* SearchRequest searchRequest = elasticsearchSvc.buildObservationCodesSearchRequest(1000); + SearchResponse response = elasticsearchSvc.executeSearchRequest(searchRequest); + List codes = elasticsearchSvc.buildCodeResult(response); + assertEquals(1, codes.size()); + CodeJson persistedObservationCode = codes.get(0); + + String persistedCodeConceptID = persistedObservationCode.getCodeableConceptId(); + assertEquals(OBSERVATIONSINGLECODEID, persistedCodeConceptID); + String persistedCodeConceptText = persistedObservationCode.getCodeableConceptText(); + assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText); + + List persistedCodeCodingSystems = persistedObservationCode.getCoding_system(); + assertEquals(3,persistedCodeCodingSystems.size()); + assertEquals(CODEFIRSTCODINGSYSTEM, persistedCodeCodingSystems.get(0)); + assertEquals(CODESECONDCODINGSYSTEM, persistedCodeCodingSystems.get(1)); + assertEquals(CODETHIRDCODINGSYSTEM, persistedCodeCodingSystems.get(2)); + + List persistedCodeCodingCodes = persistedObservationCode.getCoding_code(); + assertEquals(3, persistedCodeCodingCodes.size()); + assertEquals(CODEFIRSTCODINGCODE, persistedCodeCodingCodes.get(0)); + assertEquals(CODESECONDCODINGCODE, persistedCodeCodingCodes.get(1)); + assertEquals(CODETHIRDCODINGCODE, persistedCodeCodingCodes.get(2)); + + List persistedCodeCodingDisplays = persistedObservationCode.getCoding_display(); + assertEquals(3, persistedCodeCodingDisplays.size()); + assertEquals(CODEFIRSTCODINGDISPLAY, persistedCodeCodingDisplays.get(0)); + assertEquals(CODESECONDCODINGDISPLAY, persistedCodeCodingDisplays.get(1)); + assertEquals(CODETHIRDCODINGDISPLAY, persistedCodeCodingDisplays.get(2)); + + List persistedCodeCodingCodeSystemHashes = persistedObservationCode.getCoding_code_system_hash(); + assertEquals(3, persistedCodeCodingCodeSystemHashes.size()); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(0)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(1)); + assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE)), persistedCodeCodingCodeSystemHashes.get(2)); +*/ + + } + + private void createSingleObservation() throws IOException { + ObservationJson indexedObservation = new ObservationJson(); + indexedObservation.setIdentifier(RESOURCEPID); + indexedObservation.setSubject(SUBJECTTYPEANDID); + indexedObservation.setEffectiveDtm(EFFECTIVEDTM); + + // Add three CodeableConcepts for category + List categoryConcepts = new ArrayList<>(); + // Create three codings and first category CodeableConcept + List category1 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText(FIRSTCATEGORYTEXT); + category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY)); + category1.add(new Coding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY)); + category1.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept1.setCoding(category1); + categoryConcepts.add(categoryCodeableConcept1); + // Create three codings and second category CodeableConcept + List category2 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText(SECONDCATEGORYTEXT); + category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY)); + category2.add(new Coding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY)); + category2.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept2.setCoding(category2); + categoryConcepts.add(categoryCodeableConcept2); + // Create three codings and third category CodeableConcept + List category3 = new ArrayList<>(); + CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText(THIRDCATEGORYTEXT); + category3.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY)); + category3.add(new Coding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY)); + category3.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY)); + categoryCodeableConcept3.setCoding(category3); + categoryConcepts.add(categoryCodeableConcept3); + indexedObservation.setCategories(categoryConcepts); + + // Create CodeableConcept for Code with three codings. + indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID); + CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT); + codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY)); + codeableConceptField.addCoding(new Coding(CODESECONDCODINGSYSTEM, CODESECONDCODINGCODE, CODESECONDCODINGDISPLAY)); + codeableConceptField.addCoding(new Coding(CODETHIRDCODINGSYSTEM, CODETHIRDCODINGCODE, CODETHIRDCODINGDISPLAY)); + indexedObservation.setCode(codeableConceptField); + + String observationDocument = ourMapperNonPrettyPrint.writeValueAsString(indexedObservation); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.OBSERVATION_INDEX, RESOURCEPID, observationDocument, IndexConstants.OBSERVATION_DOCUMENT_TYPE)); + + CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID); + String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode); + assertTrue(elasticsearchSvc.performIndex(IndexConstants.CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, IndexConstants.CODE_DOCUMENT_TYPE)); + + try { + Thread.sleep(1000L); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java new file mode 100644 index 00000000000..4b358584ca7 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchConfig.java @@ -0,0 +1,53 @@ +package ca.uhn.fhir.jpa.search.lastn.config; + +import ca.uhn.fhir.context.ConfigurationException; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; +import pl.allegro.tech.embeddedelasticsearch.PopularProperties; + +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") +@EnableTransactionManagement +public class TestElasticsearchConfig { + + private final String elasticsearchHost = "127.0.0.1"; + private final String elasticsearchUserId = ""; + private final String elasticsearchPassword = ""; + + private static final String ELASTIC_VERSION = "6.5.4"; + + + @Bean() + public ElasticsearchSvcImpl myElasticsearchSvc() throws IOException { + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + + @Bean + public EmbeddedElastic embeddedElasticSearch() { + EmbeddedElastic embeddedElastic = null; + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } + + return embeddedElastic; + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java new file mode 100644 index 00000000000..a28e092bba9 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchV5Config.java @@ -0,0 +1,53 @@ +package ca.uhn.fhir.jpa.search.lastn.config; + +import ca.uhn.fhir.context.ConfigurationException; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchV5SvcImpl; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; +import pl.allegro.tech.embeddedelasticsearch.PopularProperties; + +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@Configuration +@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory") +@EnableTransactionManagement +public class TestElasticsearchV5Config { + + private final String elasticsearchHost = "127.0.0.1"; + private final String elasticsearchUserId = ""; + private final String elasticsearchPassword = ""; + + private static final String ELASTIC_VERSION = "5.6.16"; + + + @Bean() + public ElasticsearchV5SvcImpl myElasticsearchSvc() throws IOException { + int elasticsearchPort = embeddedElasticSearch().getHttpPort(); + return new ElasticsearchV5SvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUserId, elasticsearchPassword); + } + + @Bean + public EmbeddedElastic embeddedElasticSearch() { + EmbeddedElastic embeddedElastic = null; + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } + + return embeddedElastic; + } + +} diff --git a/pom.xml b/pom.xml index bf88321d7d5..2b8c3bdb66b 100644 --- a/pom.xml +++ b/pom.xml @@ -1561,7 +1561,7 @@ org.basepom.maven duplicate-finder-maven-plugin - 1.3.0 + 1.4.0 de.jpdigital @@ -2466,6 +2466,7 @@ hapi-fhir-structures-r5 hapi-fhir-validation-resources-r5 hapi-fhir-igpacks + hapi-fhir-elasticsearch-6 hapi-fhir-jpaserver-model hapi-fhir-jpaserver-searchparam hapi-fhir-jpaserver-subscription