honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changed observation indexing to bypass hibernate search and removed Observation index tables.

This commit is contained in:
ianmarshall 2020-06-04 13:20:42 -04:00
parent 6e507bf642
commit 9696af5dc0
11 changed files with 202 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/ObservationLastNIndexPersistSvc.java
View File

@ -21,36 +21,26 @@ package ca.uhn.fhir.jpa.dao;
*/
import ca.uhn.fhir.context.*;
import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodingSearchParamDao;
import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao;
import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
import ca.uhn.fhir.jpa.model.util.CodeSystemHash;
import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc;
import ca.uhn.fhir.jpa.search.lastn.json.CodeJson;
import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson;
import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor;
import ca.uhn.fhir.jpa.searchparam.extractor.PathAndRef;
import org.hl7.fhir.instance.model.api.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceContextType;
import java.util.*;
@Transactional(propagation = Propagation.REQUIRED)
public class ObservationLastNIndexPersistSvc {
@PersistenceContext(type = PersistenceContextType.TRANSACTION)
protected EntityManager myEntityManager;
@Autowired
IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao;
private ISearchParamExtractor mySearchParameterExtractor;
@Autowired
IObservationIndexedCodeCodingSearchParamDao myObservationIndexedCodeCodingSearchParamDao;
@Autowired
public ISearchParamExtractor mySearchParameterExtractor;
@Autowired(required = false)
private IElasticsearchSvc myElasticsearchSvc;
public void indexObservation(IBaseResource theResource) {
@ -95,15 +85,12 @@ public class ObservationLastNIndexPersistSvc {
List<IBase> theObservationCategoryCodeableConcepts) {
// Determine if an index already exists for Observation:
boolean observationIndexUpdate = false;
ObservationIndexedSearchParamLastNEntity indexedObservation = null;
ObservationJson indexedObservation = null;
if (resourcePID != null) {
indexedObservation = myResourceIndexedObservationLastNDao.findByIdentifier(resourcePID);
indexedObservation = myElasticsearchSvc.getObservationDocument(resourcePID);
}
if (indexedObservation == null) {
indexedObservation = new ObservationIndexedSearchParamLastNEntity();
} else {
observationIndexUpdate = true;
indexedObservation = new ObservationJson();
}
indexedObservation.setEffectiveDtm(theEffectiveDtm);
@ -114,77 +101,65 @@ public class ObservationLastNIndexPersistSvc {
addCategoriesToObservationIndex(theObservationCategoryCodeableConcepts, indexedObservation);
if (observationIndexUpdate) {
myEntityManager.merge(indexedObservation);
} else {
myEntityManager.persist(indexedObservation);
}
myElasticsearchSvc.createOrUpdateObservationIndex(resourcePID, indexedObservation);
}
private void addCodeToObservationIndex(List<IBase> theObservationCodeCodeableConcepts,
ObservationIndexedSearchParamLastNEntity theIndexedObservation) {
ObservationJson theIndexedObservation) {
// Determine if a Normalized ID was created previously for Observation Code
Optional<String> existingObservationCodeNormalizedId = getCodeCodeableConceptIdIfExists(theObservationCodeCodeableConcepts.get(0));
String existingObservationCodeNormalizedId = getCodeCodeableConceptId(theObservationCodeCodeableConcepts.get(0));
// Create/update normalized Observation Code index record
ObservationIndexedCodeCodeableConceptEntity codeableConceptField =
CodeJson codeableConceptField =
getCodeCodeableConcept(theObservationCodeCodeableConcepts.get(0),
existingObservationCodeNormalizedId.orElse(UUID.randomUUID().toString()));
existingObservationCodeNormalizedId);
if (existingObservationCodeNormalizedId.isPresent()) {
myEntityManager.merge(codeableConceptField);
} else {
myEntityManager.persist(codeableConceptField);
}
theIndexedObservation.setObservationCode(codeableConceptField);
theIndexedObservation.setCodeNormalizedId(codeableConceptField.getCodeableConceptId());
myElasticsearchSvc.createOrUpdateObservationCodeIndex(codeableConceptField.getCodeableConceptId(), codeableConceptField);
theIndexedObservation.setCode(codeableConceptField);
theIndexedObservation.setCode_concept_id(codeableConceptField.getCodeableConceptId());
}
private void addCategoriesToObservationIndex(List<IBase> observationCategoryCodeableConcepts,
ObservationIndexedSearchParamLastNEntity indexedObservation) {
ObservationJson indexedObservation) {
// Build CodeableConcept entities for Observation.Category
Set<ObservationIndexedCategoryCodeableConceptEntity> categoryCodeableConceptEntities = new HashSet<>();
List<CodeJson> categoryCodeableConceptEntities = new ArrayList<>();
for (IBase categoryCodeableConcept : observationCategoryCodeableConcepts) {
// Build CodeableConcept entities for each category CodeableConcept
categoryCodeableConceptEntities.add(getCategoryCodeableConceptEntities(categoryCodeableConcept));
}
indexedObservation.setCategoryCodeableConcepts(categoryCodeableConceptEntities);
indexedObservation.setCategories(categoryCodeableConceptEntities);
}
private ObservationIndexedCategoryCodeableConceptEntity getCategoryCodeableConceptEntities(IBase theValue) {
private CodeJson getCategoryCodeableConceptEntities(IBase theValue) {
String text = mySearchParameterExtractor.getDisplayTextFromCodeableConcept(theValue);
ObservationIndexedCategoryCodeableConceptEntity categoryCodeableConcept = new ObservationIndexedCategoryCodeableConceptEntity(text);
CodeJson categoryCodeableConcept = new CodeJson();
categoryCodeableConcept.setCodeableConceptText(text);
List<IBase> codings = mySearchParameterExtractor.getCodingsFromCodeableConcept(theValue);
Set<ObservationIndexedCategoryCodingEntity> categoryCodingEntities = new HashSet<>();
for (IBase nextCoding : codings) {
categoryCodingEntities.add(getCategoryCoding(nextCoding));
addCategoryCoding(nextCoding, categoryCodeableConcept);
}
categoryCodeableConcept.setObservationIndexedCategoryCodingEntitySet(categoryCodingEntities);
return categoryCodeableConcept;
}
private ObservationIndexedCodeCodeableConceptEntity getCodeCodeableConcept(IBase theValue, String observationCodeNormalizedId) {
private CodeJson getCodeCodeableConcept(IBase theValue, String observationCodeNormalizedId) {
String text = mySearchParameterExtractor.getDisplayTextFromCodeableConcept(theValue);
ObservationIndexedCodeCodeableConceptEntity codeCodeableConcept = new ObservationIndexedCodeCodeableConceptEntity(text, observationCodeNormalizedId);
CodeJson codeCodeableConcept = new CodeJson();
codeCodeableConcept.setCodeableConceptText(text);
codeCodeableConcept.setCodeableConceptId(observationCodeNormalizedId);
List<IBase> codings = mySearchParameterExtractor.getCodingsFromCodeableConcept(theValue);
for (IBase nextCoding : codings) {
codeCodeableConcept.addCoding(getCodeCoding(nextCoding, observationCodeNormalizedId));
addCodeCoding(nextCoding, codeCodeableConcept);
}
return codeCodeableConcept;
}
private Optional<String> getCodeCodeableConceptIdIfExists(IBase theValue) {
private String getCodeCodeableConceptId(IBase theValue) {
List<IBase> codings = mySearchParameterExtractor.getCodingsFromCodeableConcept(theValue);
String codeCodeableConceptId = null;
Optional<String> codeCodeableConceptIdOptional = Optional.empty();
for (IBase nextCoding : codings) {
@ -196,52 +171,52 @@ public class ObservationLastNIndexPersistSvc {
String system = param.getSystem();
String code = param.getValue();
String text = mySearchParameterExtractor.getDisplayTextForCoding(nextCoding);
if (code != null && system != null) {
codeCodeableConceptIdOptional = Optional.ofNullable(myObservationIndexedCodeCodingSearchParamDao.findByCodeAndSystem(code, system));
} else {
codeCodeableConceptIdOptional = Optional.ofNullable(myObservationIndexedCodeCodingSearchParamDao.findByDisplay(text));
}
if (codeCodeableConceptIdOptional.isPresent()) {
String codeSystemHash = String.valueOf(CodeSystemHash.hashCodeSystem(system, code));
CodeJson codeCodeableConceptDocument = myElasticsearchSvc.getObservationCodeDocument(codeSystemHash, text);
if (codeCodeableConceptDocument != null) {
codeCodeableConceptIdOptional = Optional.of(codeCodeableConceptDocument.getCodeableConceptId());
break;
}
}
}
return codeCodeableConceptIdOptional;
return codeCodeableConceptIdOptional.orElse(UUID.randomUUID().toString());
}
private ObservationIndexedCategoryCodingEntity getCategoryCoding(IBase theValue) {
private void addCategoryCoding(IBase theValue, CodeJson theCategoryCodeableConcept) {
ResourceIndexedSearchParamToken param = mySearchParameterExtractor.createSearchParamForCoding("Observation",
new RuntimeSearchParam(null, null, "category", null, null, null, null, null, null, null),
theValue);
ObservationIndexedCategoryCodingEntity observationIndexedCategoryCodingEntity = null;
if (param != null) {
String system = param.getSystem();
String code = param.getValue();
String text = mySearchParameterExtractor.getDisplayTextForCoding(theValue);
observationIndexedCategoryCodingEntity = new ObservationIndexedCategoryCodingEntity(system, code, text);
theCategoryCodeableConcept.addCoding(system, code, text);
}
return observationIndexedCategoryCodingEntity;
}
private ObservationIndexedCodeCodingEntity getCodeCoding(IBase theValue, String observationCodeNormalizedId) {
private void addCodeCoding(IBase theValue, CodeJson theObservationCode) {
ResourceIndexedSearchParamToken param = mySearchParameterExtractor.createSearchParamForCoding("Observation",
new RuntimeSearchParam(null, null, "code", null, null, null, null, null, null, null),
theValue);
ObservationIndexedCodeCodingEntity observationIndexedCodeCodingEntity = null;
if (param != null) {
String system = param.getSystem();
String code = param.getValue();
String text = mySearchParameterExtractor.getDisplayTextForCoding(theValue);
observationIndexedCodeCodingEntity = new ObservationIndexedCodeCodingEntity(system, code, text, observationCodeNormalizedId);
theObservationCode.addCoding(system, code, text);
}
return observationIndexedCodeCodingEntity;
}
public void deleteObservationIndex(IBasePersistedResource theEntity) {
ObservationIndexedSearchParamLastNEntity deletedObservationLastNEntity = myResourceIndexedObservationLastNDao.findByIdentifier(theEntity.getIdDt().getIdPart());
if (myElasticsearchSvc == null) {
// Elasticsearch is not enabled and therefore no index needs to be updated.
return;
}
ObservationJson deletedObservationLastNEntity = myElasticsearchSvc.getObservationDocument(theEntity.getIdDt().getIdPart());
if (deletedObservationLastNEntity != null) {
myEntityManager.remove(deletedObservationLastNEntity);
myElasticsearchSvc.deleteObservationDocument(deletedObservationLastNEntity.getIdentifier());
}
}

46
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IObservationIndexedCodeCodingSearchParamDao.java
View File

@ -1,46 +0,0 @@
package ca.uhn.fhir.jpa.dao.data;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2020 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodingEntity;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
@Repository
public interface IObservationIndexedCodeCodingSearchParamDao extends JpaRepository<ObservationIndexedCodeCodingEntity, Long> {
@Query("" +
"SELECT t.myCodeableConceptId FROM ObservationIndexedCodeCodingEntity t " +
"WHERE t.myCode = :code " +
"AND t.mySystem = :system " +
"")
String findByCodeAndSystem(@Param("code") String theCode, @Param("system") String theSystem);
@Query("" +
"SELECT t.myCodeableConceptId FROM ObservationIndexedCodeCodingEntity t " +
"WHERE t.myDisplay = :display" +
"")
String findByDisplay(@Param("display") String theDisplay);
}

6
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java
View File

@ -229,7 +229,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
private CompositeAggregationBuilder createObservationSubjectAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) {
CompositeValuesSourceBuilder<?> subjectValuesBuilder = new TermsValuesSourceBuilder("subject").field("subject");
List<CompositeValuesSourceBuilder<?>> compositeAggSubjectSources = new ArrayList();
List<CompositeValuesSourceBuilder<?>> compositeAggSubjectSources = new ArrayList<>();
compositeAggSubjectSources.add(subjectValuesBuilder);
CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder(GROUP_BY_SUBJECT, compositeAggSubjectSources);
compositeAggregationSubjectBuilder.subAggregation(createObservationCodeAggregationBuilder(theMaxNumberObservationsPerCode, theTopHitsInclude));
@ -514,12 +514,12 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
}
@VisibleForTesting
List<ObservationJson> executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) {
public List<ObservationJson> executeLastNWithAllFieldsForTest(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) {
return buildAndExecuteSearch(theSearchParameterMap, theFhirContext, null, t -> t, 100);
}
@VisibleForTesting
List<CodeJson> queryAllIndexedObservationCodes() throws IOException {
List<CodeJson> queryAllIndexedObservationCodesForTest() throws IOException {
SearchRequest codeSearchRequest = new SearchRequest(OBSERVATION_CODE_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// Query

40
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IElasticsearchSvc.java
View File

@ -21,6 +21,8 @@ package ca.uhn.fhir.jpa.search.lastn;
*/
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.search.lastn.json.CodeJson;
import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import java.util.List;
@ -35,4 +37,42 @@ public interface IElasticsearchSvc {
* @return
*/
List<String> executeLastN(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, Integer theMaxResultsToFetch);
/**
* Returns index document for a single Observation
* @param theDocumentID Identifier of Observation resource.
* @return
*/
ObservationJson getObservationDocument(String theDocumentID);
/**
* Returns index document for a single Observation Code that either has a coding that matches a specified Code value and system or that has a specified text value.
* @param theCodeSystemHash A hash string constructed from a Code value and Code system used to match to an Observation Code.
* @param theText A text value used to match to an Observation Code.
* @return
*/
CodeJson getObservationCodeDocument(String theCodeSystemHash, String theText);
/**
* Creates or updates index for an Observation Resource.
* @param theDocumentId Identifier for Observation resource.
* @param theObservationDocument Indexing document for Observation.
* @return
*/
Boolean createOrUpdateObservationIndex(String theDocumentId, ObservationJson theObservationDocument);
/**
* Creates or updates index for an Observation Code.
* @param theCodeableConceptID Identifier for Observation resource.
* @param theObservationCodeDocument Indexing document for Observation.
* @return
*/
Boolean createOrUpdateObservationCodeIndex(String theCodeableConceptID, CodeJson theObservationCodeDocument);
/**
* Deletes index for an Observation Resource.
* @param theDocumentId Identifier for Observation resource.
*/
void deleteObservationDocument(String theDocumentId);
}

34
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/CodeJson.java
View File

@ -24,8 +24,6 @@ import ca.uhn.fhir.jpa.model.util.CodeSystemHash;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.hl7.fhir.r4.model.CodeableConcept;
import org.hl7.fhir.r4.model.Coding;
import java.util.ArrayList;
import java.util.List;
@ -55,22 +53,26 @@ public class CodeJson {
public CodeJson(){
}
public CodeJson(CodeableConcept theCodeableConcept, String theCodeableConceptId) {
myCodeableConceptText = theCodeableConcept.getText();
myCodeableConceptId = theCodeableConceptId;
for (Coding theCoding : theCodeableConcept.getCoding()) {
myCoding_code.add(theCoding.getCode());
myCoding_system.add(theCoding.getSystem());
myCoding_display.add(theCoding.getDisplay());
myCoding_code_system_hash.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCoding.getSystem(), theCoding.getCode())));
}
}
public void setCodeableConceptId(String theCodeableConceptId) {
myCodeableConceptId = theCodeableConceptId;
}
public String getCodeableConceptId() {
public void addCoding(String theCoding_system, String theCoding_code, String theCoding_display) {
myCoding_code.add(theCoding_code);
myCoding_system.add(theCoding_system);
myCoding_display.add(theCoding_display);
myCoding_code_system_hash.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCoding_system, theCoding_code)));
}
public String getCodeableConceptId() {
return myCodeableConceptId;
}
public String getCodeableConceptText() {
public void setCodeableConceptText(String theCodeableConceptText) {
myCodeableConceptText = theCodeableConceptText;
}
public String getCodeableConceptText() {
return myCodeableConceptText;
}
@ -78,11 +80,11 @@ public class CodeJson {
return myCoding_code;
}
public List<String> getCoding_code_system_hash() {
public List<String> getCoding_code_system_hash() {
return myCoding_code_system_hash;
}
public List<String> getCoding_display() {
public List<String> getCoding_display() {
return myCoding_display;
}

39
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java
View File

@ -24,8 +24,6 @@ import ca.uhn.fhir.jpa.model.util.CodeSystemHash;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.hl7.fhir.r4.model.CodeableConcept;
import org.hl7.fhir.r4.model.Coding;
import java.util.ArrayList;
import java.util.Date;
@ -88,18 +86,24 @@ public class ObservationJson {
mySubject = theSubject;
}
public void setCategories(List<CodeableConcept> theCategories) {
for (CodeableConcept theConcept : theCategories) {
myCategory_concept_text.add(theConcept.getText());
public void setCategories(List<CodeJson> theCategories) {
for (CodeJson theConcept : theCategories) {
myCategory_concept_text.add(theConcept.getCodeableConceptText());
List<String> coding_code_system_hashes = new ArrayList<>();
List<String> coding_codes = new ArrayList<>();
List<String> coding_displays = new ArrayList<>();
List<String> coding_systems = new ArrayList<>();
for (Coding theCategoryCoding : theConcept.getCoding()) {
coding_code_system_hashes.add(String.valueOf(CodeSystemHash.hashCodeSystem(theCategoryCoding.getSystem(), theCategoryCoding.getCode())));
coding_codes.add(theCategoryCoding.getCode());
coding_displays.add(theCategoryCoding.getDisplay());
coding_systems.add(theCategoryCoding.getSystem());
for (String theCategoryCoding_code : theConcept.getCoding_code()) {
coding_codes.add(theCategoryCoding_code);
}
for (String theCategoryCoding_system : theConcept.getCoding_system()) {
coding_systems.add(theCategoryCoding_system);
}
for (String theCategoryCoding_code_system_hash : theConcept.getCoding_code_system_hash()) {
coding_code_system_hashes.add(theCategoryCoding_code_system_hash);
}
for (String theCategoryCoding_display : theConcept.getCoding_display()) {
coding_displays.add(theCategoryCoding_display);
}
myCategory_coding_code_system_hash.add(coding_code_system_hashes);
myCategory_coding_code.add(coding_codes);
@ -128,14 +132,13 @@ public class ObservationJson {
return myCategory_coding_system;
}
public void setCode(CodeableConcept theCode) {
myCode_concept_text = theCode.getText();
for (Coding theCodeCoding : theCode.getCoding()) {
myCode_coding_code_system_hash = String.valueOf(CodeSystemHash.hashCodeSystem(theCodeCoding.getSystem(), theCodeCoding.getCode()));
myCode_coding_code = theCodeCoding.getCode();
myCode_coding_display = theCodeCoding.getDisplay();
myCode_coding_system = theCodeCoding.getSystem();
}
public void setCode(CodeJson theCode) {
myCode_concept_text = theCode.getCodeableConceptText();
// Currently can only support one Coding for Observation Code
myCode_coding_code_system_hash = theCode.getCoding_code_system_hash().get(0);
myCode_coding_code = theCode.getCoding_code().get(0);
myCode_coding_display = theCode.getCoding_display().get(0);
myCode_coding_system = theCode.getCoding_system().get(0);
}

11
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java
View File

@ -6,7 +6,7 @@ import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoObservation;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoPatient;
import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient;
import ca.uhn.fhir.jpa.dao.BaseJpaTest;
import ca.uhn.fhir.jpa.dao.SearchBuilder;
import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.rest.param.ReferenceAndListParam;
import ca.uhn.fhir.rest.param.ReferenceOrListParam;
@ -31,6 +31,7 @@ import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.transaction.PlatformTransactionManager;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
@ -61,6 +62,9 @@ public class BaseR4SearchLastN extends BaseJpaTest {
@Autowired
protected FhirContext myFhirCtx;
@Autowired
private ElasticsearchSvcImpl myElasticsearchSvc;
@Autowired
protected PlatformTransactionManager myPlatformTransactionManager;
@ -101,7 +105,7 @@ public class BaseR4SearchLastN extends BaseJpaTest {
private static final Map<String, Date> observationEffectiveMap = new HashMap<>();
@Before
public void beforeCreateTestPatientsAndObservations() {
public void beforeCreateTestPatientsAndObservations() throws IOException {
// Using a static flag to ensure that test data and elasticsearch index is only created once.
// Creating this data and the index is time consuming and as such want to avoid having to repeat for each test.
// Normally would use a static @BeforeClass method for this purpose, but Autowired objects cannot be accessed in static methods.
@ -120,6 +124,9 @@ public class BaseR4SearchLastN extends BaseJpaTest {
createObservationsForPatient(patient2Id);
dataLoaded = true;
myElasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
myElasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
}
}

50
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java
View File

@ -3,13 +3,12 @@ package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient;
import ca.uhn.fhir.jpa.dao.data.IObservationIndexedCodeCodeableConceptSearchParamDao;
import ca.uhn.fhir.jpa.dao.ObservationLastNIndexPersistSvc;
import ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity;
import ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity;
import ca.uhn.fhir.jpa.dao.data.IObservationIndexedSearchParamLastNDao;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.util.CodeSystemHash;
import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl;
import ca.uhn.fhir.jpa.search.lastn.json.CodeJson;
import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.parser.IParser;
import ca.uhn.fhir.rest.param.*;
@ -42,12 +41,6 @@ import static org.junit.Assert.assertTrue;
@ContextConfiguration(classes = {TestR4ConfigWithElasticsearchClient.class})
public class PersistObservationIndexedSearchParamLastNR4IT {
@Autowired
IObservationIndexedSearchParamLastNDao myResourceIndexedObservationLastNDao;
@Autowired
IObservationIndexedCodeCodeableConceptSearchParamDao myCodeableConceptIndexedSearchParamNormalizedDao;
@Autowired
private ElasticsearchSvcImpl elasticsearchSvc;
@ -64,11 +57,10 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
protected FhirContext myFhirCtx;
@Before
public void before() {
myResourceIndexedObservationLastNDao.deleteAll();
myCodeableConceptIndexedSearchParamNormalizedDao.deleteAll();
public void before() throws IOException {
elasticsearchSvc.deleteAllDocumentsForTest(ElasticsearchSvcImpl.OBSERVATION_INDEX);
elasticsearchSvc.deleteAllDocumentsForTest(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
}
private final String SINGLE_SUBJECT_ID = "4567";
@ -85,7 +77,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
private ReferenceAndListParam multiSubjectParams = null;
@Test
public void testIndexObservationSingle() {
public void testIndexObservationSingle() throws IOException {
indexSingleObservation();
SearchParameterMap searchParameterMap = new SearchParameterMap();
searchParameterMap.setLastNMax(10);
@ -124,7 +116,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
assertEquals(SINGLE_OBSERVATION_PID, observationIdsOnly.get(0));
}
private void indexSingleObservation() {
private void indexSingleObservation() throws IOException {
Observation myObservation = new Observation();
IdType observationID = new IdType("Observation", SINGLE_OBSERVATION_PID, "1");
@ -139,6 +131,9 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
testObservationPersist.indexObservation(myObservation);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
}
private List<CodeableConcept> getCategoryCode() {
@ -179,7 +174,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
}
@Test
public void testIndexObservationMultiple() {
public void testIndexObservationMultiple() throws IOException {
indexMultipleObservations();
SearchParameterMap searchParameterMap = new SearchParameterMap();
searchParameterMap.setLastNMax(100);
@ -206,7 +201,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
}
private void indexMultipleObservations() {
private void indexMultipleObservations() throws IOException {
// Create two CodeableConcept values each for a Code with three codings.
CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code");
@ -275,10 +270,13 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
multiSubjectParams = new ReferenceAndListParam().addAnd(subjectParams);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
}
@Test
public void testDeleteObservation() {
public void testDeleteObservation() throws IOException {
indexMultipleObservations();
SearchParameterMap searchParameterMap = new SearchParameterMap();
searchParameterMap.setLastNMax(100);
@ -302,6 +300,7 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
entity.setVersion(0L);
testObservationPersist.deleteObservationIndex(entity);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
// Confirm that observation was deleted.
searchParameterMap = new SearchParameterMap();
@ -318,14 +317,16 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
}
@Test
public void testUpdateObservation() {
public void testUpdateObservation() throws IOException {
indexSingleObservation();
ObservationIndexedSearchParamLastNEntity observationIndexEntity = myResourceIndexedObservationLastNDao.findAll().get(0);
SearchParameterMap searchParameterMap = new SearchParameterMap();
searchParameterMap.setLastNMax(10);
ObservationJson observationIndexEntity = elasticsearchSvc.executeLastNWithAllFieldsForTest(searchParameterMap, myFhirCtx).get(0);
assertEquals(SINGLE_OBSERVATION_PID, observationIndexEntity.getIdentifier());
assertEquals(SINGLE_SUBJECT_ID, observationIndexEntity.getSubject());
assertEquals(SINGLE_EFFECTIVEDTM, observationIndexEntity.getEffectiveDtm());
SearchParameterMap searchParameterMap = new SearchParameterMap();
searchParameterMap = new SearchParameterMap();
ReferenceParam subjectParam = new ReferenceParam("Patient", "", SINGLE_SUBJECT_ID);
searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam)));
TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE);
@ -350,8 +351,9 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
updatedObservation.setCode(getObservationCode());
testObservationPersist.indexObservation(updatedObservation);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
ObservationIndexedSearchParamLastNEntity updatedObservationEntity = myResourceIndexedObservationLastNDao.findByIdentifier(SINGLE_OBSERVATION_PID);
ObservationJson updatedObservationEntity = elasticsearchSvc.getObservationDocument(SINGLE_OBSERVATION_PID);
assertEquals("1234", updatedObservationEntity.getSubject());
assertEquals(newEffectiveDtm.getValue(), updatedObservationEntity.getEffectiveDtm());
@ -413,6 +415,8 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
}
);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
SearchParameterMap searchParameterMap = new SearchParameterMap();
// execute Observation ID search - Composite Aggregation

20
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java
View File

@ -12,8 +12,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.hl7.fhir.r4.model.Observation;
import org.junit.*;
import org.hl7.fhir.r4.model.CodeableConcept;
import org.hl7.fhir.r4.model.Coding;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
@ -84,7 +82,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
searchParameterMap.setLastNMax(3);
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, myFhirContext);
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFieldsForTest(searchParameterMap, myFhirContext);
assertEquals(60, observations.size());
@ -305,16 +303,16 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
private void createMultiplePatientsAndObservations() throws IOException {
// Create CodeableConcepts for two Codes, each with three codings.
String codeableConceptId1 = UUID.randomUUID().toString();
CodeableConcept codeableConceptField1 = new CodeableConcept().setText("Test Codeable Concept Field for First Code");
codeableConceptField1.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display"));
CodeJson codeJson1 = new CodeJson(codeableConceptField1, codeableConceptId1);
String codeJson1Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson1);
CodeJson codeJson1 = new CodeJson();
codeJson1.setCodeableConceptText("Test Codeable Concept Field for First Code");
codeJson1.setCodeableConceptId(codeableConceptId1);
codeJson1.addCoding("http://mycodes.org/fhir/observation-code", "test-code-1", "test-code-1 display");
String codeableConceptId2 = UUID.randomUUID().toString();
CodeableConcept codeableConceptField2 = new CodeableConcept().setText("Test Codeable Concept Field for Second Code");
codeableConceptField2.addCoding(new Coding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display"));
CodeJson codeJson2 = new CodeJson(codeableConceptField2, codeableConceptId2);
String codeJson2Document = ourMapperNonPrettyPrint.writeValueAsString(codeJson2);
CodeJson codeJson2 = new CodeJson();
codeJson2.setCodeableConceptText("Test Codeable Concept Field for Second Code");
codeJson2.setCodeableConceptId(codeableConceptId1);
codeJson2.addCoding("http://mycodes.org/fhir/observation-code", "test-code-2", "test-code-2 display");
// Create CodeableConcepts for two categories, each with three codings.
// Create three codings and first category CodeableConcept

63
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java
View File

@ -17,8 +17,6 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.hl7.fhir.r4.model.CodeableConcept;
import org.hl7.fhir.r4.model.Coding;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@ -92,8 +90,8 @@ public class LastNElasticsearchSvcSingleObservationIT {
@After
public void after() throws IOException {
elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_INDEX);
elasticsearchSvc.deleteAllDocuments(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
elasticsearchSvc.deleteAllDocumentsForTest(ElasticsearchSvcImpl.OBSERVATION_INDEX);
elasticsearchSvc.deleteAllDocumentsForTest(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
}
@Test
@ -118,7 +116,7 @@ public class LastNElasticsearchSvcSingleObservationIT {
assertEquals(RESOURCEPID, observationIdsOnly.get(0));
// execute Observation search for all search fields
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFields(searchParameterMap, myFhirContext);
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFieldsForTest(searchParameterMap, myFhirContext);
validateFullObservationSearch(observations);
}
@ -226,7 +224,7 @@ public class LastNElasticsearchSvcSingleObservationIT {
assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE)), code_coding_code_system_hash);
// Retrieve all Observation codes
List<CodeJson> codes = elasticsearchSvc.queryAllIndexedObservationCodes();
List<CodeJson> codes = elasticsearchSvc.queryAllIndexedObservationCodesForTest();
assertEquals(1, codes.size());
CodeJson persistedObservationCode = codes.get(0);
@ -261,51 +259,44 @@ public class LastNElasticsearchSvcSingleObservationIT {
indexedObservation.setEffectiveDtm(EFFECTIVEDTM);
// Add three CodeableConcepts for category
List<CodeableConcept> categoryConcepts = new ArrayList<>();
List<CodeJson> categoryConcepts = new ArrayList<>();
// Create three codings and first category CodeableConcept
List<Coding> category1 = new ArrayList<>();
CodeableConcept categoryCodeableConcept1 = new CodeableConcept().setText(FIRSTCATEGORYTEXT);
category1.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY));
category1.add(new Coding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY));
category1.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY));
categoryCodeableConcept1.setCoding(category1);
CodeJson categoryCodeableConcept1 = new CodeJson();
categoryCodeableConcept1.setCodeableConceptText(FIRSTCATEGORYTEXT);
categoryCodeableConcept1.addCoding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY);
categoryCodeableConcept1.addCoding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY);
categoryCodeableConcept1.addCoding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY);
categoryConcepts.add(categoryCodeableConcept1);
// Create three codings and second category CodeableConcept
List<Coding> category2 = new ArrayList<>();
CodeableConcept categoryCodeableConcept2 = new CodeableConcept().setText(SECONDCATEGORYTEXT);
category2.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY));
category2.add(new Coding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY));
category2.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY));
categoryCodeableConcept2.setCoding(category2);
CodeJson categoryCodeableConcept2 = new CodeJson();
categoryCodeableConcept2.setCodeableConceptText(SECONDCATEGORYTEXT);
categoryCodeableConcept2.addCoding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY);
categoryCodeableConcept2.addCoding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY);
categoryCodeableConcept2.addCoding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY);
categoryConcepts.add(categoryCodeableConcept2);
// Create three codings and third category CodeableConcept
List<Coding> category3 = new ArrayList<>();
CodeableConcept categoryCodeableConcept3 = new CodeableConcept().setText(THIRDCATEGORYTEXT);
category3.add(new Coding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY));
category3.add(new Coding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY));
category3.add(new Coding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY));
categoryCodeableConcept3.setCoding(category3);
CodeJson categoryCodeableConcept3 = new CodeJson();
categoryCodeableConcept3.setCodeableConceptText(THIRDCATEGORYTEXT);
categoryCodeableConcept3.addCoding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY);
categoryCodeableConcept3.addCoding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY);
categoryCodeableConcept3.addCoding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY);
categoryConcepts.add(categoryCodeableConcept3);
indexedObservation.setCategories(categoryConcepts);
// Create CodeableConcept for Code with three codings.
indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID);
CodeableConcept codeableConceptField = new CodeableConcept().setText(OBSERVATIONCODETEXT);
codeableConceptField.addCoding(new Coding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY));
CodeJson codeableConceptField = new CodeJson();
codeableConceptField.setCodeableConceptText(OBSERVATIONCODETEXT);
codeableConceptField.addCoding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY);
indexedObservation.setCode(codeableConceptField);
assertTrue(elasticsearchSvc.createOrUpdateObservationIndex(RESOURCEPID, indexedObservation));
CodeJson observationCode = new CodeJson(codeableConceptField, OBSERVATIONSINGLECODEID);
String codeDocument = ourMapperNonPrettyPrint.writeValueAsString(observationCode);
assertTrue(elasticsearchSvc.performIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX, OBSERVATIONSINGLECODEID, codeDocument, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE));
try {
Thread.sleep(1000L);
} catch (InterruptedException theE) {
theE.printStackTrace();
}
codeableConceptField.setCodeableConceptId(OBSERVATIONSINGLECODEID);
assertTrue(elasticsearchSvc.createOrUpdateObservationCodeIndex(OBSERVATIONSINGLECODEID, codeableConceptField));
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
}
}

63
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ObservationIndexedCodeCodingEntity.java
View File

@ -1,63 +0,0 @@
package ca.uhn.fhir.jpa.model.entity;
/*-
* #%L
* HAPI FHIR Model
* %%
* Copyright (C) 2014 - 2020 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.model.util.CodeSystemHash;
import org.hibernate.search.annotations.Analyze;
import org.hibernate.search.annotations.Field;
import javax.persistence.*;
@Entity
@Embeddable
@Table(name = "HFJ_SPIDX_LASTN_CODING")
public class ObservationIndexedCodeCodingEntity {
public static final int MAX_LENGTH = 200;
@Id
@Column(name = "CODEABLE_CONCEPT_ID", length = MAX_LENGTH)
private String myCodeableConceptId;
@Field(name = "code", analyze = Analyze.NO)
private String myCode;
@Field(name = "system", analyze = Analyze.NO)
private String mySystem;
@Field(name = "code_system_hash", analyze = Analyze.NO)
private String myCodeSystemHash;
@Field(name = "display")
private String myDisplay;
public ObservationIndexedCodeCodingEntity() {
}
public ObservationIndexedCodeCodingEntity(String theSystem, String theCode, String theDisplay, String theCodeableConceptId) {
myCode = theCode;
mySystem = theSystem;
myCodeSystemHash = String.valueOf(CodeSystemHash.hashCodeSystem(theSystem, theCode));
myDisplay = theDisplay;
myCodeableConceptId = theCodeableConceptId;
}
}