diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4052-addition-of-properties-to-loinc-upload-process.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4052-addition-of-properties-to-loinc-upload-process.yaml new file mode 100644 index 00000000000..a05413cab7f --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4052-addition-of-properties-to-loinc-upload-process.yaml @@ -0,0 +1,5 @@ +--- +type: add +issue: 4052 +title: "LOINC terminology upload process was enhanced to consider 24 additional properties which were defined in + loinc.csv file but not uploaded." diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincDocumentOntologyHandler.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincDocumentOntologyHandler.java index f76db9aebaa..31e230bdc26 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincDocumentOntologyHandler.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincDocumentOntologyHandler.java @@ -29,6 +29,8 @@ import org.apache.commons.csv.CSVRecord; import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.ConceptMap; import org.hl7.fhir.r4.model.ValueSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; import java.util.Map; @@ -38,6 +40,7 @@ import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_CODESYS import static org.apache.commons.lang3.StringUtils.trim; public class LoincDocumentOntologyHandler extends BaseLoincHandler implements IZipContentsHandlerCsv { + private static final Logger ourLog = LoggerFactory.getLogger(LoincDocumentOntologyHandler.class); public static final String DOCUMENT_ONTOLOGY_CODES_VS_ID = "loinc-document-ontology"; public static final String DOCUMENT_ONTOLOGY_CODES_VS_URI = "http://loinc.org/vs/loinc-document-ontology"; @@ -96,6 +99,7 @@ public class LoincDocumentOntologyHandler extends BaseLoincHandler implements IZ TermConcept code = myCode2Concept.get(loincNumber); if (code != null) { code.addPropertyCoding(loincCodePropName, ITermLoaderSvc.LOINC_URI, partNumber, partName); + ourLog.debug("Adding coding property: {} to concept.code {}", loincCodePropName, partNumber); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincHandler.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincHandler.java index 41ebeeca68c..88dd49bd6dc 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincHandler.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincHandler.java @@ -25,6 +25,7 @@ import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; import ca.uhn.fhir.jpa.entity.TermConcept; import ca.uhn.fhir.jpa.term.IZipContentsHandlerCsv; import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl; +import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.Validate; @@ -32,13 +33,21 @@ import org.hl7.fhir.r4.model.CodeSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; -import static org.apache.commons.lang3.StringUtils.*; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.trim; public class LoincHandler implements IZipContentsHandlerCsv { private static final Logger ourLog = LoggerFactory.getLogger(LoincHandler.class); + + // most coding properties are not loaded by this handler, except these + private static final List myCodingPropertiesToLoad = List.of("AskAtOrderEntry", "AssociatedObservations"); + private final Map myCode2Concept; private final TermCodeSystemVersion myCodeSystemVersion; private final Map myPropertyNames; @@ -84,9 +93,18 @@ public class LoincHandler implements IZipContentsHandlerCsv { switch (nextPropertyType) { case STRING: concept.addPropertyString(nextPropertyName, nextPropertyValue); + ourLog.trace("Adding string property: {} to concept.code {}", nextPropertyName, concept.getCode()); break; + case CODING: - // These are handles by the LOINC PartLink file + if (myCodingPropertiesToLoad.contains(nextPropertyName)) { + List propertyCodeValues = parsePropertyCodeValues(nextPropertyValue); + for (String propertyCodeValue : propertyCodeValues) { + concept.addPropertyCoding(nextPropertyName, ITermLoaderSvc.LOINC_URI, propertyCodeValue, display); + ourLog.trace("Adding coding property: {} to concept.code {}", nextPropertyName, concept.getCode()); + } + } + // rest of "Coding" property types are handled by partlink, hierarchy, RsnaPlaybook or DocumentOntology handlers break; case DECIMAL: @@ -105,4 +123,10 @@ public class LoincHandler implements IZipContentsHandlerCsv { myCode2Concept.put(code, concept); } } + + private List parsePropertyCodeValues(String theValue) { + return Arrays.stream( theValue.split(";") ) + .map(String::trim) + .collect(Collectors.toList()); + } } diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java index 8a56e76f0b4..dbfcecc4e51 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java @@ -163,12 +163,7 @@ public class TestR4Config { }; - retVal.setDriver(new org.h2.Driver()); - retVal.setUrl("jdbc:h2:mem:testdb_r4"); - retVal.setMaxWaitMillis(30000); - retVal.setUsername(""); - retVal.setPassword(""); - retVal.setMaxTotal(ourMaxThreads); + setConnectionProperties(retVal); SLF4JLogLevel level = SLF4JLogLevel.INFO; DataSource dataSource = ProxyDataSourceBuilder @@ -186,6 +181,17 @@ public class TestR4Config { return dataSource; } + + public void setConnectionProperties(BasicDataSource theDataSource) { + theDataSource.setDriver(new org.h2.Driver()); + theDataSource.setUrl("jdbc:h2:mem:testdb_r4"); + theDataSource.setMaxWaitMillis(30000); + theDataSource.setUsername(""); + theDataSource.setPassword(""); + theDataSource.setMaxTotal(ourMaxThreads); + } + + @Bean public SingleQueryCountHolder singleQueryCountHolder() { return new SingleQueryCountHolder(); @@ -211,7 +217,7 @@ public class TestR4Config { extraProperties.put("hibernate.format_sql", "false"); extraProperties.put("hibernate.show_sql", "false"); extraProperties.put("hibernate.hbm2ddl.auto", "update"); - extraProperties.put("hibernate.dialect", HapiFhirH2Dialect.class.getName()); + extraProperties.put("hibernate.dialect", getHibernateDialect()); hibernateSearchConfigurer.apply(extraProperties); @@ -220,6 +226,10 @@ public class TestR4Config { return extraProperties; } + public String getHibernateDialect() { + return HapiFhirH2Dialect.class.getName(); + } + /** * Bean which validates incoming requests */ diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/term/LoincFullLoadR4SandboxIT.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/term/LoincFullLoadR4SandboxIT.java new file mode 100644 index 00000000000..297fc7c6f49 --- /dev/null +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/term/LoincFullLoadR4SandboxIT.java @@ -0,0 +1,574 @@ +package ca.uhn.fhir.jpa.term; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet; +import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao; +import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao; +import ca.uhn.fhir.jpa.dao.data.ITermConceptDao; +import ca.uhn.fhir.jpa.entity.TermCodeSystem; +import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; +import ca.uhn.fhir.jpa.entity.TermConcept; +import ca.uhn.fhir.jpa.entity.TermConceptProperty; +import ca.uhn.fhir.jpa.entity.TermValueSet; +import ca.uhn.fhir.jpa.model.entity.ForcedId; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider; +import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; +import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc; +import ca.uhn.fhir.jpa.term.api.ITermReadSvc; +import ca.uhn.fhir.jpa.test.BaseJpaTest; +import ca.uhn.fhir.jpa.test.config.TestHSearchAddInConfig; +import ca.uhn.fhir.jpa.test.config.TestR4Config; +import ca.uhn.fhir.util.StopWatch; +import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.csv.QuoteMode; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.hibernate.dialect.PostgreSQL10Dialect; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.ValueSet; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ResourceUtils; + +import javax.annotation.Nonnull; +import javax.persistence.EntityManager; +import javax.persistence.Query; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; + +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toSet; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; + + +/** + * Sandbox test (not intended to run on CI build) so must be kept disabled + * + * Requires the loinc-full resource directory to contain the following three files: + * _ Loinc.csv.gz + * _ Loinc_1.11.zip and + * _ v1.11_loincupload.properties + * + * but they are too large for the repo, so before running this test, copy them from: + * https://drive.google.com/drive/folders/18be2R5IurlWnugkl18nDG7wrwPsOtfR-?usp=sharing + * (SmileCDR has access) + * + * Can be executed with Lucene or Elastic configuration + * + * Requires 4Gb mem to run, so pom needs to be changed to run from IDE: + * -Dfile.encoding=UTF-8 -Xmx5g + * or to run from maven use: + * mvn test -pl :hapi-fhir-jpaserver-test-utilities -Dtest=LoincFullLoadR4SandboxIT#uploadLoincCodeSystem -Dsurefire_jvm_args="-Xmx5g" + * + */ +@Disabled("Sandbox test which requires 5Gb memory") +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = { + LoincFullLoadR4SandboxIT.NoopMandatoryTransactionListener.class + + // one of the following needs to be present + // TestR4Config.class // uses in-memory DB + ,LoincFullLoadR4SandboxIT.OverriddenR4Config.class // your configured persistent DB + + // pick up elastic or lucene engine: + ,TestHSearchAddInConfig.NoFT.class +}) +public class LoincFullLoadR4SandboxIT extends BaseJpaTest { + private static final Logger ourLog = LoggerFactory.getLogger(LoincFullLoadR4SandboxIT.class); + + private static final DecimalFormat ourDecimalFormat = new DecimalFormat("#,###"); + + public static final boolean USE_REAL_DB = true; + public static final boolean LOAD_DB = false; + public static final String DB_NAME = "testDB_new"; + + + public static final String LOINC_URL = "http://loinc.org"; + public static final String TEST_FILES_CLASSPATH = "loinc-full/"; + + public static final boolean CLEANUP_DATA = true; + + static { + System.setProperty("unlimited_db_connection", "true"); + } + + +// ----------------------------------------------------------------------------------------- +// full LOINC file 1.11 (initially cloned from 2.73 for tests, with custom lonc.xml file with added 24 new properties) + + public static final String CS_VERSION = "1.11"; + public static final int CS_CONCEPTS_COUNT = 234_390; + public static final int ASSOCIATED_OBSERVATIONS_COUNT = 8_058; + public static final int ASK_AT_ORDER_ENTRY_COUNT = 65; + + public static final String LOINC_PROPERTIES_CLASSPATH = + ResourceUtils.CLASSPATH_URL_PREFIX + TEST_FILES_CLASSPATH + "v1.11_loincupload.properties"; + + public static final String LOINC_ZIP_CLASSPATH = + ResourceUtils.CLASSPATH_URL_PREFIX + TEST_FILES_CLASSPATH + "Loinc_1.11.zip"; + + public static final String LOINC_CSV_ZIP_CLASSPATH = + ResourceUtils.CLASSPATH_URL_PREFIX + TEST_FILES_CLASSPATH + "Loinc.csv.gz"; +// ----------------------------------------------------------------------------------------- + + @Autowired private FhirContext myFhirCtx; + @Autowired private PlatformTransactionManager myTxManager; + @Autowired private EntityManager myEntityManager; + @Autowired private TermLoaderSvcImpl myTermLoaderSvc; + @Autowired private ITermConceptDao myTermConceptDao; + @Autowired private ITermReadSvc myTermReadSvc; + @Autowired private ITermDeferredStorageSvc myTerminologyDeferredStorageSvc; + @Autowired private ITermCodeSystemDao myTermCodeSystemDao; + @Autowired private ITermCodeSystemVersionDao myTermCodeSystemVersionDao; + + + @Autowired + @Qualifier("myValueSetDaoR4") + protected IFhirResourceDaoValueSet myValueSetDao; + + + + private long termCodeSystemVersionWithVersionId; + + private int associatedObservationsCount = 0; + private int askAtOrderEntryCount = 0; + private int processedPropertiesCounter = 0; + + private static List recordPropertyNames; + private static List newRecordPropertyNames = List.of( + "CHNG_TYPE", + "DefinitionDescription", + "CONSUMER_NAME", + "FORMULA", + "EXMPL_ANSWERS", + "SURVEY_QUEST_TEXT", + "SURVEY_QUEST_SRC", + "UNITSREQUIRED", + "RELATEDNAMES2", + "SHORTNAME", + "HL7_FIELD_SUBFIELD_ID", + "EXTERNAL_COPYRIGHT_NOTICE", + "EXAMPLE_UNITS", + "LONG_COMMON_NAME", + "EXAMPLE_UCUM_UNITS", + "STATUS_REASON", + "STATUS_TEXT", + "CHANGE_REASON_PUBLIC", + "COMMON_TEST_RANK", + "COMMON_ORDER_RANK", + "EXTERNAL_COPYRIGHT_LINK", + "AskAtOrderEntry", // coding + "AssociatedObservations", // Coding + "ValidHL7AttachmentRequest" + ); + + @Test() + public void uploadLoincCodeSystem() throws Exception { + + if (USE_REAL_DB && LOAD_DB) { + List myFileDescriptors = buildFileDescriptors(); + + // upload terminology + StopWatch sw = new StopWatch(); + myTermLoaderSvc.loadLoinc(myFileDescriptors, mySrd); + ourLog.info("=================> Uploading terminology took {}", sw); + + // save all deferred concepts, properties, links, etc + sw.restart(); + myTerminologyDeferredStorageSvc.saveAllDeferred(); + ourLog.info("=================> Saving all terminology deferred entities took {}", sw); + validateSavedConceptsCount(); + + sw.restart(); + myTermReadSvc.preExpandDeferredValueSetsToTerminologyTables(); + ourLog.info("=================> Pre-expanding ValueSets took {}", sw); + + return; + } + + // validation: + // create from loinc.csv file map of code | set of not-blank-properties + // query each code and validate that all properties in map are set (can we check type also) + + List> conceptPropertyRecords = readCsvRecordsAsMap(); + + validateCreatedConceptsHaveAllProperties( conceptPropertyRecords ); + + ourLog.info("Processed properties : {}", processedPropertiesCounter); + ourLog.info("associatedObservationsCount : {}", associatedObservationsCount); + ourLog.info("askAtOrderEntryCount : {}", askAtOrderEntryCount); + ourLog.info(""); + + assertEquals(ASK_AT_ORDER_ENTRY_COUNT, askAtOrderEntryCount); + assertEquals(ASSOCIATED_OBSERVATIONS_COUNT, associatedObservationsCount); + +} + + /** + * Used occasionally for some manual validation - don't delete + */ + private void queryForSpecificValueSet() { + runInTransaction(() -> { + Query q = myEntityManager.createQuery("from ForcedId where myForcedId like 'LG8749-6%'"); + @SuppressWarnings("unchecked") + List fIds = (List) q.getResultList(); + long res_id = fIds.stream().map(ForcedId::getId).sorted().findFirst().get(); + + Query q1 = myEntityManager.createQuery("from ResourceTable where id = " + res_id); + @SuppressWarnings("unchecked") + List vsList = (List) q1.getResultList(); + assertEquals(1, vsList.size()); + long vsLongId = vsList.get(0).getId(); + ValueSet vs = (ValueSet) myValueSetDao.toResource( vsList.get(0), false ); + assertNotNull(vs); + + Query q2 = myEntityManager.createQuery("from TermValueSet where myResource = " + vsLongId); + @SuppressWarnings("unchecked") + List tvsList = (List) q2.getResultList(); + assertEquals(1, tvsList.size()); + + TermValueSet termValueSet = tvsList.get(0); + }); + } + + + private void validateCreatedConceptsHaveAllProperties(List> theConceptPropertyInputMap) { + TermCodeSystemVersion tcsVersion = getTermCodeSystemVersion(); + + ourLog.info("Properties to process: {}", ourDecimalFormat.format(theConceptPropertyInputMap.size())); + + for (Map tcRecordMap : theConceptPropertyInputMap) { + String recordCode = getRecordCode(tcRecordMap); + processedPropertiesCounter++; + + runInTransaction(() -> { + Optional tcFomDbOpt = myTermConceptDao.findByCodeSystemAndCode(tcsVersion, recordCode); + tcFomDbOpt.ifPresentOrElse( + tc -> validateTermConceptEntry(tc, tcRecordMap), + () -> ourLog.error("Couldn't find TermConcept with code: {} in DB", recordCode)); + }); + + if (processedPropertiesCounter % 10_000 == 0) { + ourLog.info("Processed properties: {}", ourDecimalFormat.format(processedPropertiesCounter)); + } + } + ourLog.info(""); + } + + + private String getRecordCode(Map tcRecordMap) { + String recordCode = tcRecordMap.get("LOINC_NUM"); + assertNotNull(recordCode, "Record without LOINC_NUM filed ???"); + assertFalse(recordCode.isEmpty(), "Record with empty LOINC_NUM filed ???"); + return recordCode; + } + + + private void validateTermConceptEntry(TermConcept theTermConcept, Map theRecordMap) { + String recordCode = getRecordCode(theRecordMap); + if ( ! theTermConcept.getCode().equals(recordCode) ) { + fail("Received non matching inputs code from file: " + recordCode + ", code from DB: " + theTermConcept.getCode()); + } + + ourLog.trace("Validating TC with code: {}", theTermConcept.getCode()); + Map> tcConceptPropertyMap = theTermConcept.getProperties().stream() + .collect(Collectors.groupingBy(TermConceptProperty::getKey, HashMap::new, mapping(TermConceptProperty::getValue, toSet()))); + + validateNewProperties(recordCode, theRecordMap, tcConceptPropertyMap); + } + + + private void validateNewProperties(String theTcCode, Map theRecordPropsMap, Map> theTcConceptPropertyMap) { + // make sure we are good so far and both entries to compare are for same TermConcept code + assertEquals(theTcCode, theRecordPropsMap.get("LOINC_NUM"), "theTcCode and record key (LOINC_NUM) must match"); + + for (Map.Entry recordEntry : theRecordPropsMap.entrySet()) { + + // match each non-blank property of type String from the file (except LOINC_NUM) to be a property of the concept + if (recordEntry.getKey().equals("LOINC_NUM") || StringUtils.isEmpty(recordEntry.getValue()) ) { continue; } + + // bypass old properties + if ( ! newRecordPropertyNames.contains(recordEntry.getKey()) ) { continue; } + + Set tcConceptPropValues = theTcConceptPropertyMap.get(recordEntry.getKey()); + if (CollectionUtils.isEmpty(tcConceptPropValues)) { + ourLog.error("TCConcept with code: {} does not have property: {} which in csv file has value: {}", + theTcCode, recordEntry.getKey(), recordEntry.getValue()); + continue; + } + + // special case because we need to parse ';' separated codes from file property value + if ( "AssociatedObservations".equals(recordEntry.getKey()) ) { + associatedObservationsCount++; + validateAssociatedObservations(theTcCode, recordEntry, tcConceptPropValues); + continue; + } + + if ( "AskAtOrderEntry".equals(recordEntry.getKey()) ) { askAtOrderEntryCount++; } + + if ( ! tcConceptPropValues.contains(recordEntry.getValue()) ) { + ourLog.error("For TC code: {}, prop: {}, values don't match. Record value: {} TC prop value: {}", + theTcCode, recordEntry.getKey(), recordEntry.getValue(), String.join(" - ", tcConceptPropValues)); + } + } + } + + /** + * Validate that all file property codes become a "Coding" property on the TermConcept + */ + private void validateAssociatedObservations(String theTcCode, Map.Entry recordEntry, Set tcConceptPropValues) { + List recordPropertyCodes = parsePropertyCodeValues(recordEntry.getValue()); + + for (String recordPropertyCode : recordPropertyCodes) { + if ( ! tcConceptPropValues.contains(recordPropertyCode) ) { + ourLog.error("For TC code: {}, prop: {}, record code: {} not found among properties: {}", + theTcCode, recordEntry.getKey(), recordPropertyCode, String.join(" - ", tcConceptPropValues)); + } + + } + } + + + private List parsePropertyCodeValues(String theValue) { + return Arrays.stream( theValue.split(";") ) + .map(String::trim) + .collect(Collectors.toList()); + } + + + private List> readCsvRecordsAsMap() throws Exception { + CSVParser parser = getCsvRecords(); + Iterator iter = parser.iterator(); + + Map headerMap = parser.getHeaderMap(); + recordPropertyNames = headerMap.entrySet().stream() + .sorted(Comparator.comparingInt(Map.Entry::getValue)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + ourLog.debug("Header map: {}", parser.getHeaderMap()); + + int count = 0; + + List> records = new ArrayList<>(); + while (iter.hasNext()) { + CSVRecord nextRecord = iter.next(); + if (! nextRecord.isConsistent()) { + ourLog.error("Inconsistent record"); + continue; + + } + records.add( nextRecord.toMap() ); + count++; + } + ourLog.info("Read and mapped {} csv file lines", count); + return records; + } + + + public String getCvsStringFromZip(String theFilePath) throws Exception { + InputStream stream = new FileInputStream(ResourceUtils.getFile(theFilePath)); + assertNotNull(stream); + stream = new GZIPInputStream(stream); + return IOUtils.toString(stream, StandardCharsets.UTF_8); + } + + + @Nonnull + private CSVParser getCsvRecords() throws Exception { + Reader reader = new StringReader(getCvsStringFromZip(LOINC_CSV_ZIP_CLASSPATH)); + + CSVFormat format = CSVFormat + .newFormat(',') + .withFirstRecordAsHeader() + .withTrim() + .withQuote('"') + .withQuoteMode(QuoteMode.NON_NUMERIC); + return new CSVParser(reader, format); + } + + + private void validateSavedConceptsCount() { + termCodeSystemVersionWithVersionId = getTermCodeSystemVersion().getPid(); + int dbVersionedTermConceptCount = runInTransaction(() -> + myTermConceptDao.countByCodeSystemVersion(termCodeSystemVersionWithVersionId) ); + ourLog.info("=================> Number of stored concepts for version {}: {}", CS_VERSION, dbVersionedTermConceptCount); + assertEquals(CS_CONCEPTS_COUNT, dbVersionedTermConceptCount); + } + + + private TermCodeSystemVersion getTermCodeSystemVersion() { + return runInTransaction(() -> { + TermCodeSystem myTermCodeSystem = myTermCodeSystemDao.findByCodeSystemUri(LOINC_URL); + TermCodeSystemVersion termCodeSystemVersion = myTermCodeSystemVersionDao + .findByCodeSystemPidAndVersion(myTermCodeSystem.getPid(), CS_VERSION); + assertNotNull(termCodeSystemVersion); + return termCodeSystemVersion; + }); + } + + + private List buildFileDescriptors() throws FileNotFoundException { + List fileDescriptors = new ArrayList<>(); + + File propsFile = ResourceUtils.getFile(LOINC_PROPERTIES_CLASSPATH); + fileDescriptors.add( new TerminologyUploaderProvider.FileBackedFileDescriptor(propsFile) ); + + File zipFile = ResourceUtils.getFile(LOINC_ZIP_CLASSPATH); + fileDescriptors.add( new TerminologyUploaderProvider.FileBackedFileDescriptor(zipFile) ); + + return fileDescriptors; + } + + +// List of all columns in Loinc.csv input file +// private static final String[] recordFieldNames = { +// "LOINC_NUM" +// ,"COMPONENT" +// ,"PROPERTY" +// ,"TIME_ASPCT" +// ,"SYSTEM" +// ,"SCALE_TYP" +// ,"METHOD_TYP" +// ,"CLASS" +// ,"VersionLastChanged" +// ,"CHNG_TYPE" +// ,"DefinitionDescription" +// ,"STATUS" +// ,"CONSUMER_NAME" +// ,"CLASSTYPE" +// ,"FORMULA" +// ,"EXMPL_ANSWERS" +// ,"SURVEY_QUEST_TEXT" +// ,"SURVEY_QUEST_SRC" +// ,"UNITSREQUIRED" +// ,"RELATEDNAMES2" +// ,"SHORTNAME" +// ,"ORDER_OBS" +// ,"HL7_FIELD_SUBFIELD_ID" +// ,"EXTERNAL_COPYRIGHT_NOTICE" +// ,"EXAMPLE_UNITS" +// ,"LONG_COMMON_NAME" +// ,"EXAMPLE_UCUM_UNITS" +// ,"STATUS_REASON" +// ,"STATUS_TEXT" +// ,"CHANGE_REASON_PUBLIC" +// ,"COMMON_TEST_RANK" +// ,"COMMON_ORDER_RANK" +// ,"COMMON_SI_TEST_RANK" +// ,"HL7_ATTACHMENT_STRUCTURE" +// ,"EXTERNAL_COPYRIGHT_LINK" +// ,"PanelType" +// ,"AskAtOrderEntry" +// ,"AssociatedObservations" +// ,"VersionFirstReleased" +// ,"ValidHL7AttachmentRequest" +// ,"DisplayName" +// }; + + + + /** + * This configuration bypasses the MandatoryTransactionListener, which breaks this test + */ + @Configuration + public static class NoopMandatoryTransactionListener { + + @Bean + public ProxyDataSourceBuilder.SingleQueryExecution getMandatoryTransactionListener() { + return getNoopTXListener(); + } + } + + + private static ProxyDataSourceBuilder.SingleQueryExecution getNoopTXListener() { + return (execInfo, queryInfoList) -> { }; + } + + + @Override + protected FhirContext getFhirContext() { + return myFhirCtx; + } + + @Override + protected PlatformTransactionManager getTxManager() { + return myTxManager; + } + + + + @Configuration + public static class OverriddenR4Config extends TestR4Config { + + @Override + public void setConnectionProperties(BasicDataSource theDataSource) { + if (USE_REAL_DB) { + theDataSource.setDriver(new org.postgresql.Driver()); + theDataSource.setUrl("jdbc:postgresql://localhost/" + DB_NAME); + theDataSource.setMaxWaitMillis(-1); // indefinite + theDataSource.setUsername("cdr"); + theDataSource.setPassword("smileCDR"); + theDataSource.setMaxTotal(ourMaxThreads); + return; + } + + super.setConnectionProperties(theDataSource); + } + + @Override + public String getHibernateDialect() { + if (USE_REAL_DB) { + return PostgreSQL10Dialect.class.getName(); + } + + return super.getHibernateDialect(); + } + + + @Override + public ProxyDataSourceBuilder.SingleQueryExecution getMandatoryTransactionListener() { + return getNoopTXListener(); + } + + } + + +} diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/resources/loinc-full/readme.txt b/hapi-fhir-jpaserver-test-utilities/src/test/resources/loinc-full/readme.txt new file mode 100644 index 00000000000..3c8904c22ac --- /dev/null +++ b/hapi-fhir-jpaserver-test-utilities/src/test/resources/loinc-full/readme.txt @@ -0,0 +1,10 @@ +LoincFullLoadR4SandboxIT requires this directory must contain the following +three files: + _ Loinc.csv.gz + _ Loinc_1.11.zip and + _ v1.11_loincupload.properties + +but those files are too large for the repo, so before running this test you need to copy them from: +https://drive.google.com/drive/folders/18be2R5IurlWnugkl18nDG7wrwPsOtfR-?usp=sharing +(SmileCDR has access) +