honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Speed up loinc ingestion (#1991) 

* Speed up loinc ingestion

* Add some logging
This commit is contained in:
James Agnew 2020-07-23 09:20:33 -04:00 committed by GitHub
parent 160778d3fd
commit a6fd6254db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerConfigR4.java
View File

@ -31,6 +31,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.EnableTransactionManagement;
@ -97,8 +98,9 @@ public class FhirServerConfigR4 extends BaseJavaConfigR4 {
return retVal; return retVal;
} }
@Primary
@Bean @Bean
public JpaTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { public JpaTransactionManager hapiTransactionManager(EntityManagerFactory entityManagerFactory) {
JpaTransactionManager retVal = new JpaTransactionManager(); JpaTransactionManager retVal = new JpaTransactionManager();
retVal.setEntityManagerFactory(entityManagerFactory); retVal.setEntityManagerFactory(entityManagerFactory);
return retVal; return retVal;

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java
View File

@ -1649,6 +1649,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
while (true) { while (true) {
StopWatch sw = new StopWatch();
TermValueSet valueSetToExpand = txTemplate.execute(t -> { TermValueSet valueSetToExpand = txTemplate.execute(t -> {
Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded(); Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded();
if (!optionalTermValueSet.isPresent()) { if (!optionalTermValueSet.isPresent()) {
@ -1669,7 +1670,9 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).get(); TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).get();
return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource()); return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
}); });
expandValueSet(null, valueSet, new ValueSetConceptAccumulator(valueSetToExpand, myValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao));
ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator(valueSetToExpand, myValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao);
expandValueSet(null, valueSet, accumulator);
// We are done with this ValueSet. // We are done with this ValueSet.
txTemplate.execute(t -> { txTemplate.execute(t -> {
@ -1678,6 +1681,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
return null; return null;
}); });
ourLog.info("Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", valueSet.getId(), valueSet.getUrl(), accumulator.getConceptsSaved(), sw.toString());
} catch (Exception e) { } catch (Exception e) {
ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e); ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e);
txTemplate.execute(t -> { txTemplate.execute(t -> {

12
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImpl.java
View File

@ -141,7 +141,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
int codeCount = 0, relCount = 0; int codeCount = 0, relCount = 0;
StopWatch stopwatch = new StopWatch(); StopWatch stopwatch = new StopWatch();
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myDeferredConcepts.size()); int count = Math.min(1000, myDeferredConcepts.size());
ourLog.info("Saving {} deferred concepts...", count); ourLog.info("Saving {} deferred concepts...", count);
while (codeCount < count && myDeferredConcepts.size() > 0) { while (codeCount < count && myDeferredConcepts.size() > 0) {
TermConcept next = myDeferredConcepts.remove(0); TermConcept next = myDeferredConcepts.remove(0);
@ -164,7 +164,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
} }
if (codeCount == 0) { if (codeCount == 0) {
count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptLinksToSaveLater.size()); count = Math.min(1000, myConceptLinksToSaveLater.size());
ourLog.info("Saving {} deferred concept relationships...", count); ourLog.info("Saving {} deferred concept relationships...", count);
while (relCount < count && myConceptLinksToSaveLater.size() > 0) { while (relCount < count && myConceptLinksToSaveLater.size() > 0) {
TermConceptParentChildLink next = myConceptLinksToSaveLater.remove(0); TermConceptParentChildLink next = myConceptLinksToSaveLater.remove(0);
@ -193,7 +193,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
} }
private void processDeferredValueSets() { private void processDeferredValueSets() {
int count = Math.min(myDeferredValueSets.size(), 20); int count = Math.min(myDeferredValueSets.size(), 200);
for (ValueSet nextValueSet : new ArrayList<>(myDeferredValueSets.subList(0, count))) { for (ValueSet nextValueSet : new ArrayList<>(myDeferredValueSets.subList(0, count))) {
ourLog.info("Creating ValueSet: {}", nextValueSet.getId()); ourLog.info("Creating ValueSet: {}", nextValueSet.getId());
myTerminologyVersionAdapterSvc.createOrUpdateValueSet(nextValueSet); myTerminologyVersionAdapterSvc.createOrUpdateValueSet(nextValueSet);
@ -237,6 +237,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
processDeferredConcepts(); processDeferredConcepts();
return null; return null;
}); });
continue;
} }
if (isDeferredValueSets()) { if (isDeferredValueSets()) {
@ -244,6 +246,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
processDeferredValueSets(); processDeferredValueSets();
return null; return null;
}); });
continue;
} }
if (isDeferredConceptMaps()) { if (isDeferredConceptMaps()) {
@ -251,6 +255,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
processDeferredConceptMaps(); processDeferredConceptMaps();
return null; return null;
}); });
continue;
} }
if (isDeferredCodeSystemDeletions()) { if (isDeferredCodeSystemDeletions()) {

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java
View File

@ -48,6 +48,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
private int myConceptsSaved; private int myConceptsSaved;
private int myDesignationsSaved; private int myDesignationsSaved;
private int myConceptsExcluded; private int myConceptsExcluded;
private int myCount;
public ValueSetConceptAccumulator(@Nonnull TermValueSet theTermValueSet, @Nonnull ITermValueSetDao theValueSetDao, @Nonnull ITermValueSetConceptDao theValueSetConceptDao, @Nonnull ITermValueSetConceptDesignationDao theValueSetConceptDesignationDao) { public ValueSetConceptAccumulator(@Nonnull TermValueSet theTermValueSet, @Nonnull ITermValueSetDao theValueSetDao, @Nonnull ITermValueSetConceptDao theValueSetConceptDao, @Nonnull ITermValueSetConceptDesignationDao theValueSetConceptDesignationDao) {
myTermValueSet = theTermValueSet; myTermValueSet = theTermValueSet;
@ -171,7 +172,11 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
return true; return true;
} }
// TODO: DM 2019-07-16 - We may need TermValueSetConceptProperty, similar to TermConceptProperty. public int getConceptsSaved() {
return myConceptsSaved;
}
// TODO: DM 2019-07-16 - We may need TermValueSetConceptProperty, similar to TermConceptProperty.
// TODO: DM 2019-07-16 - If so, we should also populate TermValueSetConceptProperty entities here. // TODO: DM 2019-07-16 - If so, we should also populate TermValueSetConceptProperty entities here.
// TODO: DM 2019-07-30 - Expansions don't include the properties themselves; they may be needed to facilitate filters and parameterized expansions. // TODO: DM 2019-07-30 - Expansions don't include the properties themselves; they may be needed to facilitate filters and parameterized expansions.
} }

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartLinkHandler.java
View File

@ -94,7 +94,7 @@ public class LoincPartLinkHandler implements IRecordHandler {
return; return;
} }
ourLog.info("Adding new property {} = {}", propertyPart, partNumber); ourLog.debug("Adding new property {} = {}", propertyPart, partNumber);
if (propertyType == CodeSystem.PropertyType.STRING) { if (propertyType == CodeSystem.PropertyType.STRING) {
loincConcept.addPropertyString(propertyPart, partName); loincConcept.addPropertyString(propertyPart, partName);
} else { } else {