honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Speed up loinc ingestion (#1991) 

* Speed up loinc ingestion

* Add some logging
This commit is contained in:
James Agnew 2020-07-23 09:20:33 -04:00 committed by GitHub
parent 160778d3fd
commit a6fd6254db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerConfigR4.java
View File

@ -31,6 +31,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.annotation.EnableTransactionManagement;
@ -97,8 +98,9 @@ public class FhirServerConfigR4 extends BaseJavaConfigR4 {
return retVal;
}
@Primary
@Bean
public JpaTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) {
public JpaTransactionManager hapiTransactionManager(EntityManagerFactory entityManagerFactory) {
JpaTransactionManager retVal = new JpaTransactionManager();
retVal.setEntityManagerFactory(entityManagerFactory);
return retVal;

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java
View File

@ -1649,6 +1649,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
while (true) {
StopWatch sw = new StopWatch();
TermValueSet valueSetToExpand = txTemplate.execute(t -> {
Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded();
if (!optionalTermValueSet.isPresent()) {
@ -1669,7 +1670,9 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).get();
return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
});
expandValueSet(null, valueSet, new ValueSetConceptAccumulator(valueSetToExpand, myValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao));
ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator(valueSetToExpand, myValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao);
expandValueSet(null, valueSet, accumulator);
// We are done with this ValueSet.
txTemplate.execute(t -> {
@ -1678,6 +1681,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
return null;
});
ourLog.info("Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", valueSet.getId(), valueSet.getUrl(), accumulator.getConceptsSaved(), sw.toString());
} catch (Exception e) {
ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e);
txTemplate.execute(t -> {

12
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImpl.java
View File

@ -141,7 +141,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
int codeCount = 0, relCount = 0;
StopWatch stopwatch = new StopWatch();
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myDeferredConcepts.size());
int count = Math.min(1000, myDeferredConcepts.size());
ourLog.info("Saving {} deferred concepts...", count);
while (codeCount < count && myDeferredConcepts.size() > 0) {
TermConcept next = myDeferredConcepts.remove(0);
@ -164,7 +164,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
}
if (codeCount == 0) {
count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptLinksToSaveLater.size());
count = Math.min(1000, myConceptLinksToSaveLater.size());
ourLog.info("Saving {} deferred concept relationships...", count);
while (relCount < count && myConceptLinksToSaveLater.size() > 0) {
TermConceptParentChildLink next = myConceptLinksToSaveLater.remove(0);
@ -193,7 +193,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
}
private void processDeferredValueSets() {
int count = Math.min(myDeferredValueSets.size(), 20);
int count = Math.min(myDeferredValueSets.size(), 200);
for (ValueSet nextValueSet : new ArrayList<>(myDeferredValueSets.subList(0, count))) {
ourLog.info("Creating ValueSet: {}", nextValueSet.getId());
myTerminologyVersionAdapterSvc.createOrUpdateValueSet(nextValueSet);
@ -237,6 +237,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
processDeferredConcepts();
return null;
});
continue;
}
if (isDeferredValueSets()) {
@ -244,6 +246,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
processDeferredValueSets();
return null;
});
continue;
}
if (isDeferredConceptMaps()) {
@ -251,6 +255,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
processDeferredConceptMaps();
return null;
});
continue;
}
if (isDeferredCodeSystemDeletions()) {

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java
View File

@ -48,6 +48,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
private int myConceptsSaved;
private int myDesignationsSaved;
private int myConceptsExcluded;
private int myCount;
public ValueSetConceptAccumulator(@Nonnull TermValueSet theTermValueSet, @Nonnull ITermValueSetDao theValueSetDao, @Nonnull ITermValueSetConceptDao theValueSetConceptDao, @Nonnull ITermValueSetConceptDesignationDao theValueSetConceptDesignationDao) {
myTermValueSet = theTermValueSet;
@ -171,7 +172,11 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
return true;
}
// TODO: DM 2019-07-16 - We may need TermValueSetConceptProperty, similar to TermConceptProperty.
public int getConceptsSaved() {
return myConceptsSaved;
}
// TODO: DM 2019-07-16 - We may need TermValueSetConceptProperty, similar to TermConceptProperty.
// TODO: DM 2019-07-16 - If so, we should also populate TermValueSetConceptProperty entities here.
// TODO: DM 2019-07-30 - Expansions don't include the properties themselves; they may be needed to facilitate filters and parameterized expansions.
}

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartLinkHandler.java
View File

@ -94,7 +94,7 @@ public class LoincPartLinkHandler implements IRecordHandler {
return;
}
ourLog.info("Adding new property {} = {}", propertyPart, partNumber);
ourLog.debug("Adding new property {} = {}", propertyPart, partNumber);
if (propertyType == CodeSystem.PropertyType.STRING) {
loincConcept.addPropertyString(propertyPart, partName);
} else {