Speed up loinc ingestion (#1991)
* Speed up loinc ingestion * Add some logging
This commit is contained in:
parent
160778d3fd
commit
a6fd6254db
|
@ -31,6 +31,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
|
|||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.orm.jpa.JpaTransactionManager;
|
||||
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
|
||||
import org.springframework.transaction.annotation.EnableTransactionManagement;
|
||||
|
@ -97,8 +98,9 @@ public class FhirServerConfigR4 extends BaseJavaConfigR4 {
|
|||
return retVal;
|
||||
}
|
||||
|
||||
@Primary
|
||||
@Bean
|
||||
public JpaTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) {
|
||||
public JpaTransactionManager hapiTransactionManager(EntityManagerFactory entityManagerFactory) {
|
||||
JpaTransactionManager retVal = new JpaTransactionManager();
|
||||
retVal.setEntityManagerFactory(entityManagerFactory);
|
||||
return retVal;
|
||||
|
|
|
@ -1649,6 +1649,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
|
|||
TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
|
||||
|
||||
while (true) {
|
||||
StopWatch sw = new StopWatch();
|
||||
TermValueSet valueSetToExpand = txTemplate.execute(t -> {
|
||||
Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded();
|
||||
if (!optionalTermValueSet.isPresent()) {
|
||||
|
@ -1669,7 +1670,9 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
|
|||
TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).get();
|
||||
return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
|
||||
});
|
||||
expandValueSet(null, valueSet, new ValueSetConceptAccumulator(valueSetToExpand, myValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao));
|
||||
|
||||
ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator(valueSetToExpand, myValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao);
|
||||
expandValueSet(null, valueSet, accumulator);
|
||||
|
||||
// We are done with this ValueSet.
|
||||
txTemplate.execute(t -> {
|
||||
|
@ -1678,6 +1681,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
|
|||
return null;
|
||||
});
|
||||
|
||||
ourLog.info("Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", valueSet.getId(), valueSet.getUrl(), accumulator.getConceptsSaved(), sw.toString());
|
||||
|
||||
} catch (Exception e) {
|
||||
ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e);
|
||||
txTemplate.execute(t -> {
|
||||
|
|
|
@ -141,7 +141,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
|
|||
int codeCount = 0, relCount = 0;
|
||||
StopWatch stopwatch = new StopWatch();
|
||||
|
||||
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myDeferredConcepts.size());
|
||||
int count = Math.min(1000, myDeferredConcepts.size());
|
||||
ourLog.info("Saving {} deferred concepts...", count);
|
||||
while (codeCount < count && myDeferredConcepts.size() > 0) {
|
||||
TermConcept next = myDeferredConcepts.remove(0);
|
||||
|
@ -164,7 +164,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
|
|||
}
|
||||
|
||||
if (codeCount == 0) {
|
||||
count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptLinksToSaveLater.size());
|
||||
count = Math.min(1000, myConceptLinksToSaveLater.size());
|
||||
ourLog.info("Saving {} deferred concept relationships...", count);
|
||||
while (relCount < count && myConceptLinksToSaveLater.size() > 0) {
|
||||
TermConceptParentChildLink next = myConceptLinksToSaveLater.remove(0);
|
||||
|
@ -193,7 +193,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
|
|||
}
|
||||
|
||||
private void processDeferredValueSets() {
|
||||
int count = Math.min(myDeferredValueSets.size(), 20);
|
||||
int count = Math.min(myDeferredValueSets.size(), 200);
|
||||
for (ValueSet nextValueSet : new ArrayList<>(myDeferredValueSets.subList(0, count))) {
|
||||
ourLog.info("Creating ValueSet: {}", nextValueSet.getId());
|
||||
myTerminologyVersionAdapterSvc.createOrUpdateValueSet(nextValueSet);
|
||||
|
@ -237,6 +237,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
|
|||
processDeferredConcepts();
|
||||
return null;
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isDeferredValueSets()) {
|
||||
|
@ -244,6 +246,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
|
|||
processDeferredValueSets();
|
||||
return null;
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isDeferredConceptMaps()) {
|
||||
|
@ -251,6 +255,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
|
|||
processDeferredConceptMaps();
|
||||
return null;
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isDeferredCodeSystemDeletions()) {
|
||||
|
|
|
@ -48,6 +48,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
|
|||
private int myConceptsSaved;
|
||||
private int myDesignationsSaved;
|
||||
private int myConceptsExcluded;
|
||||
private int myCount;
|
||||
|
||||
public ValueSetConceptAccumulator(@Nonnull TermValueSet theTermValueSet, @Nonnull ITermValueSetDao theValueSetDao, @Nonnull ITermValueSetConceptDao theValueSetConceptDao, @Nonnull ITermValueSetConceptDesignationDao theValueSetConceptDesignationDao) {
|
||||
myTermValueSet = theTermValueSet;
|
||||
|
@ -171,6 +172,10 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
|
|||
return true;
|
||||
}
|
||||
|
||||
public int getConceptsSaved() {
|
||||
return myConceptsSaved;
|
||||
}
|
||||
|
||||
// TODO: DM 2019-07-16 - We may need TermValueSetConceptProperty, similar to TermConceptProperty.
|
||||
// TODO: DM 2019-07-16 - If so, we should also populate TermValueSetConceptProperty entities here.
|
||||
// TODO: DM 2019-07-30 - Expansions don't include the properties themselves; they may be needed to facilitate filters and parameterized expansions.
|
||||
|
|
|
@ -94,7 +94,7 @@ public class LoincPartLinkHandler implements IRecordHandler {
|
|||
return;
|
||||
}
|
||||
|
||||
ourLog.info("Adding new property {} = {}", propertyPart, partNumber);
|
||||
ourLog.debug("Adding new property {} = {}", propertyPart, partNumber);
|
||||
if (propertyType == CodeSystem.PropertyType.STRING) {
|
||||
loincConcept.addPropertyString(propertyPart, partName);
|
||||
} else {
|
||||
|
|
Loading…
Reference in New Issue