honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Completed work on large ValueSet expansion support; ready for review.

This commit is contained in:
Diederik Muylwyk 2019-08-23 17:40:45 -04:00
parent 10958a8e4d
commit b41926307c
6 changed files with 56 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hapi-fhir-base/src/main/resources/ca/uhn/fhir/i18n/hapi-messages.properties
View File

@ -134,3 +134,4 @@ ca.uhn.fhir.jpa.util.jsonpatch.JsonPatchUtils.failedToApplyPatch=Failed to apply
ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum.notExpanded=The ValueSet is waiting to be picked up and pre-expanded by a scheduled task.
ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum.expansionInProgress=The ValueSet has been picked up by a scheduled task and pre-expansion is in progress.
ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum.expanded=The ValueSet has been picked up by a scheduled task and pre-expansion is complete.
ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum.failedToExpand=The ValueSet has been picked up by a scheduled task and pre-expansion has failed.

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java
View File

@ -149,7 +149,7 @@ public class DaoConfig {
/**
* EXPERIMENTAL - Do not use in production! Do not change default of {@code false}!
*/
private boolean myPreExpandValueSetsExperimental = true; // FIXME: DM 2019-08-19 - Return to false;
private boolean myPreExpandValueSetsExperimental = false;
private boolean myFilterParameterEnabled = false;
private StoreMetaSourceInformation myStoreMetaSourceInformation = StoreMetaSourceInformation.SOURCE_URI_AND_REQUEST_ID;
/**

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetPreExpansionStatusEnum.java
View File

@ -32,7 +32,7 @@ public enum TermValueSetPreExpansionStatusEnum {
/**
* Sorting agnostic.
*/
// FIXME: add a unit test that verifies a message exists for each code
NOT_EXPANDED("notExpanded"),
EXPANSION_IN_PROGRESS("expansionInProgress"),
EXPANDED("expanded"),
@ -41,7 +41,7 @@ public enum TermValueSetPreExpansionStatusEnum {
private static Map<String, TermValueSetPreExpansionStatusEnum> ourValues;
private String myCode;
private TermValueSetPreExpansionStatusEnum(String theCode) {
TermValueSetPreExpansionStatusEnum(String theCode) {
myCode = theCode;
}

51
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java
View File

@ -351,7 +351,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
if (optionalExistingTermConceptMapById.isPresent()) {
TermConceptMap existingTermConceptMap = optionalExistingTermConceptMapById.get();
ourLog.info("Deleting existing TermConceptMap {} and its children...", existingTermConceptMap.getId());
ourLog.info("Deleting existing TermConceptMap[{}] and its children...", existingTermConceptMap.getId());
for (TermConceptMapGroup group : existingTermConceptMap.getConceptMapGroups()) {
for (TermConceptMapGroupElement element : group.getConceptMapGroupElements()) {
@ -368,7 +368,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
}
myConceptMapDao.deleteTermConceptMapById(existingTermConceptMap.getId());
ourLog.info("Done deleting existing TermConceptMap {} and its children.", existingTermConceptMap.getId());
ourLog.info("Done deleting existing TermConceptMap[{}] and its children.", existingTermConceptMap.getId());
ourLog.info("Flushing...");
myConceptMapGroupElementTargetDao.flush();
@ -392,11 +392,11 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
if (optionalExistingTermValueSetById.isPresent()) {
TermValueSet existingTermValueSet = optionalExistingTermValueSetById.get();
ourLog.info("Deleting existing TermValueSet {} and its children...", existingTermValueSet.getId());
ourLog.info("Deleting existing TermValueSet[{}] and its children...", existingTermValueSet.getId());
myValueSetConceptDesignationDao.deleteByTermValueSetId(existingTermValueSet.getId());
myValueSetConceptDao.deleteByTermValueSetId(existingTermValueSet.getId());
myValueSetDao.deleteByTermValueSetId(existingTermValueSet.getId());
ourLog.info("Done deleting existing TermValueSet {} and its children.", existingTermValueSet.getId());
ourLog.info("Done deleting existing TermValueSet[{}] and its children.", existingTermValueSet.getId());
ourLog.info("Flushing...");
myValueSetConceptDesignationDao.flush();
@ -622,13 +622,16 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
private void expandValueSet(ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator, AtomicInteger theCodeCounter) {
Set<String> addedCodes = new HashSet<>();
StopWatch sw = new StopWatch();
String valueSetInfo = getValueSetInfo(theValueSetToExpand);
ourLog.info("Working with {}", valueSetInfo);
// Handle includes
ourLog.debug("Handling includes");
for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getInclude()) {
ourLog.info("Working with " + identifyValueSetForLogging(theValueSetToExpand));
for (int i = 0; ; i++) {
int finalI = i;
boolean shouldContinue = myTxTemplate.execute(t -> {
Boolean shouldContinue = myTxTemplate.execute(t -> {
boolean add = true;
return expandValueSetHandleIncludeOrExclude(theValueSetCodeAccumulator, addedCodes, include, add, theCodeCounter, finalI);
});
@ -641,10 +644,9 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
// Handle excludes
ourLog.debug("Handling excludes");
for (ValueSet.ConceptSetComponent exclude : theValueSetToExpand.getCompose().getExclude()) {
ourLog.info("Working with " + identifyValueSetForLogging(theValueSetToExpand));
for (int i = 0; ; i++) {
int finalI = i;
boolean shouldContinue = myTxTemplate.execute(t -> {
Boolean shouldContinue = myTxTemplate.execute(t -> {
boolean add = false;
return expandValueSetHandleIncludeOrExclude(theValueSetCodeAccumulator, addedCodes, exclude, add, theCodeCounter, finalI);
});
@ -653,9 +655,11 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
}
}
}
ourLog.info("Done working with {} in {}ms", valueSetInfo, sw.getMillis());
}
private String identifyValueSetForLogging(ValueSet theValueSet) {
private String getValueSetInfo(ValueSet theValueSet) {
StringBuilder sb = new StringBuilder();
boolean isIdentified = false;
sb
@ -703,16 +707,16 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
}
/**
* @return Returns true if there are potentially more results to return
* @return Returns true if there are potentially more results to process.
*/
private boolean expandValueSetHandleIncludeOrExclude(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, boolean theAdd, AtomicInteger theCodeCounter, int theQueryIndex) {
private Boolean expandValueSetHandleIncludeOrExclude(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, boolean theAdd, AtomicInteger theCodeCounter, int theQueryIndex) {
String system = theInclude.getSystem();
boolean hasSystem = isNotBlank(system);
boolean hasValueSet = theInclude.getValueSet().size() > 0;
if (hasSystem) {
ourLog.info("Starting {} expansion around code system: {}", (theAdd ? "inclusion" : "exclusion"), system);
ourLog.info("Starting {} expansion around CodeSystem: {}", (theAdd ? "inclusion" : "exclusion"), system);
TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system);
if (cs != null) {
@ -859,7 +863,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
AtomicInteger countForBatch = new AtomicInteger(0);
List resultList = jpaQuery.getResultList();
int resultsInBatch = jpaQuery.getResultSize();
int resultsInBatch = resultList.size();
int firstResult = jpaQuery.getFirstResult();
for (Object next : resultList) {
count.incrementAndGet();
@ -910,7 +914,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
} else if (hasValueSet) {
for (CanonicalType nextValueSet : theInclude.getValueSet()) {
ourLog.info("Starting {} expansion around ValueSet URI: {}", (theAdd ? "inclusion" : "exclusion"), nextValueSet.getValueAsString());
ourLog.info("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), nextValueSet.getValueAsString());
List<VersionIndependentConcept> expanded = expandValueSet(nextValueSet.getValueAsString());
for (VersionIndependentConcept nextConcept : expanded) {
@ -1750,8 +1754,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
ourLog.info("Done storing TermConceptMap.");
}
// @Scheduled(fixedDelay = 600000) // 10 minutes.
@Scheduled(fixedDelay = 60000) // FIXME: DM 2019-08-19 - Remove this!
@Scheduled(fixedDelay = 600000) // 10 minutes.
@Override
public synchronized void preExpandValueSetToTerminologyTables() {
if (isNotSafeToPreExpandValueSets()) {
@ -1761,10 +1764,9 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
while (true) {
TermValueSet valueSetToExpand = txTemplate.execute(t -> {
Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded();
if (optionalTermValueSet.isPresent() == false) {
if (!optionalTermValueSet.isPresent()) {
return null;
}
@ -1776,12 +1778,15 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
return;
}
// Ok so we have a VS to expand
// We have a ValueSet to pre-expand.
try {
ValueSet valueSet = txTemplate.execute(t -> getValueSetFromResourceTable(valueSetToExpand.getResource()));
ValueSet valueSet = txTemplate.execute(t -> {
TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).get();
return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
});
expandValueSet(valueSet, new ValueSetConceptAccumulator(valueSetToExpand, myValueSetConceptDao, myValueSetConceptDesignationDao));
// We're done with this guy
// We are done with this ValueSet.
txTemplate.execute(t -> {
valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANDED);
myValueSetDao.saveAndFlush(valueSetToExpand);
@ -1789,7 +1794,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
});
} catch (Exception e) {
ourLog.error("Failed to expand valueset: " + e.getMessage(), e);
ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e);
txTemplate.execute(t -> {
valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.FAILED_TO_EXPAND);
myValueSetDao.saveAndFlush(valueSetToExpand);
@ -1797,8 +1802,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
});
}
}
}
private boolean isNotSafeToPreExpandValueSets() {

10
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java
View File

@ -103,10 +103,9 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
}
myValueSetConceptDao.save(concept);
// if (myConceptsSaved++ % 2 == 0) { // FIXME: DM 2019-08-23 - Reset to 250.
if (myConceptsSaved++ % 250 == 0) { // TODO: DM 2019-08-23 - This message never appears in the log. Fix it!
ourLog.info("Have pre-expanded {} concepts in ValueSet[{}]", myConceptsSaved, myTermValueSet.getUrl());
// myValueSetConceptDao.flush();
// }
}
return concept;
}
@ -128,9 +127,8 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
designation.setValue(theDesignation.getValue());
myValueSetConceptDesignationDao.save(designation);
if (myDesignationsSaved++ % 2 == 0) { // FIXME: DM 2019-08-23 - Reset to 250.
ourLog.info("Have pre-expanded {} designations in ValueSet[{}]", myDesignationsSaved, myTermValueSet.getUrl());
myValueSetConceptDesignationDao.flush();
if (myDesignationsSaved++ % 250 == 0) { // TODO: DM 2019-08-23 - This message never appears in the log. Fix it!
ourLog.info("Have pre-expanded {} designations for Concept[{}|{}] in ValueSet[{}]", myDesignationsSaved, theConcept.getSystem(), theConcept.getCode(), myTermValueSet.getUrl());
}
return designation;

21
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/entity/TermValueSetPreExpansionStatusEnumTest.java Normal file
View File

@ -0,0 +1,21 @@
package ca.uhn.fhir.jpa.entity;
import ca.uhn.fhir.i18n.HapiLocalizer;
import org.junit.Test;
import static org.junit.Assert.fail;
public class TermValueSetPreExpansionStatusEnumTest {
@Test
public void testHaveDescriptions() {
HapiLocalizer localizer = new HapiLocalizer();
for (TermValueSetPreExpansionStatusEnum next : TermValueSetPreExpansionStatusEnum.values()) {
String key = "ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum." + next.getCode();
String msg = localizer.getMessage(key);
if (msg.equals(HapiLocalizer.UNKNOWN_I18N_KEY_MESSAGE)) {
fail("No value for key: " + key);
}
}
}
}