honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Incremental work on large ValueSet expansion support; need pageable results while querying TRM_VALUESET_CONCEPT.

This commit is contained in:
Diederik Muylwyk 2019-08-19 18:43:43 -04:00
parent 1bc75d1c3b
commit 70aef7fbbb
6 changed files with 73 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java
View File

@ -149,7 +149,7 @@ public class DaoConfig {
/**
* EXPERIMENTAL - Do not use in production! Do not change default of {@code false}!
*/
private boolean myPreExpandValueSetsExperimental = false;
private boolean myPreExpandValueSetsExperimental = true; // FIXME: DM 2019-08-19 - Return to false;
private boolean myFilterParameterEnabled = false;
/**
* EXPERIMENTAL - Do not use in production! Do not change default of {@code 0}!

17
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java
View File

@ -194,6 +194,7 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
}
ValueSet source = new ValueSet();
source.setUrl(theUri);
source.getCompose().addInclude().addValueSet(theUri);
@ -210,16 +211,16 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
}
@Override
public ValueSet expand(ValueSet source, String theFilter) {
public ValueSet expand(ValueSet theSource, String theFilter) {
ValueSet toExpand = new ValueSet();
// for (UriType next : source.getCompose().getInclude()) {
// for (UriType next : theSource.getCompose().getInclude()) {
// ConceptSetComponent include = toExpand.getCompose().addInclude();
// include.setSystem(next.getValue());
// addFilterIfPresent(theFilter, include);
// }
for (ConceptSetComponent next : source.getCompose().getInclude()) {
for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
toExpand.getCompose().addInclude(next);
addFilterIfPresent(theFilter, next);
}
@ -228,7 +229,7 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
throw new InvalidRequestException("ValueSet does not have any compose.include or compose.import values, can not expand");
}
toExpand.getCompose().getExclude().addAll(source.getCompose().getExclude());
toExpand.getCompose().getExclude().addAll(theSource.getCompose().getExclude());
ValueSet retVal = doExpand(toExpand);
@ -240,10 +241,12 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
}
@Override
public ValueSet expand(ValueSet source, String theFilter, int theOffset, int theCount) {
public ValueSet expand(ValueSet theSource, String theFilter, int theOffset, int theCount) {
ValueSet toExpand = new ValueSet();
toExpand.setId(theSource.getId());
toExpand.setUrl(theSource.getUrl());
for (ConceptSetComponent next : source.getCompose().getInclude()) {
for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
toExpand.getCompose().addInclude(next);
addFilterIfPresent(theFilter, next);
}
@ -252,7 +255,7 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
throw new InvalidRequestException("ValueSet does not have any compose.include or compose.import values, can not expand");
}
toExpand.getCompose().getExclude().addAll(source.getCompose().getExclude());
toExpand.getCompose().getExclude().addAll(theSource.getCompose().getExclude());
ValueSet retVal = doExpand(toExpand, theOffset, theCount);

17
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java
View File

@ -190,6 +190,7 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4<ValueSet> imple
}
ValueSet source = new ValueSet();
source.setUrl(theUri);
source.getCompose().addInclude().addValueSet(theUri);
@ -206,16 +207,16 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4<ValueSet> imple
}
@Override
public ValueSet expand(ValueSet source, String theFilter) {
public ValueSet expand(ValueSet theSource, String theFilter) {
ValueSet toExpand = new ValueSet();
// for (UriType next : source.getCompose().getInclude()) {
// for (UriType next : theSource.getCompose().getInclude()) {
// ConceptSetComponent include = toExpand.getCompose().addInclude();
// include.setSystem(next.getValue());
// addFilterIfPresent(theFilter, include);
// }
for (ConceptSetComponent next : source.getCompose().getInclude()) {
for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
toExpand.getCompose().addInclude(next);
addFilterIfPresent(theFilter, next);
}
@ -224,7 +225,7 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4<ValueSet> imple
throw new InvalidRequestException("ValueSet does not have any compose.include or compose.import values, can not expand");
}
toExpand.getCompose().getExclude().addAll(source.getCompose().getExclude());
toExpand.getCompose().getExclude().addAll(theSource.getCompose().getExclude());
ValueSet retVal = doExpand(toExpand);
@ -236,10 +237,12 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4<ValueSet> imple
}
@Override
public ValueSet expand(ValueSet source, String theFilter, int theOffset, int theCount) {
public ValueSet expand(ValueSet theSource, String theFilter, int theOffset, int theCount) {
ValueSet toExpand = new ValueSet();
toExpand.setId(theSource.getId());
toExpand.setUrl(theSource.getUrl());
for (ConceptSetComponent next : source.getCompose().getInclude()) {
for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
toExpand.getCompose().addInclude(next);
addFilterIfPresent(theFilter, next);
}
@ -248,7 +251,7 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4<ValueSet> imple
throw new InvalidRequestException("ValueSet does not have any compose.include or compose.import values, can not expand");
}
toExpand.getCompose().getExclude().addAll(source.getCompose().getExclude());
toExpand.getCompose().getExclude().addAll(theSource.getCompose().getExclude());
ValueSet retVal = doExpand(toExpand, theOffset, theCount);

17
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoValueSetR5.java
View File

@ -196,6 +196,7 @@ public class FhirResourceDaoValueSetR5 extends FhirResourceDaoR5<ValueSet> imple
}
ValueSet source = new ValueSet();
source.setUrl(theUri);
source.getCompose().addInclude().addValueSet(theUri);
@ -212,16 +213,16 @@ public class FhirResourceDaoValueSetR5 extends FhirResourceDaoR5<ValueSet> imple
}
@Override
public ValueSet expand(ValueSet source, String theFilter) {
public ValueSet expand(ValueSet theSource, String theFilter) {
ValueSet toExpand = new ValueSet();
// for (UriType next : source.getCompose().getInclude()) {
// for (UriType next : theSource.getCompose().getInclude()) {
// ConceptSetComponent include = toExpand.getCompose().addInclude();
// include.setSystem(next.getValue());
// addFilterIfPresent(theFilter, include);
// }
for (ConceptSetComponent next : source.getCompose().getInclude()) {
for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
toExpand.getCompose().addInclude(next);
addFilterIfPresent(theFilter, next);
}
@ -230,7 +231,7 @@ public class FhirResourceDaoValueSetR5 extends FhirResourceDaoR5<ValueSet> imple
throw new InvalidRequestException("ValueSet does not have any compose.include or compose.import values, can not expand");
}
toExpand.getCompose().getExclude().addAll(source.getCompose().getExclude());
toExpand.getCompose().getExclude().addAll(theSource.getCompose().getExclude());
ValueSet retVal = doExpand(toExpand);
@ -242,10 +243,12 @@ public class FhirResourceDaoValueSetR5 extends FhirResourceDaoR5<ValueSet> imple
}
@Override
public ValueSet expand(ValueSet source, String theFilter, int theOffset, int theCount) {
public ValueSet expand(ValueSet theSource, String theFilter, int theOffset, int theCount) {
ValueSet toExpand = new ValueSet();
toExpand.setId(theSource.getId());
toExpand.setUrl(theSource.getUrl());
for (ConceptSetComponent next : source.getCompose().getInclude()) {
for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
toExpand.getCompose().addInclude(next);
addFilterIfPresent(theFilter, next);
}
@ -254,7 +257,7 @@ public class FhirResourceDaoValueSetR5 extends FhirResourceDaoR5<ValueSet> imple
throw new InvalidRequestException("ValueSet does not have any compose.include or compose.import values, can not expand");
}
toExpand.getCompose().getExclude().addAll(source.getCompose().getExclude());
toExpand.getCompose().getExclude().addAll(theSource.getCompose().getExclude());
ValueSet retVal = doExpand(toExpand, theOffset, theCount);

48
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java
View File

@ -482,10 +482,17 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
@Transactional(propagation = Propagation.REQUIRED)
public ValueSet expandValueSet(ValueSet theValueSetToExpand, int theOffset, int theCount) {
ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null");
ValidateUtil.isTrueOrThrowInvalidRequest(theValueSetToExpand.hasUrl(), "ValueSet to be expanded must provide ValueSet.url", theValueSetToExpand);
ValidateUtil.isNotBlankOrThrowUnprocessableEntity(theValueSetToExpand.getUrl(), theValueSetToExpand.getIdElement().toUnqualifiedVersionless().getValue() + " to be expanded must provide ValueSet.url");
Optional<TermValueSet> optionalTermValueSet = myValueSetDao.findByUrl(theValueSetToExpand.getUrl());
Optional<TermValueSet> optionalTermValueSet;
// FIXME: DM 2019-08-19 - This is no good. I need pageable results while querying TRM_VALUESET_CONCEPT
if (theValueSetToExpand.hasId()) {
optionalTermValueSet = myValueSetDao.findByResourcePid(theValueSetToExpand.getIdElement().getIdPartAsLong());
} else if (theValueSetToExpand.hasUrl()) {
optionalTermValueSet = myValueSetDao.findByUrl(theValueSetToExpand.getUrl());
} else {
throw new UnprocessableEntityException("ValueSet to be expanded must provide either ValueSet.id or ValueSet.url");
}
if (!optionalTermValueSet.isPresent()) {
throw new InvalidRequestException("ValueSet is not present in terminology tables: " + theValueSetToExpand.getUrl());
}
@ -705,8 +712,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
*/
FullTextQuery jpaQuery = em.createFullTextQuery(luceneQuery, TermConcept.class);
int maxResult = 50000;
jpaQuery.setMaxResults(maxResult);
StopWatch sw = new StopWatch();
AtomicInteger count = new AtomicInteger(0);
@ -717,11 +722,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, theCodeCounter);
}
if (maxResult == count.get()) {
throw new InternalErrorException("Expansion fragment produced too many (>= " + maxResult + ") results");
}
ourLog.info("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis());
} else {
@ -1554,25 +1554,31 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
ourLog.info("Done storing TermConceptMap.");
}
@Scheduled(fixedDelay = 600000) // 10 minutes.
// @Scheduled(fixedDelay = 600000) // 10 minutes.
@Scheduled(fixedDelay = 60000) // FIXME: DM 2019-08-19 - Remove this!
@Override
public synchronized void preExpandValueSetToTerminologyTables() {
new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus theStatus) {
Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded();
if (optionalTermValueSet.isPresent()) {
TermValueSet termValueSet = optionalTermValueSet.get();
termValueSet.setExpansionStatus(TermValueSetExpansionStatusEnum.EXPANSION_IN_PROGRESS);
myValueSetDao.saveAndFlush(termValueSet);
boolean hasNextTermValueSetNotExpanded = true;
do {
Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded();
if (optionalTermValueSet.isPresent()) {
TermValueSet termValueSet = optionalTermValueSet.get();
termValueSet.setExpansionStatus(TermValueSetExpansionStatusEnum.EXPANSION_IN_PROGRESS);
myValueSetDao.saveAndFlush(termValueSet);
ValueSet valueSet = getValueSetFromResourceTable(termValueSet.getResource());
ValueSet valueSet = getValueSetFromResourceTable(termValueSet.getResource());
expandValueSet(valueSet, new ValueSetConceptAccumulator(termValueSet, myValueSetConceptDao, myValueSetConceptDesignationDao));
expandValueSet(valueSet, new ValueSetConceptAccumulator(termValueSet, myValueSetConceptDao, myValueSetConceptDesignationDao));
termValueSet.setExpansionStatus(TermValueSetExpansionStatusEnum.EXPANDED);
myValueSetDao.saveAndFlush(termValueSet);
}
termValueSet.setExpansionStatus(TermValueSetExpansionStatusEnum.EXPANDED);
myValueSetDao.saveAndFlush(termValueSet);
} else {
hasNextTermValueSetNotExpanded = false;
}
} while (hasNextTermValueSetNotExpanded);
}
});
}

15
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithConceptAccumulator.java
View File

@ -22,15 +22,23 @@ package ca.uhn.fhir.jpa.term;
import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
import ca.uhn.fhir.model.api.annotation.Block;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import org.hl7.fhir.r4.model.ValueSet;
import java.util.Collection;
@Block()
public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.ValueSetExpansionComponent implements IValueSetConceptAccumulator {
private final int myMaxResults = 50000;
private int myConceptsCount;
public ValueSetExpansionComponentWithConceptAccumulator() {
myConceptsCount = 0;
}
@Override
public void includeConcept(String theSystem, String theCode, String theDisplay) {
incrementConceptsCount();
ValueSet.ValueSetExpansionContainsComponent contains = this.addContains();
contains.setSystem(theSystem);
contains.setCode(theCode);
@ -39,6 +47,7 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
@Override
public void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, Collection<TermConceptDesignation> theDesignations) {
incrementConceptsCount();
ValueSet.ValueSetExpansionContainsComponent contains = this.addContains();
contains.setSystem(theSystem);
contains.setCode(theCode);
@ -65,4 +74,10 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
theSystem.equals(t.getSystem()) &&
theCode.equals(t.getCode()));
}
private void incrementConceptsCount() {
if (++myConceptsCount > myMaxResults) {
throw new InternalErrorException("Expansion produced too many (>= " + myMaxResults + ") results");
}
}
}