honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More work on updating term services

This commit is contained in:
James Agnew 2018-04-06 17:06:34 -04:00
parent 7d6582fe73
commit f013448917
4 changed files with 78 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java
View File

@ -89,7 +89,9 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
private List<TermConceptParentChildLink> myConceptLinksToSaveLater = new ArrayList<>(); private List<TermConceptParentChildLink> myConceptLinksToSaveLater = new ArrayList<>();
@Autowired @Autowired
private ITermConceptParentChildLinkDao myConceptParentChildLinkDao; private ITermConceptParentChildLinkDao myConceptParentChildLinkDao;
private List<TermConcept> myConceptsToSaveLater = new ArrayList<>(); private List<TermConcept> myDeferredConcepts = Collections.synchronizedList(new ArrayList<>());
private List<ValueSet> myDeferredValueSets = Collections.synchronizedList(new ArrayList<>());
private List<ConceptMap> myDeferredConceptMaps = Collections.synchronizedList(new ArrayList<>());
@Autowired @Autowired
private DaoConfig myDaoConfig; private DaoConfig myDaoConfig;
private long myNextReindexPass; private long myNextReindexPass;
@ -130,7 +132,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
} }
} }
private void addDisplayFilterExact(QueryBuilder qb, BooleanJunction<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) { private void addDisplayFilterExact(QueryBuilder qb, BooleanJunction<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
bool.must(qb.phrase().onField("myDisplay").sentence(nextFilter.getValue()).createQuery()); bool.must(qb.phrase().onField("myDisplay").sentence(nextFilter.getValue()).createQuery());
} }
@ -158,11 +159,11 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
return retVal; return retVal;
} }
protected abstract IIdType createOrUpdateCodeSystem(CodeSystem theCodeSystemResource, RequestDetails theRequestDetails); protected abstract IIdType createOrUpdateCodeSystem(CodeSystem theCodeSystemResource);
protected abstract void createOrUpdateConceptMap(ConceptMap theNextConceptMap, RequestDetails theRequestDetails); protected abstract void createOrUpdateConceptMap(ConceptMap theNextConceptMap);
abstract void createOrUpdateValueSet(ValueSet theValueSet, RequestDetails theRequestDetails); abstract void createOrUpdateValueSet(ValueSet theValueSet);
@Override @Override
public void deleteCodeSystem(TermCodeSystem theCodeSystem) { public void deleteCodeSystem(TermCodeSystem theCodeSystem) {
@ -188,7 +189,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
} }
myCodeSystemVersionDao.delete(next); myCodeSystemVersionDao.delete(next);
if (i % 1000 == 0) { if (i++ % 1000 == 0) {
myEntityManager.flush(); myEntityManager.flush();
} }
} }
@ -243,12 +244,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
TermConcept code = findCode(system, nextCode); TermConcept code = findCode(system, nextCode);
if (code != null) { if (code != null) {
addCodeIfNotAlreadyAdded(system, expansionComponent, addedCodes, code); addCodeIfNotAlreadyAdded(system, expansionComponent, addedCodes, code);
//
// addedCodes.add(nextCode);
// ValueSet.ValueSetExpansionContainsComponent contains = expansionComponent.addContains();
// contains.setCode(nextCode);
// contains.setSystem(system);
// contains.setDisplay(code.getDisplay());
} }
} }
} }
@ -513,7 +508,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
if (theConceptsStack.size() <= myDaoConfig.getDeferIndexingForCodesystemsOfSize()) { if (theConceptsStack.size() <= myDaoConfig.getDeferIndexingForCodesystemsOfSize()) {
saveConcept(theConcept); saveConcept(theConcept);
} else { } else {
myConceptsToSaveLater.add(theConcept); myDeferredConcepts.add(theConcept);
} }
for (TermConceptParentChildLink next : theConcept.getChildren()) { for (TermConceptParentChildLink next : theConcept.getChildren()) {
@ -544,16 +539,16 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
int codeCount = 0, relCount = 0; int codeCount = 0, relCount = 0;
StopWatch stopwatch = new StopWatch(); StopWatch stopwatch = new StopWatch();
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptsToSaveLater.size()); int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myDeferredConcepts.size());
ourLog.info("Saving {} deferred concepts...", count); ourLog.info("Saving {} deferred concepts...", count);
while (codeCount < count && myConceptsToSaveLater.size() > 0) { while (codeCount < count && myDeferredConcepts.size() > 0) {
TermConcept next = myConceptsToSaveLater.remove(0); TermConcept next = myDeferredConcepts.remove(0);
codeCount += saveConcept(next); codeCount += saveConcept(next);
} }
if (codeCount > 0) { if (codeCount > 0) {
ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)", ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)",
new Object[] {codeCount, myConceptsToSaveLater.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)}); new Object[] {codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
} }
if (codeCount == 0) { if (codeCount == 0) {
@ -577,7 +572,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
new Object[] {relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)}); new Object[] {relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
} }
if ((myConceptsToSaveLater.size() + myConceptLinksToSaveLater.size()) == 0) { if ((myDeferredConcepts.size() + myConceptLinksToSaveLater.size()) == 0) {
ourLog.info("All deferred concepts and relationships have now been synchronized to the database"); ourLog.info("All deferred concepts and relationships have now been synchronized to the database");
} }
} }
@ -699,20 +694,51 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
public synchronized void saveDeferred() { public synchronized void saveDeferred() {
if (!myProcessDeferred) { if (!myProcessDeferred) {
return; return;
} else if (myConceptsToSaveLater.isEmpty() && myConceptLinksToSaveLater.isEmpty()) { } else if (myDeferredConcepts.isEmpty() && myConceptLinksToSaveLater.isEmpty()) {
processReindexing(); processReindexing();
return; return;
} }
TransactionTemplate tt = new TransactionTemplate(myTransactionMgr); TransactionTemplate tt = new TransactionTemplate(myTransactionMgr);
tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW); tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
tt.execute(new TransactionCallbackWithoutResult() { tt.execute(t->{
@Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
processDeferredConcepts(); processDeferredConcepts();
return null;
});
if (myDeferredValueSets.size() > 0) {
tt.execute(t -> {
processDeferredValueSets();
return null;
});
}
if (myDeferredConceptMaps.size() > 0) {
tt.execute(t -> {
processDeferredConceptMaps();
return null;
});
} }
}); }
private void processDeferredValueSets() {
int count = Math.min(myDeferredValueSets.size(), 5);
for (ValueSet nextValueSet : myDeferredValueSets.subList(0, count)) {
ourLog.info("Creating ValueSet: {}", nextValueSet.getId());
createOrUpdateValueSet(nextValueSet);
myDeferredValueSets.remove(nextValueSet);
}
ourLog.info("Saved {} deferred ValueSet resources, have {} remaining", count, myDeferredConceptMaps.size());
}
private void processDeferredConceptMaps() {
int count = Math.min(myDeferredConceptMaps.size(), 5);
for (ConceptMap nextConceptMap : myDeferredConceptMaps.subList(0, count)) {
ourLog.info("Creating ConceptMap: {}", nextConceptMap.getId());
createOrUpdateConceptMap(nextConceptMap);
myDeferredConceptMaps.remove(nextConceptMap);
}
ourLog.info("Saved {} deferred ConceptMap resources, have {} remaining", count, myDeferredConceptMaps.size());
} }
@Override @Override
@ -815,8 +841,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
ourLog.info("Done deleting old code system versions"); ourLog.info("Done deleting old code system versions");
if (myConceptsToSaveLater.size() > 0 || myConceptLinksToSaveLater.size() > 0) { if (myDeferredConcepts.size() > 0 || myConceptLinksToSaveLater.size() > 0) {
ourLog.info("Note that some concept saving was deferred - still have {} concepts and {} relationships", myConceptsToSaveLater.size(), myConceptLinksToSaveLater.size()); ourLog.info("Note that some concept saving was deferred - still have {} concepts and {} relationships", myDeferredConcepts.size(), myConceptLinksToSaveLater.size());
} }
} }
@ -825,7 +851,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
public void storeNewCodeSystemVersion(CodeSystem theCodeSystemResource, TermCodeSystemVersion theCodeSystemVersion, RequestDetails theRequestDetails, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) { public void storeNewCodeSystemVersion(CodeSystem theCodeSystemResource, TermCodeSystemVersion theCodeSystemVersion, RequestDetails theRequestDetails, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) {
Validate.notBlank(theCodeSystemResource.getUrl(), "theCodeSystemResource must have a URL"); Validate.notBlank(theCodeSystemResource.getUrl(), "theCodeSystemResource must have a URL");
IIdType csId = createOrUpdateCodeSystem(theCodeSystemResource, theRequestDetails); IIdType csId = createOrUpdateCodeSystem(theCodeSystemResource);
ResourceTable resource = (ResourceTable) myCodeSystemResourceDao.readEntity(csId); ResourceTable resource = (ResourceTable) myCodeSystemResourceDao.readEntity(csId);
Long codeSystemResourcePid = resource.getId(); Long codeSystemResourcePid = resource.getId();
@ -835,16 +861,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
theCodeSystemVersion.setResource(resource); theCodeSystemVersion.setResource(resource);
storeNewCodeSystemVersion(codeSystemResourcePid, theCodeSystemResource.getUrl(), theCodeSystemResource.getName(), theCodeSystemVersion); storeNewCodeSystemVersion(codeSystemResourcePid, theCodeSystemResource.getUrl(), theCodeSystemResource.getName(), theCodeSystemVersion);
for (ValueSet nextValueSet : theValueSets) { myDeferredConceptMaps.addAll(theConceptMaps);
ourLog.info("Creating ValueSet: {}", nextValueSet.getId()); myDeferredValueSets.addAll(theValueSets);
createOrUpdateValueSet(nextValueSet, theRequestDetails);
}
for (ConceptMap nextConceptMap : theConceptMaps) {
ourLog.info("Creating ConceptMap: {}", nextConceptMap.getId());
createOrUpdateConceptMap(nextConceptMap, theRequestDetails);
}
} }
@Override @Override

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java
View File

@ -20,7 +20,6 @@ package ca.uhn.fhir.jpa.term;
* #L% * #L%
*/ */
import ca.uhn.fhir.rest.api.server.RequestDetails;
import org.hl7.fhir.instance.hapi.validation.IValidationSupport; import org.hl7.fhir.instance.hapi.validation.IValidationSupport;
import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.CodeSystem;
@ -62,17 +61,17 @@ public class HapiTerminologySvcDstu2 extends BaseHapiTerminologySvcImpl {
} }
@Override @Override
protected IIdType createOrUpdateCodeSystem(CodeSystem theCodeSystemResource, RequestDetails theRequestDetails) { protected IIdType createOrUpdateCodeSystem(CodeSystem theCodeSystemResource) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected void createOrUpdateConceptMap(ConceptMap theNextConceptMap, RequestDetails theRequestDetails) { protected void createOrUpdateConceptMap(ConceptMap theNextConceptMap) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected void createOrUpdateValueSet(ValueSet theValueSet, RequestDetails theRequestDetails) { protected void createOrUpdateValueSet(ValueSet theValueSet) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }

19
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java
View File

@ -5,7 +5,6 @@ import ca.uhn.fhir.jpa.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem; import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao; import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
import ca.uhn.fhir.jpa.entity.TermConcept; import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.util.CoverageIgnore; import ca.uhn.fhir.util.CoverageIgnore;
import ca.uhn.fhir.util.UrlUtil; import ca.uhn.fhir.util.UrlUtil;
@ -104,7 +103,7 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
} }
@Override @Override
protected IIdType createOrUpdateCodeSystem(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource, RequestDetails theRequestDetails) { protected IIdType createOrUpdateCodeSystem(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource) {
CodeSystem resourceToStore; CodeSystem resourceToStore;
try { try {
resourceToStore = VersionConvertor_30_40.convertCodeSystem(theCodeSystemResource); resourceToStore = VersionConvertor_30_40.convertCodeSystem(theCodeSystemResource);
@ -113,14 +112,14 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
} }
if (isBlank(resourceToStore.getIdElement().getIdPart())) { if (isBlank(resourceToStore.getIdElement().getIdPart())) {
String matchUrl = "CodeSystem?url=" + UrlUtil.escapeUrlParam(theCodeSystemResource.getUrl()); String matchUrl = "CodeSystem?url=" + UrlUtil.escapeUrlParam(theCodeSystemResource.getUrl());
return myCodeSystemResourceDao.update(resourceToStore, matchUrl, theRequestDetails).getId(); return myCodeSystemResourceDao.update(resourceToStore, matchUrl).getId();
} else { } else {
return myCodeSystemResourceDao.update(resourceToStore, theRequestDetails).getId(); return myCodeSystemResourceDao.update(resourceToStore).getId();
} }
} }
@Override @Override
protected void createOrUpdateConceptMap(org.hl7.fhir.r4.model.ConceptMap theConceptMap, RequestDetails theRequestDetails) { protected void createOrUpdateConceptMap(org.hl7.fhir.r4.model.ConceptMap theConceptMap) {
ConceptMap resourceToStore; ConceptMap resourceToStore;
try { try {
resourceToStore = VersionConvertor_30_40.convertConceptMap(theConceptMap); resourceToStore = VersionConvertor_30_40.convertConceptMap(theConceptMap);
@ -129,14 +128,14 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
} }
if (isBlank(resourceToStore.getIdElement().getIdPart())) { if (isBlank(resourceToStore.getIdElement().getIdPart())) {
String matchUrl = "ConceptMap?url=" + UrlUtil.escapeUrlParam(theConceptMap.getUrl()); String matchUrl = "ConceptMap?url=" + UrlUtil.escapeUrlParam(theConceptMap.getUrl());
myConceptMapResourceDao.update(resourceToStore, matchUrl, theRequestDetails).getId(); myConceptMapResourceDao.update(resourceToStore, matchUrl);
} else { } else {
myConceptMapResourceDao.update(resourceToStore, theRequestDetails).getId(); myConceptMapResourceDao.update(resourceToStore);
} }
} }
@Override @Override
protected void createOrUpdateValueSet(org.hl7.fhir.r4.model.ValueSet theValueSet, RequestDetails theRequestDetails) { protected void createOrUpdateValueSet(org.hl7.fhir.r4.model.ValueSet theValueSet) {
ValueSet valueSetDstu3; ValueSet valueSetDstu3;
try { try {
valueSetDstu3 = VersionConvertor_30_40.convertValueSet(theValueSet); valueSetDstu3 = VersionConvertor_30_40.convertValueSet(theValueSet);
@ -146,9 +145,9 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
if (isBlank(valueSetDstu3.getIdElement().getIdPart())) { if (isBlank(valueSetDstu3.getIdElement().getIdPart())) {
String matchUrl = "ValueSet?url=" + UrlUtil.escapeUrlParam(theValueSet.getUrl()); String matchUrl = "ValueSet?url=" + UrlUtil.escapeUrlParam(theValueSet.getUrl());
myValueSetResourceDao.update(valueSetDstu3, matchUrl, theRequestDetails); myValueSetResourceDao.update(valueSetDstu3, matchUrl);
} else { } else {
myValueSetResourceDao.update(valueSetDstu3, theRequestDetails); myValueSetResourceDao.update(valueSetDstu3);
} }
} }

19
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java
View File

@ -4,7 +4,6 @@ import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao; import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
import ca.uhn.fhir.jpa.entity.TermConcept; import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.util.CoverageIgnore; import ca.uhn.fhir.util.CoverageIgnore;
import ca.uhn.fhir.util.UrlUtil; import ca.uhn.fhir.util.UrlUtil;
import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IBaseResource;
@ -96,32 +95,32 @@ public class HapiTerminologySvcR4 extends BaseHapiTerminologySvcImpl implements
} }
@Override @Override
protected IIdType createOrUpdateCodeSystem(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource, RequestDetails theRequestDetails) { protected IIdType createOrUpdateCodeSystem(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource) {
if (isBlank(theCodeSystemResource.getIdElement().getIdPart())) { if (isBlank(theCodeSystemResource.getIdElement().getIdPart())) {
String matchUrl = "CodeSystem?url=" + UrlUtil.escapeUrlParam(theCodeSystemResource.getUrl()); String matchUrl = "CodeSystem?url=" + UrlUtil.escapeUrlParam(theCodeSystemResource.getUrl());
return myCodeSystemResourceDao.update(theCodeSystemResource, matchUrl, theRequestDetails).getId(); return myCodeSystemResourceDao.update(theCodeSystemResource, matchUrl).getId();
} else { } else {
return myCodeSystemResourceDao.update(theCodeSystemResource, theRequestDetails).getId(); return myCodeSystemResourceDao.update(theCodeSystemResource).getId();
} }
} }
@Override @Override
protected void createOrUpdateConceptMap(org.hl7.fhir.r4.model.ConceptMap theConceptMap, RequestDetails theRequestDetails) { protected void createOrUpdateConceptMap(org.hl7.fhir.r4.model.ConceptMap theConceptMap) {
if (isBlank(theConceptMap.getIdElement().getIdPart())) { if (isBlank(theConceptMap.getIdElement().getIdPart())) {
String matchUrl = "ConceptMap?url=" + UrlUtil.escapeUrlParam(theConceptMap.getUrl()); String matchUrl = "ConceptMap?url=" + UrlUtil.escapeUrlParam(theConceptMap.getUrl());
myConceptMapResourceDao.update(theConceptMap, matchUrl, theRequestDetails).getId(); myConceptMapResourceDao.update(theConceptMap, matchUrl);
} else { } else {
myConceptMapResourceDao.update(theConceptMap, theRequestDetails).getId(); myConceptMapResourceDao.update(theConceptMap);
} }
} }
@Override @Override
protected void createOrUpdateValueSet(org.hl7.fhir.r4.model.ValueSet theValueSet, RequestDetails theRequestDetails) { protected void createOrUpdateValueSet(org.hl7.fhir.r4.model.ValueSet theValueSet) {
if (isBlank(theValueSet.getIdElement().getIdPart())) { if (isBlank(theValueSet.getIdElement().getIdPart())) {
String matchUrl = "ValueSet?url=" + UrlUtil.escapeUrlParam(theValueSet.getUrl()); String matchUrl = "ValueSet?url=" + UrlUtil.escapeUrlParam(theValueSet.getUrl());
myValueSetResourceDao.update(theValueSet, matchUrl, theRequestDetails); myValueSetResourceDao.update(theValueSet, matchUrl);
} else { } else {
myValueSetResourceDao.update(theValueSet, theRequestDetails); myValueSetResourceDao.update(theValueSet);
} }
} }