honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More work on updating term services

This commit is contained in:
James Agnew 2018-04-06 17:06:34 -04:00
parent 7d6582fe73
commit f013448917
4 changed files with 78 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java
View File

@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -89,7 +89,9 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
private List<TermConceptParentChildLink> myConceptLinksToSaveLater = new ArrayList<>();
@Autowired
private ITermConceptParentChildLinkDao myConceptParentChildLinkDao;
private List<TermConcept> myConceptsToSaveLater = new ArrayList<>();
private List<TermConcept> myDeferredConcepts = Collections.synchronizedList(new ArrayList<>());
private List<ValueSet> myDeferredValueSets = Collections.synchronizedList(new ArrayList<>());
private List<ConceptMap> myDeferredConceptMaps = Collections.synchronizedList(new ArrayList<>());
@Autowired
private DaoConfig myDaoConfig;
private long myNextReindexPass;
@ -130,7 +132,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
}
}
private void addDisplayFilterExact(QueryBuilder qb, BooleanJunction<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
bool.must(qb.phrase().onField("myDisplay").sentence(nextFilter.getValue()).createQuery());
}
@ -158,11 +159,11 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
return retVal;
}
protected abstract IIdType createOrUpdateCodeSystem(CodeSystem theCodeSystemResource, RequestDetails theRequestDetails);
protected abstract IIdType createOrUpdateCodeSystem(CodeSystem theCodeSystemResource);
protected abstract void createOrUpdateConceptMap(ConceptMap theNextConceptMap, RequestDetails theRequestDetails);
protected abstract void createOrUpdateConceptMap(ConceptMap theNextConceptMap);
abstract void createOrUpdateValueSet(ValueSet theValueSet, RequestDetails theRequestDetails);
abstract void createOrUpdateValueSet(ValueSet theValueSet);
@Override
public void deleteCodeSystem(TermCodeSystem theCodeSystem) {
@ -188,7 +189,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
}
myCodeSystemVersionDao.delete(next);
if (i % 1000 == 0) {
if (i++ % 1000 == 0) {
myEntityManager.flush();
}
}
@ -243,12 +244,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
TermConcept code = findCode(system, nextCode);
if (code != null) {
addCodeIfNotAlreadyAdded(system, expansionComponent, addedCodes, code);
//
// addedCodes.add(nextCode);
// ValueSet.ValueSetExpansionContainsComponent contains = expansionComponent.addContains();
// contains.setCode(nextCode);
// contains.setSystem(system);
// contains.setDisplay(code.getDisplay());
}
}
}
@ -513,7 +508,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
if (theConceptsStack.size() <= myDaoConfig.getDeferIndexingForCodesystemsOfSize()) {
saveConcept(theConcept);
} else {
myConceptsToSaveLater.add(theConcept);
myDeferredConcepts.add(theConcept);
}
for (TermConceptParentChildLink next : theConcept.getChildren()) {
@ -544,16 +539,16 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
int codeCount = 0, relCount = 0;
StopWatch stopwatch = new StopWatch();
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptsToSaveLater.size());
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myDeferredConcepts.size());
ourLog.info("Saving {} deferred concepts...", count);
while (codeCount < count && myConceptsToSaveLater.size() > 0) {
TermConcept next = myConceptsToSaveLater.remove(0);
while (codeCount < count && myDeferredConcepts.size() > 0) {
TermConcept next = myDeferredConcepts.remove(0);
codeCount += saveConcept(next);
}
if (codeCount > 0) {
ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)",
new Object[] {codeCount, myConceptsToSaveLater.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
new Object[] {codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
}
if (codeCount == 0) {
@ -577,7 +572,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
new Object[] {relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
}
if ((myConceptsToSaveLater.size() + myConceptLinksToSaveLater.size()) == 0) {
if ((myDeferredConcepts.size() + myConceptLinksToSaveLater.size()) == 0) {
ourLog.info("All deferred concepts and relationships have now been synchronized to the database");
}
}
@ -699,20 +694,51 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
public synchronized void saveDeferred() {
if (!myProcessDeferred) {
return;
} else if (myConceptsToSaveLater.isEmpty() && myConceptLinksToSaveLater.isEmpty()) {
} else if (myDeferredConcepts.isEmpty() && myConceptLinksToSaveLater.isEmpty()) {
processReindexing();
return;
}
TransactionTemplate tt = new TransactionTemplate(myTransactionMgr);
tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
tt.execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
tt.execute(t->{
processDeferredConcepts();
}
return null;
});
if (myDeferredValueSets.size() > 0) {
tt.execute(t -> {
processDeferredValueSets();
return null;
});
}
if (myDeferredConceptMaps.size() > 0) {
tt.execute(t -> {
processDeferredConceptMaps();
return null;
});
}
}
private void processDeferredValueSets() {
int count = Math.min(myDeferredValueSets.size(), 5);
for (ValueSet nextValueSet : myDeferredValueSets.subList(0, count)) {
ourLog.info("Creating ValueSet: {}", nextValueSet.getId());
createOrUpdateValueSet(nextValueSet);
myDeferredValueSets.remove(nextValueSet);
}
ourLog.info("Saved {} deferred ValueSet resources, have {} remaining", count, myDeferredConceptMaps.size());
}
private void processDeferredConceptMaps() {
int count = Math.min(myDeferredConceptMaps.size(), 5);
for (ConceptMap nextConceptMap : myDeferredConceptMaps.subList(0, count)) {
ourLog.info("Creating ConceptMap: {}", nextConceptMap.getId());
createOrUpdateConceptMap(nextConceptMap);
myDeferredConceptMaps.remove(nextConceptMap);
}
ourLog.info("Saved {} deferred ConceptMap resources, have {} remaining", count, myDeferredConceptMaps.size());
}
@Override
@ -815,8 +841,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
ourLog.info("Done deleting old code system versions");
if (myConceptsToSaveLater.size() > 0 || myConceptLinksToSaveLater.size() > 0) {
ourLog.info("Note that some concept saving was deferred - still have {} concepts and {} relationships", myConceptsToSaveLater.size(), myConceptLinksToSaveLater.size());
if (myDeferredConcepts.size() > 0 || myConceptLinksToSaveLater.size() > 0) {
ourLog.info("Note that some concept saving was deferred - still have {} concepts and {} relationships", myDeferredConcepts.size(), myConceptLinksToSaveLater.size());
}
}
@ -825,7 +851,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
public void storeNewCodeSystemVersion(CodeSystem theCodeSystemResource, TermCodeSystemVersion theCodeSystemVersion, RequestDetails theRequestDetails, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) {
Validate.notBlank(theCodeSystemResource.getUrl(), "theCodeSystemResource must have a URL");
IIdType csId = createOrUpdateCodeSystem(theCodeSystemResource, theRequestDetails);
IIdType csId = createOrUpdateCodeSystem(theCodeSystemResource);
ResourceTable resource = (ResourceTable) myCodeSystemResourceDao.readEntity(csId);
Long codeSystemResourcePid = resource.getId();
@ -835,16 +861,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
theCodeSystemVersion.setResource(resource);
storeNewCodeSystemVersion(codeSystemResourcePid, theCodeSystemResource.getUrl(), theCodeSystemResource.getName(), theCodeSystemVersion);
for (ValueSet nextValueSet : theValueSets) {
ourLog.info("Creating ValueSet: {}", nextValueSet.getId());
createOrUpdateValueSet(nextValueSet, theRequestDetails);
}
for (ConceptMap nextConceptMap : theConceptMaps) {
ourLog.info("Creating ConceptMap: {}", nextConceptMap.getId());
createOrUpdateConceptMap(nextConceptMap, theRequestDetails);
}
myDeferredConceptMaps.addAll(theConceptMaps);
myDeferredValueSets.addAll(theValueSets);
}
@Override

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java
View File

@ -20,7 +20,6 @@ package ca.uhn.fhir.jpa.term;
* #L%
*/
import ca.uhn.fhir.rest.api.server.RequestDetails;
import org.hl7.fhir.instance.hapi.validation.IValidationSupport;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.CodeSystem;
@ -62,17 +61,17 @@ public class HapiTerminologySvcDstu2 extends BaseHapiTerminologySvcImpl {
}
@Override
protected IIdType createOrUpdateCodeSystem(CodeSystem theCodeSystemResource, RequestDetails theRequestDetails) {
protected IIdType createOrUpdateCodeSystem(CodeSystem theCodeSystemResource) {
throw new UnsupportedOperationException();
}
@Override
protected void createOrUpdateConceptMap(ConceptMap theNextConceptMap, RequestDetails theRequestDetails) {
protected void createOrUpdateConceptMap(ConceptMap theNextConceptMap) {
throw new UnsupportedOperationException();
}
@Override
protected void createOrUpdateValueSet(ValueSet theValueSet, RequestDetails theRequestDetails) {
protected void createOrUpdateValueSet(ValueSet theValueSet) {
throw new UnsupportedOperationException();
}

19
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java
View File

@ -5,7 +5,6 @@ import ca.uhn.fhir.jpa.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.util.CoverageIgnore;
import ca.uhn.fhir.util.UrlUtil;
@ -104,7 +103,7 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
}
@Override
protected IIdType createOrUpdateCodeSystem(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource, RequestDetails theRequestDetails) {
protected IIdType createOrUpdateCodeSystem(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource) {
CodeSystem resourceToStore;
try {
resourceToStore = VersionConvertor_30_40.convertCodeSystem(theCodeSystemResource);
@ -113,14 +112,14 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
}
if (isBlank(resourceToStore.getIdElement().getIdPart())) {
String matchUrl = "CodeSystem?url=" + UrlUtil.escapeUrlParam(theCodeSystemResource.getUrl());
return myCodeSystemResourceDao.update(resourceToStore, matchUrl, theRequestDetails).getId();
return myCodeSystemResourceDao.update(resourceToStore, matchUrl).getId();
} else {
return myCodeSystemResourceDao.update(resourceToStore, theRequestDetails).getId();
return myCodeSystemResourceDao.update(resourceToStore).getId();
}
}
@Override
protected void createOrUpdateConceptMap(org.hl7.fhir.r4.model.ConceptMap theConceptMap, RequestDetails theRequestDetails) {
protected void createOrUpdateConceptMap(org.hl7.fhir.r4.model.ConceptMap theConceptMap) {
ConceptMap resourceToStore;
try {
resourceToStore = VersionConvertor_30_40.convertConceptMap(theConceptMap);
@ -129,14 +128,14 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
}
if (isBlank(resourceToStore.getIdElement().getIdPart())) {
String matchUrl = "ConceptMap?url=" + UrlUtil.escapeUrlParam(theConceptMap.getUrl());
myConceptMapResourceDao.update(resourceToStore, matchUrl, theRequestDetails).getId();
myConceptMapResourceDao.update(resourceToStore, matchUrl);
} else {
myConceptMapResourceDao.update(resourceToStore, theRequestDetails).getId();
myConceptMapResourceDao.update(resourceToStore);
}
}
@Override
protected void createOrUpdateValueSet(org.hl7.fhir.r4.model.ValueSet theValueSet, RequestDetails theRequestDetails) {
protected void createOrUpdateValueSet(org.hl7.fhir.r4.model.ValueSet theValueSet) {
ValueSet valueSetDstu3;
try {
valueSetDstu3 = VersionConvertor_30_40.convertValueSet(theValueSet);
@ -146,9 +145,9 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
if (isBlank(valueSetDstu3.getIdElement().getIdPart())) {
String matchUrl = "ValueSet?url=" + UrlUtil.escapeUrlParam(theValueSet.getUrl());
myValueSetResourceDao.update(valueSetDstu3, matchUrl, theRequestDetails);
myValueSetResourceDao.update(valueSetDstu3, matchUrl);
} else {
myValueSetResourceDao.update(valueSetDstu3, theRequestDetails);
myValueSetResourceDao.update(valueSetDstu3);
}
}

19
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java
View File

@ -4,7 +4,6 @@ import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.util.CoverageIgnore;
import ca.uhn.fhir.util.UrlUtil;
import org.hl7.fhir.instance.model.api.IBaseResource;
@ -96,32 +95,32 @@ public class HapiTerminologySvcR4 extends BaseHapiTerminologySvcImpl implements
}
@Override
protected IIdType createOrUpdateCodeSystem(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource, RequestDetails theRequestDetails) {
protected IIdType createOrUpdateCodeSystem(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource) {
if (isBlank(theCodeSystemResource.getIdElement().getIdPart())) {
String matchUrl = "CodeSystem?url=" + UrlUtil.escapeUrlParam(theCodeSystemResource.getUrl());
return myCodeSystemResourceDao.update(theCodeSystemResource, matchUrl, theRequestDetails).getId();
return myCodeSystemResourceDao.update(theCodeSystemResource, matchUrl).getId();
} else {
return myCodeSystemResourceDao.update(theCodeSystemResource, theRequestDetails).getId();
return myCodeSystemResourceDao.update(theCodeSystemResource).getId();
}
}
@Override
protected void createOrUpdateConceptMap(org.hl7.fhir.r4.model.ConceptMap theConceptMap, RequestDetails theRequestDetails) {
protected void createOrUpdateConceptMap(org.hl7.fhir.r4.model.ConceptMap theConceptMap) {
if (isBlank(theConceptMap.getIdElement().getIdPart())) {
String matchUrl = "ConceptMap?url=" + UrlUtil.escapeUrlParam(theConceptMap.getUrl());
myConceptMapResourceDao.update(theConceptMap, matchUrl, theRequestDetails).getId();
myConceptMapResourceDao.update(theConceptMap, matchUrl);
} else {
myConceptMapResourceDao.update(theConceptMap, theRequestDetails).getId();
myConceptMapResourceDao.update(theConceptMap);
}
}
@Override
protected void createOrUpdateValueSet(org.hl7.fhir.r4.model.ValueSet theValueSet, RequestDetails theRequestDetails) {
protected void createOrUpdateValueSet(org.hl7.fhir.r4.model.ValueSet theValueSet) {
if (isBlank(theValueSet.getIdElement().getIdPart())) {
String matchUrl = "ValueSet?url=" + UrlUtil.escapeUrlParam(theValueSet.getUrl());
myValueSetResourceDao.update(theValueSet, matchUrl, theRequestDetails);
myValueSetResourceDao.update(theValueSet, matchUrl);
} else {
myValueSetResourceDao.update(theValueSet, theRequestDetails);
myValueSetResourceDao.update(theValueSet);
}
}