honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

One more deadlock

This commit is contained in:
James Agnew 2017-07-20 11:19:53 -04:00
parent 9d08e1e211
commit 82171da0cc
1 changed files with 121 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvc.java
View File

@ -19,21 +19,10 @@ package ca.uhn.fhir.jpa.term;
* limitations under the License. * limitations under the License.
* #L% * #L%
*/ */
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import javax.persistence.EntityManager; import javax.persistence.*;
import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceContextType;
import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.time.DateUtils; import org.apache.commons.lang3.time.DateUtils;
@ -51,19 +40,12 @@ import org.springframework.transaction.support.TransactionTemplate;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Stopwatch; import com.google.common.base.Stopwatch;
import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimaps;
import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao; import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao; import ca.uhn.fhir.jpa.dao.data.*;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao; import ca.uhn.fhir.jpa.entity.*;
import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum; import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
import ca.uhn.fhir.jpa.util.StopWatch; import ca.uhn.fhir.jpa.util.StopWatch;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
@ -76,6 +58,8 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseHapiTerminologySvc.class); private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseHapiTerminologySvc.class);
private static final Object PLACEHOLDER_OBJECT = new Object(); private static final Object PLACEHOLDER_OBJECT = new Object();
private ArrayListMultimap<Long, Long> myChildToParentPidCache;
@Autowired @Autowired
protected ITermCodeSystemDao myCodeSystemDao; protected ITermCodeSystemDao myCodeSystemDao;
@ -100,8 +84,8 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
@PersistenceContext(type = PersistenceContextType.TRANSACTION) @PersistenceContext(type = PersistenceContextType.TRANSACTION)
protected EntityManager myEntityManager; protected EntityManager myEntityManager;
private long myNextReindexPass; private long myNextReindexPass;
private boolean myProcessDeferred = true; private boolean myProcessDeferred = true;
@Autowired @Autowired
@ -206,8 +190,11 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
/** /**
* Subclasses may override * Subclasses may override
* @param theSystem The code system *
* @param theCode The code * @param theSystem
* The code system
* @param theCode
* The code
*/ */
protected List<VersionIndependentConcept> findCodesAboveUsingBuiltInSystems(String theSystem, String theCode) { protected List<VersionIndependentConcept> findCodesAboveUsingBuiltInSystems(String theSystem, String theCode) {
return Collections.emptyList(); return Collections.emptyList();
@ -244,10 +231,14 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
ArrayList<VersionIndependentConcept> retVal = toVersionIndependentConcepts(theSystem, codes); ArrayList<VersionIndependentConcept> retVal = toVersionIndependentConcepts(theSystem, codes);
return retVal; return retVal;
} }
/** /**
* Subclasses may override * Subclasses may override
* @param theSystem The code system *
* @param theCode The code * @param theSystem
* The code system
* @param theCode
* The code
*/ */
protected List<VersionIndependentConcept> findCodesBelowUsingBuiltInSystems(String theSystem, String theCode) { protected List<VersionIndependentConcept> findCodesBelowUsingBuiltInSystems(String theSystem, String theCode) {
return Collections.emptyList(); return Collections.emptyList();
@ -274,7 +265,7 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
if (theConceptsStack.size() == 1 || theConceptsStack.size() % 10000 == 0) { if (theConceptsStack.size() == 1 || theConceptsStack.size() % 10000 == 0) {
float pct = (float) theConceptsStack.size() / (float) theTotalConcepts; float pct = (float) theConceptsStack.size() / (float) theTotalConcepts;
ourLog.info("Have processed {}/{} concepts ({}%)", theConceptsStack.size(), theTotalConcepts, (int)( pct*100.0f)); ourLog.info("Have processed {}/{} concepts ({}%)", theConceptsStack.size(), theTotalConcepts, (int) (pct * 100.0f));
} }
theConcept.setCodeSystem(theCodeSystem); theConcept.setCodeSystem(theCodeSystem);
@ -310,7 +301,47 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
} }
} }
private ArrayListMultimap<Long, Long> myChildToParentPidCache; private void processDeferredConcepts() {
int codeCount = 0, relCount = 0;
StopWatch stopwatch = new StopWatch();
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptsToSaveLater.size());
ourLog.info("Saving {} deferred concepts...", count);
while (codeCount < count && myConceptsToSaveLater.size() > 0) {
TermConcept next = myConceptsToSaveLater.remove(0);
codeCount += saveConcept(next);
}
if (codeCount > 0) {
ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)",
new Object[] { codeCount, myConceptsToSaveLater.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount) });
}
if (codeCount == 0) {
count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptLinksToSaveLater.size());
ourLog.info("Saving {} deferred concept relationships...", count);
while (relCount < count && myConceptLinksToSaveLater.size() > 0) {
TermConceptParentChildLink next = myConceptLinksToSaveLater.remove(0);
if (myConceptDao.findOne(next.getChild().getId()) == null || myConceptDao.findOne(next.getParent().getId()) == null) {
ourLog.warn("Not inserting link from child {} to parent {} because it appears to have been deleted", next.getParent().getCode(), next.getChild().getCode());
continue;
}
saveConceptLink(next);
relCount++;
}
}
if (relCount > 0) {
ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({}ms / code)",
new Object[] { relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount) });
}
if ((myConceptsToSaveLater.size() + myConceptLinksToSaveLater.size()) == 0) {
ourLog.info("All deferred concepts and relationships have now been synchronized to the database");
}
}
private void processReindexing() { private void processReindexing() {
if (System.currentTimeMillis() < myNextReindexPass && !ourForceSaveDeferredAlwaysForUnitTest) { if (System.currentTimeMillis() < myNextReindexPass && !ourForceSaveDeferredAlwaysForUnitTest) {
@ -320,6 +351,32 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
TransactionTemplate tt = new TransactionTemplate(myTransactionMgr); TransactionTemplate tt = new TransactionTemplate(myTransactionMgr);
tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW); tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
tt.execute(new TransactionCallbackWithoutResult() { tt.execute(new TransactionCallbackWithoutResult() {
private void createParentsString(StringBuilder theParentsBuilder, Long theConceptPid) {
Validate.notNull(theConceptPid, "theConceptPid must not be null");
List<Long> parents = myChildToParentPidCache.get(theConceptPid);
if (parents.contains(-1L)) {
return;
} else if (parents.isEmpty()) {
Collection<TermConceptParentChildLink> parentLinks = myConceptParentChildLinkDao.findAllWithChild(theConceptPid);
if (parentLinks.isEmpty()) {
myChildToParentPidCache.put(theConceptPid, -1L);
return;
} else {
for (TermConceptParentChildLink next : parentLinks) {
myChildToParentPidCache.put(theConceptPid, next.getParentPid());
}
}
}
for (Long nextParent : parents) {
if (theParentsBuilder.length() > 0) {
theParentsBuilder.append(' ');
}
theParentsBuilder.append(nextParent);
createParentsString(theParentsBuilder, nextParent);
}
}
@Override @Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) { protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
int maxResult = 1000; int maxResult = 1000;
@ -351,33 +408,6 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", new Object[] { count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count) }); ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", new Object[] { count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count) });
} }
private void createParentsString(StringBuilder theParentsBuilder, Long theConceptPid) {
Validate.notNull(theConceptPid, "theConceptPid must not be null");
List<Long> parents = myChildToParentPidCache.get(theConceptPid);
if (parents.contains(-1L)) {
return;
} else if (parents.isEmpty()) {
Collection<TermConceptParentChildLink> parentLinks = myConceptParentChildLinkDao.findAllWithChild(theConceptPid);
if (parentLinks.isEmpty()) {
myChildToParentPidCache.put(theConceptPid, -1L);
return;
} else {
for (TermConceptParentChildLink next : parentLinks) {
myChildToParentPidCache.put(theConceptPid, next.getParentPid());
}
}
}
for (Long nextParent : parents) {
if (theParentsBuilder.length() > 0) {
theParentsBuilder.append(' ');
}
theParentsBuilder.append(nextParent);
createParentsString(theParentsBuilder, nextParent);
}
}
}); });
} }
@ -409,8 +439,8 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
} }
} }
@Scheduled(fixedRate=5000) @Scheduled(fixedRate = 5000)
@Transactional(propagation=Propagation.REQUIRED) @Transactional(propagation = Propagation.NOT_SUPPORTED)
@Override @Override
public synchronized void saveDeferred() { public synchronized void saveDeferred() {
if (!myProcessDeferred) { if (!myProcessDeferred) {
@ -420,43 +450,15 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
return; return;
} }
int codeCount = 0, relCount = 0; TransactionTemplate tt = new TransactionTemplate(myTransactionMgr);
StopWatch stopwatch = new StopWatch(); tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
tt.execute(new TransactionCallbackWithoutResult() {
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptsToSaveLater.size()); @Override
ourLog.info("Saving {} deferred concepts...", count); protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
while (codeCount < count && myConceptsToSaveLater.size() > 0) { processDeferredConcepts();
TermConcept next = myConceptsToSaveLater.remove(0);
codeCount += saveConcept(next);
} }
if (codeCount > 0) { });
ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)", new Object[] {codeCount, myConceptsToSaveLater.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
}
if (codeCount == 0) {
count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptLinksToSaveLater.size());
ourLog.info("Saving {} deferred concept relationships...", count);
while (relCount < count && myConceptLinksToSaveLater.size() > 0) {
TermConceptParentChildLink next = myConceptLinksToSaveLater.remove(0);
if (myConceptDao.findOne(next.getChild().getId()) == null || myConceptDao.findOne(next.getParent().getId()) == null) {
ourLog.warn("Not inserting link from child {} to parent {} because it appears to have been deleted", next.getParent().getCode(), next.getChild().getCode());
continue;
}
saveConceptLink(next);
relCount++;
}
}
if (relCount > 0) {
ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({}ms / code)", new Object[] {relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
}
if ((myConceptsToSaveLater.size() + myConceptLinksToSaveLater.size()) == 0) {
ourLog.info("All deferred concepts and relationships have now been synchronized to the database");
}
} }
@Override @Override