One more deadlock
This commit is contained in:
James Agnew
parent
9d08e1e211
commit
82171da0cc
|
|
@ -10,7 +10,7 @@ package ca.uhn.fhir.jpa.term;
|
|||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
|
|
@ -19,21 +19,10 @@ package ca.uhn.fhir.jpa.term;
|
|||
* limitations under the License.
|
||||
* #L%
|
||||
*/
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.persistence.EntityManager;
|
||||
import javax.persistence.PersistenceContext;
|
||||
import javax.persistence.PersistenceContextType;
|
||||
import javax.persistence.*;
|
||||
|
||||
import org.apache.commons.lang3.Validate;
|
||||
import org.apache.commons.lang3.time.DateUtils;
|
||||
|
|
@ -51,19 +40,12 @@ import org.springframework.transaction.support.TransactionTemplate;
|
|||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.Multimaps;
|
||||
|
||||
import ca.uhn.fhir.context.FhirContext;
|
||||
import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
|
||||
import ca.uhn.fhir.jpa.dao.DaoConfig;
|
||||
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
|
||||
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
|
||||
import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
|
||||
import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao;
|
||||
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
|
||||
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
|
||||
import ca.uhn.fhir.jpa.entity.TermConcept;
|
||||
import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
|
||||
import ca.uhn.fhir.jpa.dao.data.*;
|
||||
import ca.uhn.fhir.jpa.entity.*;
|
||||
import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
|
||||
import ca.uhn.fhir.jpa.util.StopWatch;
|
||||
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
|
||||
|
|
@ -76,6 +58,8 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseHapiTerminologySvc.class);
|
||||
private static final Object PLACEHOLDER_OBJECT = new Object();
|
||||
|
||||
private ArrayListMultimap<Long, Long> myChildToParentPidCache;
|
||||
|
||||
@Autowired
|
||||
protected ITermCodeSystemDao myCodeSystemDao;
|
||||
|
||||
|
|
@ -100,8 +84,8 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
|
||||
@PersistenceContext(type = PersistenceContextType.TRANSACTION)
|
||||
protected EntityManager myEntityManager;
|
||||
|
||||
private long myNextReindexPass;
|
||||
|
||||
private boolean myProcessDeferred = true;
|
||||
|
||||
@Autowired
|
||||
|
|
@ -206,8 +190,11 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
|
||||
/**
|
||||
* Subclasses may override
|
||||
* @param theSystem The code system
|
||||
* @param theCode The code
|
||||
*
|
||||
* @param theSystem
|
||||
* The code system
|
||||
* @param theCode
|
||||
* The code
|
||||
*/
|
||||
protected List<VersionIndependentConcept> findCodesAboveUsingBuiltInSystems(String theSystem, String theCode) {
|
||||
return Collections.emptyList();
|
||||
|
|
@ -244,10 +231,14 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
ArrayList<VersionIndependentConcept> retVal = toVersionIndependentConcepts(theSystem, codes);
|
||||
return retVal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Subclasses may override
|
||||
* @param theSystem The code system
|
||||
* @param theCode The code
|
||||
*
|
||||
* @param theSystem
|
||||
* The code system
|
||||
* @param theCode
|
||||
* The code
|
||||
*/
|
||||
protected List<VersionIndependentConcept> findCodesBelowUsingBuiltInSystems(String theSystem, String theCode) {
|
||||
return Collections.emptyList();
|
||||
|
|
@ -274,7 +265,7 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
|
||||
if (theConceptsStack.size() == 1 || theConceptsStack.size() % 10000 == 0) {
|
||||
float pct = (float) theConceptsStack.size() / (float) theTotalConcepts;
|
||||
ourLog.info("Have processed {}/{} concepts ({}%)", theConceptsStack.size(), theTotalConcepts, (int)( pct*100.0f));
|
||||
ourLog.info("Have processed {}/{} concepts ({}%)", theConceptsStack.size(), theTotalConcepts, (int) (pct * 100.0f));
|
||||
}
|
||||
|
||||
theConcept.setCodeSystem(theCodeSystem);
|
||||
|
|
@ -310,7 +301,47 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
}
|
||||
}
|
||||
|
||||
private ArrayListMultimap<Long, Long> myChildToParentPidCache;
|
||||
private void processDeferredConcepts() {
|
||||
int codeCount = 0, relCount = 0;
|
||||
StopWatch stopwatch = new StopWatch();
|
||||
|
||||
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptsToSaveLater.size());
|
||||
ourLog.info("Saving {} deferred concepts...", count);
|
||||
while (codeCount < count && myConceptsToSaveLater.size() > 0) {
|
||||
TermConcept next = myConceptsToSaveLater.remove(0);
|
||||
codeCount += saveConcept(next);
|
||||
}
|
||||
|
||||
if (codeCount > 0) {
|
||||
ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)",
|
||||
new Object[] { codeCount, myConceptsToSaveLater.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount) });
|
||||
}
|
||||
|
||||
if (codeCount == 0) {
|
||||
count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptLinksToSaveLater.size());
|
||||
ourLog.info("Saving {} deferred concept relationships...", count);
|
||||
while (relCount < count && myConceptLinksToSaveLater.size() > 0) {
|
||||
TermConceptParentChildLink next = myConceptLinksToSaveLater.remove(0);
|
||||
|
||||
if (myConceptDao.findOne(next.getChild().getId()) == null || myConceptDao.findOne(next.getParent().getId()) == null) {
|
||||
ourLog.warn("Not inserting link from child {} to parent {} because it appears to have been deleted", next.getParent().getCode(), next.getChild().getCode());
|
||||
continue;
|
||||
}
|
||||
|
||||
saveConceptLink(next);
|
||||
relCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (relCount > 0) {
|
||||
ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({}ms / code)",
|
||||
new Object[] { relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount) });
|
||||
}
|
||||
|
||||
if ((myConceptsToSaveLater.size() + myConceptLinksToSaveLater.size()) == 0) {
|
||||
ourLog.info("All deferred concepts and relationships have now been synchronized to the database");
|
||||
}
|
||||
}
|
||||
|
||||
private void processReindexing() {
|
||||
if (System.currentTimeMillis() < myNextReindexPass && !ourForceSaveDeferredAlwaysForUnitTest) {
|
||||
|
|
@ -320,6 +351,32 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
TransactionTemplate tt = new TransactionTemplate(myTransactionMgr);
|
||||
tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
|
||||
tt.execute(new TransactionCallbackWithoutResult() {
|
||||
private void createParentsString(StringBuilder theParentsBuilder, Long theConceptPid) {
|
||||
Validate.notNull(theConceptPid, "theConceptPid must not be null");
|
||||
List<Long> parents = myChildToParentPidCache.get(theConceptPid);
|
||||
if (parents.contains(-1L)) {
|
||||
return;
|
||||
} else if (parents.isEmpty()) {
|
||||
Collection<TermConceptParentChildLink> parentLinks = myConceptParentChildLinkDao.findAllWithChild(theConceptPid);
|
||||
if (parentLinks.isEmpty()) {
|
||||
myChildToParentPidCache.put(theConceptPid, -1L);
|
||||
return;
|
||||
} else {
|
||||
for (TermConceptParentChildLink next : parentLinks) {
|
||||
myChildToParentPidCache.put(theConceptPid, next.getParentPid());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Long nextParent : parents) {
|
||||
if (theParentsBuilder.length() > 0) {
|
||||
theParentsBuilder.append(' ');
|
||||
}
|
||||
theParentsBuilder.append(nextParent);
|
||||
createParentsString(theParentsBuilder, nextParent);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
|
||||
int maxResult = 1000;
|
||||
|
|
@ -351,33 +408,6 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
|
||||
ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", new Object[] { count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count) });
|
||||
}
|
||||
|
||||
private void createParentsString(StringBuilder theParentsBuilder, Long theConceptPid) {
|
||||
Validate.notNull(theConceptPid, "theConceptPid must not be null");
|
||||
List<Long> parents = myChildToParentPidCache.get(theConceptPid);
|
||||
if (parents.contains(-1L)) {
|
||||
return;
|
||||
} else if (parents.isEmpty()) {
|
||||
Collection<TermConceptParentChildLink> parentLinks = myConceptParentChildLinkDao.findAllWithChild(theConceptPid);
|
||||
if (parentLinks.isEmpty()) {
|
||||
myChildToParentPidCache.put(theConceptPid, -1L);
|
||||
return;
|
||||
} else {
|
||||
for (TermConceptParentChildLink next : parentLinks) {
|
||||
myChildToParentPidCache.put(theConceptPid, next.getParentPid());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
for (Long nextParent : parents) {
|
||||
if (theParentsBuilder.length() > 0) {
|
||||
theParentsBuilder.append(' ');
|
||||
}
|
||||
theParentsBuilder.append(nextParent);
|
||||
createParentsString(theParentsBuilder, nextParent);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
|
@ -409,8 +439,8 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
}
|
||||
}
|
||||
|
||||
@Scheduled(fixedRate=5000)
|
||||
@Transactional(propagation=Propagation.REQUIRED)
|
||||
@Scheduled(fixedRate = 5000)
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
@Override
|
||||
public synchronized void saveDeferred() {
|
||||
if (!myProcessDeferred) {
|
||||
|
|
@ -420,43 +450,15 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
|
|||
return;
|
||||
}
|
||||
|
||||
int codeCount = 0, relCount = 0;
|
||||
StopWatch stopwatch = new StopWatch();
|
||||
|
||||
int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptsToSaveLater.size());
|
||||
ourLog.info("Saving {} deferred concepts...", count);
|
||||
while (codeCount < count && myConceptsToSaveLater.size() > 0) {
|
||||
TermConcept next = myConceptsToSaveLater.remove(0);
|
||||
codeCount += saveConcept(next);
|
||||
}
|
||||
|
||||
if (codeCount > 0) {
|
||||
ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)", new Object[] {codeCount, myConceptsToSaveLater.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
|
||||
}
|
||||
|
||||
if (codeCount == 0) {
|
||||
count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptLinksToSaveLater.size());
|
||||
ourLog.info("Saving {} deferred concept relationships...", count);
|
||||
while (relCount < count && myConceptLinksToSaveLater.size() > 0) {
|
||||
TermConceptParentChildLink next = myConceptLinksToSaveLater.remove(0);
|
||||
|
||||
if (myConceptDao.findOne(next.getChild().getId()) == null || myConceptDao.findOne(next.getParent().getId()) == null) {
|
||||
ourLog.warn("Not inserting link from child {} to parent {} because it appears to have been deleted", next.getParent().getCode(), next.getChild().getCode());
|
||||
continue;
|
||||
}
|
||||
|
||||
saveConceptLink(next);
|
||||
relCount++;
|
||||
TransactionTemplate tt = new TransactionTemplate(myTransactionMgr);
|
||||
tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
|
||||
tt.execute(new TransactionCallbackWithoutResult() {
|
||||
@Override
|
||||
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
|
||||
processDeferredConcepts();
|
||||
}
|
||||
}
|
||||
|
||||
if (relCount > 0) {
|
||||
ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({}ms / code)", new Object[] {relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
|
||||
}
|
||||
|
||||
if ((myConceptsToSaveLater.size() + myConceptLinksToSaveLater.size()) == 0) {
|
||||
ourLog.info("All deferred concepts and relationships have now been synchronized to the database");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
Loading…
Reference in New Issue