honeymoose/hapi-fhir
1
0
Fork 0
mirror of https://github.com/hapifhir/hapi-fhir.git synced 2025-02-17 02:15:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Further reduce memory usage in the terminology delta operations

Browse Source
This commit is contained in:
James Agnew 2020-01-28 14:36:15 -05:00
parent 3edeb03edc
commit 471073103f
3 changed files with 57 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/4_2_0/changes.yaml
View File

@ -84,4 +84,4 @@ Sean McIlvenna for reporting!"
Petro Mykhailysyn for the pull request!" Petro Mykhailysyn for the pull request!"
- item: - item:
type: "fix" type: "fix"
title: "A meomery leak was resolved in the JPA terminology service delta upload operations." title: "A memory leak was resolved in the JPA terminology service delta upload operations."

115
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcImpl.java
View File

@ -164,86 +164,12 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
IIdType codeSystemId = cs.getResource().getIdDt(); IIdType codeSystemId = cs.getResource().getIdDt();
// Load all concepts for the code system
Map<String, Long> codeToConceptPid = new HashMap<>();
{
ourLog.info("Loading all concepts in CodeSystem versionPid[{}] and url[{}]", cs.getPid(), theSystem);
StopWatch sw = new StopWatch();
CriteriaBuilder criteriaBuilder = myEntityManager.getCriteriaBuilder();
CriteriaQuery<TermConcept> query = criteriaBuilder.createQuery(TermConcept.class);
Root<TermConcept> root = query.from(TermConcept.class);
Predicate predicate = criteriaBuilder.equal(root.get("myCodeSystemVersionPid").as(Long.class), csv.getPid());
query.where(predicate);
TypedQuery<TermConcept> typedQuery = myEntityManager.createQuery(query.select(root));
org.hibernate.query.Query<TermConcept> hibernateQuery = (org.hibernate.query.Query<TermConcept>) typedQuery;
ScrollableResults scrollableResults = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY);
int count = 0;
try (ScrollableResultsIterator<TermConcept> scrollableResultsIterator = new ScrollableResultsIterator<>(scrollableResults)) {
while (scrollableResultsIterator.hasNext()) {
TermConcept next = scrollableResultsIterator.next();
codeToConceptPid.put(next.getCode(), next.getId());
// We don't want to keep the loaded entities in the L1 cache because they can take up a loooot of memory
if (count % 100 == 0) {
myEntityManager.clear();
}
count++;
}
}
ourLog.info("Loaded {} concepts in {}", codeToConceptPid.size(), sw.toString());
}
// Load all parent/child links
ListMultimap<String, String> parentCodeToChildCodes = ArrayListMultimap.create();
ListMultimap<String, String> childCodeToParentCodes = ArrayListMultimap.create();
{
ourLog.info("Loading all parent/child relationships in CodeSystem url[" + theSystem + "]");
int count = 0;
StopWatch sw = new StopWatch();
CriteriaBuilder criteriaBuilder = myEntityManager.getCriteriaBuilder();
CriteriaQuery<TermConceptParentChildLink> query = criteriaBuilder.createQuery(TermConceptParentChildLink.class);
Root<TermConceptParentChildLink> root = query.from(TermConceptParentChildLink.class);
Predicate predicate = criteriaBuilder.equal(root.get("myCodeSystemVersionPid").as(Long.class), csv.getPid());
root.fetch("myChild");
root.fetch("myParent");
query.where(predicate);
TypedQuery<TermConceptParentChildLink> typedQuery = myEntityManager.createQuery(query.select(root));
org.hibernate.query.Query<TermConceptParentChildLink> hibernateQuery = (org.hibernate.query.Query<TermConceptParentChildLink>) typedQuery;
ScrollableResults scrollableResults = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY);
try (ScrollableResultsIterator<TermConceptParentChildLink> scrollableResultsIterator = new ScrollableResultsIterator<>(scrollableResults)) {
while (scrollableResultsIterator.hasNext()) {
TermConceptParentChildLink next = scrollableResultsIterator.next();
String parentCode = next.getParent().getCode();
String childCode = next.getChild().getCode();
parentCodeToChildCodes.put(parentCode, childCode);
childCodeToParentCodes.put(childCode, parentCode);
// We don't want to keep the loaded entities in the L1 cache because they can take up a loooot of memory
if (count % 100 == 0) {
myEntityManager.clear();
}
count++;
}
}
ourLog.info("Loaded {} parent/child relationships in {}", count, sw.toString());
}
ourLog.trace("Starting delta application");
// Account for root codes in the parent->child map
for (String nextCode : codeToConceptPid.keySet()) {
if (childCodeToParentCodes.get(nextCode).isEmpty()) {
parentCodeToChildCodes.put("", nextCode);
}
}
UploadStatistics retVal = new UploadStatistics(codeSystemId); UploadStatistics retVal = new UploadStatistics(codeSystemId);
// Add root concepts // Add root concepts
for (TermConcept nextRootConcept : theAdditions.getRootConcepts()) { for (TermConcept nextRootConcept : theAdditions.getRootConcepts()) {
List<String> parentCodes = Collections.emptyList(); List<String> parentCodes = Collections.emptyList();
addConcept(csv, codeToConceptPid, parentCodes, nextRootConcept, parentCodeToChildCodes, retVal, theAdditions.getRootConceptCodes(), true); addConcept(csv, parentCodes, nextRootConcept, retVal, true, 0);
} }
return retVal; return retVal;
@ -519,48 +445,37 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
Validate.isTrue(myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3), "Terminology operations only supported in DSTU3+ mode"); Validate.isTrue(myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3), "Terminology operations only supported in DSTU3+ mode");
} }
private void addConcept(TermCodeSystemVersion theCsv, Map<String, Long> theCodeToConceptPid, Collection<String> theParentCodes, TermConcept theConceptToAdd, ListMultimap<String, String> theParentCodeToChildCodes, UploadStatistics theStatisticsTracker, Set<String> theAdditionSetRootConceptCodes, boolean theRootConcept) { private void addConcept(TermCodeSystemVersion theCsv, Collection<String> theParentCodes, TermConcept theConceptToAdd, UploadStatistics theStatisticsTracker, boolean theRootConcept, int theSequence) {
TermConcept conceptToAdd = theConceptToAdd; TermConcept conceptToAdd = theConceptToAdd;
List<TermConceptParentChildLink> childrenToAdd = theConceptToAdd.getChildren(); List<TermConceptParentChildLink> childrenToAdd = theConceptToAdd.getChildren();
String nextCodeToAdd = conceptToAdd.getCode(); String nextCodeToAdd = conceptToAdd.getCode();
String parentDescription = "(root concept)"; String parentDescription = "(root concept)";
Set<TermConcept> parentConcepts = new HashSet<>(); Set<TermConcept> parentConcepts = new HashSet<>();
if (!theParentCodes.isEmpty()) { if (!theParentCodes.isEmpty()) {
parentDescription = "[" + String.join(", ", theParentCodes) + "]"; parentDescription = "[" + String.join(", ", theParentCodes) + "]";
for (String nextParentCode : theParentCodes) { for (String nextParentCode : theParentCodes) {
Long nextParentCodePid = theCodeToConceptPid.get(nextParentCode); Optional<TermConcept> nextParentOpt = myConceptDao.findByCodeSystemAndCode(theCsv, nextParentCode);
if (nextParentCodePid == null) { if (nextParentOpt.isPresent() == false) {
throw new InvalidRequestException("Unable to add code \"" + nextCodeToAdd + "\" to unknown parent: " + nextParentCode); throw new InvalidRequestException("Unable to add code \"" + nextCodeToAdd + "\" to unknown parent: " + nextParentCode);
} }
parentConcepts.add(myConceptDao.getOne(nextParentCodePid)); parentConcepts.add(nextParentOpt.get());
} }
} }
ourLog.info("Saving concept {} with parent {}", theStatisticsTracker.getUpdatedConceptCount(), parentDescription); ourLog.info("Saving concept {} with parent {}", theStatisticsTracker.getUpdatedConceptCount(), parentDescription);
if (theCodeToConceptPid.containsKey(nextCodeToAdd)) { Optional<TermConcept> existingCodeOpt = myConceptDao.findByCodeSystemAndCode(theCsv, nextCodeToAdd);
if (existingCodeOpt.isPresent()) {
TermConcept existingCode = myConceptDao.getOne(theCodeToConceptPid.get(nextCodeToAdd)); TermConcept existingCode = existingCodeOpt.get();
existingCode.setIndexStatus(null); existingCode.setIndexStatus(null);
existingCode.setDisplay(conceptToAdd.getDisplay()); existingCode.setDisplay(conceptToAdd.getDisplay());
conceptToAdd = existingCode; conceptToAdd = existingCode;
} }
if (conceptToAdd.getSequence() == null || !theRootConcept) { if (conceptToAdd.getSequence() == null) {
// If this is a new code, give it a sequence number based on how many concepts the conceptToAdd.setSequence(theSequence);
// parent already has (or the highest number, if the code has multiple parents)
int sequence = 0;
for (String nextParentCode : theParentCodes) {
theParentCodeToChildCodes.put(nextParentCode, nextCodeToAdd);
sequence = Math.max(sequence, theParentCodeToChildCodes.get(nextParentCode).size());
}
if (theParentCodes.isEmpty()) {
theParentCodeToChildCodes.put("", nextCodeToAdd);
sequence = Math.max(sequence, theParentCodeToChildCodes.get("").size());
}
conceptToAdd.setSequence(sequence);
} }
// Drop any old parent-child links if they aren't explicitly specified in the // Drop any old parent-child links if they aren't explicitly specified in the
@ -586,7 +501,6 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
Long nextConceptPid = conceptToAdd.getId(); Long nextConceptPid = conceptToAdd.getId();
Validate.notNull(nextConceptPid); Validate.notNull(nextConceptPid);
theCodeToConceptPid.put(nextCodeToAdd, nextConceptPid);
theStatisticsTracker.incrementUpdatedConceptCount(); theStatisticsTracker.incrementUpdatedConceptCount();
// Add link to new child to the parent // Add link to new child to the parent
@ -605,6 +519,7 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
ourLog.trace("About to save parent-child links"); ourLog.trace("About to save parent-child links");
// Save children recursively // Save children recursively
int childIndex = 0;
for (TermConceptParentChildLink nextChildConceptLink : new ArrayList<>(childrenToAdd)) { for (TermConceptParentChildLink nextChildConceptLink : new ArrayList<>(childrenToAdd)) {
TermConcept nextChild = nextChildConceptLink.getChild(); TermConcept nextChild = nextChildConceptLink.getChild();
@ -612,15 +527,15 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
for (int i = 0; i < nextChild.getParents().size(); i++) { for (int i = 0; i < nextChild.getParents().size(); i++) {
if (nextChild.getParents().get(i).getId() == null) { if (nextChild.getParents().get(i).getId() == null) {
String parentCode = nextChild.getParents().get(i).getParent().getCode(); String parentCode = nextChild.getParents().get(i).getParent().getCode();
Long parentPid = theCodeToConceptPid.get(parentCode); TermConcept parentConcept = myConceptDao.findByCodeSystemAndCode(theCsv, parentCode).orElseThrow(() -> new IllegalArgumentException("Unknown parent code: " + parentCode));
TermConcept parentConcept = myConceptDao.findById(parentPid).orElseThrow(() -> new IllegalArgumentException("Unknown parent code: " + parentCode));
nextChild.getParents().get(i).setParent(parentConcept); nextChild.getParents().get(i).setParent(parentConcept);
} }
} }
Collection<String> parentCodes = nextChild.getParents().stream().map(t -> t.getParent().getCode()).collect(Collectors.toList()); Collection<String> parentCodes = nextChild.getParents().stream().map(t -> t.getParent().getCode()).collect(Collectors.toList());
addConcept(theCsv, theCodeToConceptPid, parentCodes, nextChild, theParentCodeToChildCodes, theStatisticsTracker, theAdditionSetRootConceptCodes, false); addConcept(theCsv, parentCodes, nextChild, theStatisticsTracker, false, childIndex);
childIndex++;
} }
} }

82
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcDeltaR4Test.java
View File

@ -47,8 +47,8 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
delta.addRootConcept("RootB", "Root B"); delta.addRootConcept("RootB", "Root B");
myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta); myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta);
assertHierarchyContains( assertHierarchyContains(
"RootA seq=1", "RootA seq=0",
"RootB seq=2" "RootB seq=0"
); );
delta = new CustomTerminologySet(); delta = new CustomTerminologySet();
@ -56,10 +56,10 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
delta.addRootConcept("RootD", "Root D"); delta.addRootConcept("RootD", "Root D");
myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta); myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta);
assertHierarchyContains( assertHierarchyContains(
"RootA seq=1", "RootA seq=0",
"RootB seq=2", "RootB seq=0",
"RootC seq=3", "RootC seq=0",
"RootD seq=4" "RootD seq=0"
); );
} }
@ -103,8 +103,8 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
delta.addRootConcept("RootB", "Root B"); delta.addRootConcept("RootB", "Root B");
myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta); myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta);
assertHierarchyContains( assertHierarchyContains(
"RootA seq=1", "RootA seq=0",
"RootB seq=2" "RootB seq=0"
); );
ourLog.info("Have performed add"); ourLog.info("Have performed add");
@ -120,10 +120,10 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
root.addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA).setCode("ChildAB").setDisplay("Child AB"); root.addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA).setCode("ChildAB").setDisplay("Child AB");
myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta); myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta);
assertHierarchyContains( assertHierarchyContains(
"RootA seq=1", "RootA seq=0",
" ChildAA seq=1", " ChildAA seq=0",
" ChildAB seq=2", " ChildAB seq=1",
"RootB seq=2" "RootB seq=0"
); );
} }
@ -142,10 +142,10 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
delta.addRootConcept("RootB", "Root B"); delta.addRootConcept("RootB", "Root B");
outcome = myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta); outcome = myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta);
assertHierarchyContains( assertHierarchyContains(
"RootA seq=1", "RootA seq=0",
" ChildAA seq=1", " ChildAA seq=0",
" ChildAAA seq=1", " ChildAAA seq=0",
"RootB seq=2" "RootB seq=0"
); );
assertEquals(4, outcome.getUpdatedConceptCount()); assertEquals(4, outcome.getUpdatedConceptCount());
@ -154,10 +154,10 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
.addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA).setCode("ChildAA").setDisplay("Child AA"); .addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA).setCode("ChildAA").setDisplay("Child AA");
outcome = myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta); outcome = myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta);
assertHierarchyContains( assertHierarchyContains(
"RootA seq=1", "RootA seq=0",
"RootB seq=2", "RootB seq=0",
" ChildAA seq=1", " ChildAA seq=0",
" ChildAAA seq=1" " ChildAAA seq=0"
); );
assertEquals(2, outcome.getUpdatedConceptCount()); assertEquals(2, outcome.getUpdatedConceptCount());
@ -199,8 +199,8 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
// Check so far // Check so far
assertHierarchyContains( assertHierarchyContains(
"ParentA seq=1", "ParentA seq=0",
" ChildA seq=1" " ChildA seq=0"
); );
// Add sub-child to existing child // Add sub-child to existing child
@ -212,9 +212,9 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
// Check so far // Check so far
assertHierarchyContains( assertHierarchyContains(
"ParentA seq=1", "ParentA seq=0",
" ChildA seq=1", " ChildA seq=0",
" ChildAA seq=1" " ChildAA seq=0"
); );
} }
@ -285,14 +285,14 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
UploadStatistics outcome = myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta); UploadStatistics outcome = myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta);
assertEquals(8, outcome.getUpdatedConceptCount()); assertEquals(8, outcome.getUpdatedConceptCount());
assertHierarchyContains( assertHierarchyContains(
"CodeA seq=1", "CodeA seq=0",
" CodeAA seq=1", " CodeAA seq=0",
" CodeAAA seq=1", " CodeAAA seq=0",
" CodeAAB seq=2", " CodeAAB seq=1",
"CodeB seq=2", "CodeB seq=0",
" CodeBA seq=1", " CodeBA seq=0",
" CodeBAA seq=1", " CodeBAA seq=0",
" CodeBAB seq=2" " CodeBAB seq=1"
); );
// Move a single child code to a new spot and make sure the hierarchy comes along // Move a single child code to a new spot and make sure the hierarchy comes along
@ -304,14 +304,14 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
outcome = myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta); outcome = myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta);
assertEquals(2, outcome.getUpdatedConceptCount()); assertEquals(2, outcome.getUpdatedConceptCount());
assertHierarchyContains( assertHierarchyContains(
"CodeA seq=1", "CodeA seq=0",
"CodeB seq=2", "CodeB seq=0",
" CodeBA seq=1", " CodeBA seq=0",
" CodeBAA seq=1", " CodeBAA seq=0",
" CodeBAB seq=2", " CodeBAB seq=1",
" CodeAA seq=2", // <-- CodeAA got added here so it comes second " CodeAA seq=0", // <-- CodeAA got added here so it comes second
" CodeAAA seq=1", " CodeAAA seq=0",
" CodeAAB seq=2" " CodeAAB seq=1"
); );
} }