honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Modify expunge everything to use small batches of deletes

This commit is contained in:
James Agnew 2019-05-31 11:05:17 -04:00
parent c811fe8a2f
commit f6c7ff1dbb
2 changed files with 76 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ExpungeEverythingService.java
View File

@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.dao.expunge;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -34,15 +34,19 @@ import org.springframework.transaction.support.TransactionTemplate;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceContextType;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
@Service
public class ExpungeEverythingService {
private static final Logger ourLog = LoggerFactory.getLogger(ExpungeEverythingService.class);
@Autowired
private PlatformTransactionManager myPlatformTransactionManager;
@PersistenceContext(type = PersistenceContextType.TRANSACTION)
protected EntityManager myEntityManager;
@Autowired
private PlatformTransactionManager myPlatformTransactionManager;
void expungeEverything() {
@ -57,52 +61,43 @@ public class ExpungeEverythingService {
counter.addAndGet(doExpungeEverythingQuery("UPDATE " + TermCodeSystem.class.getSimpleName() + " d SET d.myCurrentVersion = null"));
return null;
});
counter.addAndGet(doExpungeEverythingQuery(txTemplate, SearchParamPresent.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ForcedId.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceIndexedSearchParamDate.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceIndexedSearchParamNumber.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceIndexedSearchParamQuantity.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceIndexedSearchParamString.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceIndexedSearchParamToken.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceIndexedSearchParamUri.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceIndexedSearchParamCoords.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceIndexedCompositeStringUnique.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceLink.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, SearchResult.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, SearchInclude.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermConceptParentChildLink.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermConceptMapGroupElementTarget.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermConceptMapGroupElement.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermConceptMapGroup.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermConceptMap.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermConceptProperty.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermConceptDesignation.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermConcept.class));
txTemplate.execute(t -> {
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + SearchParamPresent.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ForcedId.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceIndexedSearchParamDate.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceIndexedSearchParamNumber.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceIndexedSearchParamQuantity.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceIndexedSearchParamString.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceIndexedSearchParamToken.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceIndexedSearchParamUri.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceIndexedSearchParamCoords.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceIndexedCompositeStringUnique.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceLink.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + SearchResult.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + SearchInclude.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermConceptParentChildLink.class.getSimpleName() + " d"));
return null;
});
txTemplate.execute(t -> {
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermConceptMapGroupElementTarget.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermConceptMapGroupElement.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermConceptMapGroup.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermConceptMap.class.getSimpleName() + " d"));
return null;
});
txTemplate.execute(t -> {
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermConceptProperty.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermConceptDesignation.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermConcept.class.getSimpleName() + " d"));
for (TermCodeSystem next : myEntityManager.createQuery("SELECT c FROM " + TermCodeSystem.class.getName() + " c", TermCodeSystem.class).getResultList()) {
next.setCurrentVersion(null);
myEntityManager.merge(next);
}
return null;
});
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermCodeSystemVersion.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TermCodeSystem.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, SubscriptionTable.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceHistoryTag.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceTag.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, TagDefinition.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceHistoryTable.class));
counter.addAndGet(doExpungeEverythingQuery(txTemplate, ResourceTable.class));
txTemplate.execute(t -> {
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermCodeSystemVersion.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TermCodeSystem.class.getSimpleName() + " d"));
return null;
});
txTemplate.execute(t -> {
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + SubscriptionTable.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceHistoryTag.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceTag.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + TagDefinition.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceHistoryTable.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + ResourceTable.class.getSimpleName() + " d"));
counter.addAndGet(doExpungeEverythingQuery("DELETE from " + org.hibernate.search.jpa.Search.class.getSimpleName() + " d"));
return null;
});
@ -110,10 +105,42 @@ public class ExpungeEverythingService {
ourLog.info("COMPLETED GLOBAL $expunge - Deleted {} rows", counter.get());
}
private int doExpungeEverythingQuery(TransactionTemplate txTemplate, Class<?> theEntityType) {
int outcome = 0;
while (true) {
StopWatch sw = new StopWatch();
@SuppressWarnings("ConstantConditions")
int count = txTemplate.execute(t -> {
CriteriaBuilder cb = myEntityManager.getCriteriaBuilder();
CriteriaQuery<?> cq = cb.createQuery(theEntityType);
cq.from(theEntityType);
TypedQuery<?> query = myEntityManager.createQuery(cq);
query.setMaxResults(1000);
List<?> results = query.getResultList();
for (Object result : results) {
myEntityManager.remove(result);
}
return results.size();
});
outcome += count;
if (count == 0) {
break;
}
ourLog.info("Have deleted {} entities of type {} in {}", outcome, theEntityType.getSimpleName(), sw.toString());
}
return outcome;
}
private int doExpungeEverythingQuery(String theQuery) {
StopWatch sw = new StopWatch();
int outcome = myEntityManager.createQuery(theQuery).executeUpdate();
ourLog.debug("Query affected {} rows in {}: {}", outcome, sw.toString(), theQuery);
return outcome;
}
}

7
src/changes/changes.xml
View File

@ -6,6 +6,13 @@
<title>HAPI FHIR Changelog</title>
</properties>
<body>
<release version="4.0.0" date="TBD" description="Igloo">
<action type="add">
The $expunge global everything operation has been refactored to do deletes
in small batches. This change will likely reduce performance, but does allow
for the operation to succeed without timing out in larger system.
</action>
</release>
<release version="3.8.0" date="2019-05-30" description="Hippo">
<action type="fix">
A potential security vulnerability in the hapi-fhir-testpage-overlay project was corrected: A URL