mdm-clear will now use batch-size parameter (#4985)

* using delete expunge service

* test fixing

* update version

* review points

---------

Co-authored-by: leif stawnyczy <leifstawnyczy@leifs-mbp.home>
This commit is contained in:
TipzCM 2023-06-15 20:32:21 -04:00 committed by GitHub
parent f79ac1992e
commit 65ae1785e8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
84 changed files with 282 additions and 115 deletions

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-bom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<packaging>pom</packaging>
<name>HAPI FHIR BOM</name>
@ -12,7 +12,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -6,7 +6,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-cli</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -0,0 +1,8 @@
---
type: fix
issue: 4982
title: "
$mdm-clear has been fixed to respect batch-size parameter
(unless it exceeds 500).
It has also been made slightly more efficient.
"

View File

@ -11,7 +11,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -0,0 +1,19 @@
package ca.uhn.fhir.jpa.bulk.mdm;
import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc;
import ca.uhn.fhir.jpa.api.svc.IMdmClearHelperSvc;
import ca.uhn.fhir.jpa.model.dao.JpaPid;
public class MdmClearHelperSvcImpl implements IMdmClearHelperSvc<JpaPid> {
private final IDeleteExpungeSvc<JpaPid> myDeleteExpungeSvc;
public MdmClearHelperSvcImpl(IDeleteExpungeSvc<JpaPid> theDeleteExpungeSvc) {
myDeleteExpungeSvc = theDeleteExpungeSvc;
}
@Override
public IDeleteExpungeSvc<JpaPid> getDeleteExpungeSvc() {
return myDeleteExpungeSvc;
}
}

View File

@ -31,6 +31,7 @@ import ca.uhn.fhir.interceptor.model.RequestPartitionId;
import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.api.model.ExpungeOptions;
import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc;
import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
import ca.uhn.fhir.jpa.api.svc.ISearchUrlJobMaintenanceSvc;
import ca.uhn.fhir.jpa.binary.interceptor.BinaryStorageInterceptor;
@ -41,6 +42,8 @@ import ca.uhn.fhir.jpa.bulk.export.svc.BulkDataExportJobSchedulingHelperImpl;
import ca.uhn.fhir.jpa.bulk.export.svc.BulkExportHelperService;
import ca.uhn.fhir.jpa.bulk.imprt.api.IBulkDataImportSvc;
import ca.uhn.fhir.jpa.bulk.imprt.svc.BulkDataImportSvcImpl;
import ca.uhn.fhir.jpa.api.svc.IMdmClearHelperSvc;
import ca.uhn.fhir.jpa.bulk.mdm.MdmClearHelperSvcImpl;
import ca.uhn.fhir.jpa.cache.IResourceVersionSvc;
import ca.uhn.fhir.jpa.cache.ResourceVersionSvcDaoImpl;
import ca.uhn.fhir.jpa.dao.DaoSearchParamProvider;
@ -831,4 +834,9 @@ public class JpaConfig {
public ISearchUrlJobMaintenanceSvc searchUrlJobMaintenanceSvc(ResourceSearchUrlSvc theResourceSearchUrlSvc){
return new SearchUrlJobMaintenanceSvcImpl(theResourceSearchUrlSvc);
}
@Bean
public IMdmClearHelperSvc<JpaPid> helperSvc(IDeleteExpungeSvc<JpaPid> theDeleteExpungeSvc) {
return new MdmClearHelperSvcImpl(theDeleteExpungeSvc);
}
}

View File

@ -6,7 +6,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -3,7 +3,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -6,7 +6,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -1,25 +1,38 @@
package ca.uhn.fhir.jpa.mdm.provider;
import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
import ca.uhn.fhir.jpa.entity.MdmLink;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.mdm.api.MdmConstants;
import ca.uhn.fhir.mdm.api.MdmLinkSourceEnum;
import ca.uhn.fhir.mdm.batch2.clear.MdmClearStep;
import ca.uhn.fhir.mdm.model.MdmTransactionContext;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.rest.api.server.IBundleProvider;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.r4.hapi.rest.server.helper.BatchHelperR4;
import org.hl7.fhir.r4.model.DecimalType;
import org.hl7.fhir.r4.model.Parameters;
import org.hl7.fhir.r4.model.Patient;
import org.hl7.fhir.r4.model.Practitioner;
import org.hl7.fhir.r4.model.StringType;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static ca.uhn.fhir.mdm.api.MdmMatchOutcome.POSSIBLE_MATCH;
import static org.hamcrest.MatcherAssert.assertThat;
@ -28,9 +41,16 @@ import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.slf4j.LoggerFactory.getLogger;
public class MdmProviderClearLinkR4Test extends BaseLinkR4Test {
private static final org.slf4j.Logger ourLog = getLogger(MdmProviderClearLinkR4Test.class);
protected Practitioner myPractitioner;
protected StringType myPractitionerId;
protected IAnyResource myPractitionerGoldenResource;
@ -48,6 +68,104 @@ public class MdmProviderClearLinkR4Test extends BaseLinkR4Test {
setMdmRuleJson("mdm/nickname-mdm-rules.json");
}
@Test
public void test_MDMClear_usesBatchSize() {
int batchSize = 5;
int total = 100;
String idTemplate = "RED_%d";
String name = "Yui";
MdmTransactionContext context = createContextForCreate("Patient");
for (int i = 0; i < total; i++) {
if (i % 20 == 0) {
name += i;
}
Patient patient = buildPatientWithNameAndId(name, String.format(idTemplate, i));
DaoMethodOutcome result = myPatientDao.create(patient, myRequestDetails);
myMdmMatchLinkSvc.updateMdmLinksForMdmSource(patient, context);
}
// + 2 because the before() method in the base class
// adds some resources with links; we don't need these, but
// we'll account for them
assertLinkCount(total + 2);
// set log appender
Logger clearStepLogger = (Logger) LoggerFactory.getLogger(MdmClearStep.class);
Level initialLevel = clearStepLogger.getLevel();
clearStepLogger.setLevel(Level.TRACE);
// mocks
@SuppressWarnings("unchecked")
ListAppender<ILoggingEvent> appender = mock(ListAppender.class);
clearStepLogger.addAppender(appender);
// test
try {
Parameters result = (Parameters) myMdmProvider.clearMdmLinks(
null, // resource names (everything if null)
new DecimalType(batchSize), // batch size
myRequestDetails // request details
);
myBatch2JobHelper.awaitJobCompletion(BatchHelperR4.jobIdFromBatch2Parameters(result));
// verify
assertNoLinksExist();
// the trace log we're inspecting is in MdmClearStep
// "Deleted {} of {} golden resources in {}"
String regex = ".(\\d)+";
ArgumentCaptor<ILoggingEvent> loggingCaptor = ArgumentCaptor.forClass(ILoggingEvent.class);
verify(appender, atLeastOnce())
.doAppend(loggingCaptor.capture());
Pattern pattern = Pattern.compile(regex);
boolean hasMsgs = false;
for (ILoggingEvent event : loggingCaptor.getAllValues()) {
if (event.getLevel() != Level.TRACE) {
continue; // we are only looking at the trace measures
}
String msg = event.getFormattedMessage();
ourLog.info(msg);
if (msg.contains("golden resources")) {
hasMsgs = true;
// golden resources deleted
boolean contains = msg.contains(String.format("Deleted %d of %d golden resources in", batchSize, batchSize));
if (!contains) {
// if we didn't delete exactly <batchsize>, we should've deleted < batchsize
Matcher matcher = pattern.matcher(msg);
int count = 0;
int deletedCount = -1;
int deletedTotal = -1;
while (matcher.find()) {
String group = matcher.group().trim();
int i = Integer.parseInt(group);
if (count == 0) {
deletedCount = i;
} else if (count == 1) {
deletedTotal = i;
}
count++;
}
// we have < batch size, but it should be the total deleted still
assertTrue( deletedTotal < batchSize, msg);
assertEquals(deletedTotal, deletedCount, msg);
} else {
// pointless, but...
assertTrue(contains);
}
}
}
// want to make sure we found the trace messages
// or what's the point
assertTrue(hasMsgs);
} finally {
clearStepLogger.detachAppender(appender);
clearStepLogger.setLevel(initialLevel);
}
}
@Test
public void testClearAllLinks() {
assertLinkCount(2);

View File

@ -6,15 +6,15 @@ import ca.uhn.fhir.batch2.model.JobInstance;
import ca.uhn.fhir.jpa.entity.MdmLink;
import ca.uhn.fhir.jpa.mdm.BaseMdmR4Test;
import ca.uhn.fhir.jpa.mdm.helper.MdmHelperR4;
import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.mdm.api.MdmLinkSourceEnum;
import ca.uhn.fhir.mdm.api.MdmMatchResultEnum;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException;
import org.hl7.fhir.r4.model.Patient;
import org.hl7.fhir.r4.model.Reference;
import org.junit.jupiter.api.BeforeEach;
@ -85,10 +85,13 @@ class MdmClearStepTest extends BaseMdmR4Test {
try {
mdmClearGoldenResource();
fail();
} catch (ResourceVersionConflictException e) {
assertEquals("HAPI-0550: HAPI-0515: Unable to delete " + myGoldenId +
" because at least one resource has a reference to this resource. First reference found was resource " +
husbandId + " in path Patient.link.other", e.getMessage());
} catch (InvalidRequestException e) {
assertEquals(
String.format("HAPI-0822: DELETE with _expunge=true failed. Unable to delete %s because %s refers to it via the path Patient.link.other",
myGoldenId,
husbandId
),
e.getMessage());
}
}

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -6,7 +6,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -6,7 +6,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -6,7 +6,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -6,7 +6,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -6,7 +6,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -6,7 +6,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -7,7 +7,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -7,7 +7,7 @@
<parent>
<artifactId>hapi-fhir-serviceloaders</artifactId>
<groupId>ca.uhn.hapi.fhir</groupId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -7,7 +7,7 @@
<parent>
<artifactId>hapi-fhir-serviceloaders</artifactId>
<groupId>ca.uhn.hapi.fhir</groupId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@ -21,7 +21,7 @@
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-caching-api</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
</dependency>
<dependency>

View File

@ -7,7 +7,7 @@
<parent>
<artifactId>hapi-fhir-serviceloaders</artifactId>
<groupId>ca.uhn.hapi.fhir</groupId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -7,7 +7,7 @@
<parent>
<artifactId>hapi-fhir</artifactId>
<groupId>ca.uhn.hapi.fhir</groupId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>hapi-deployable-pom</artifactId>
<groupId>ca.uhn.hapi.fhir</groupId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-spring-boot-samples</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
</parent>
<artifactId>hapi-fhir-spring-boot-sample-client-apache</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-spring-boot-samples</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-spring-boot-samples</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-spring-boot</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -7,7 +7,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -7,7 +7,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -63,9 +63,10 @@ public class ResourceIdListStep<PT extends PartitionedJobParameters, IT extends
Date start = data.getStart();
Date end = data.getEnd();
Integer pageSize = theStepExecutionDetails.getParameters().getBatchSize();
if (pageSize == null) {
pageSize = DEFAULT_PAGE_SIZE;
Integer batchSize = theStepExecutionDetails.getParameters().getBatchSize();
int pageSize = DEFAULT_PAGE_SIZE;
if (batchSize != null) {
pageSize = batchSize.intValue();
}
ourLog.info("Beginning scan for reindex IDs in range {} to {}", start, end);
@ -76,6 +77,12 @@ public class ResourceIdListStep<PT extends PartitionedJobParameters, IT extends
long previousLastTime = 0L;
int totalIdsFound = 0;
int chunkCount = 0;
int maxBatchId = MAX_BATCH_OF_IDS;
if (batchSize != null) {
// we won't go over MAX_BATCH_OF_IDS
maxBatchId = Math.min(batchSize.intValue(), maxBatchId);
}
while (true) {
IResourcePidList nextChunk = myIdChunkProducer.fetchResourceIdsPage(nextStart, end, pageSize, requestPartitionId, theStepExecutionDetails.getData());
@ -105,12 +112,12 @@ public class ResourceIdListStep<PT extends PartitionedJobParameters, IT extends
previousLastTime = nextChunk.getLastDate().getTime();
nextStart = nextChunk.getLastDate();
while (idBuffer.size() > MAX_BATCH_OF_IDS) {
while (idBuffer.size() > maxBatchId) {
List<TypedPidJson> submissionIds = new ArrayList<>();
for (Iterator<TypedPidJson> iter = idBuffer.iterator(); iter.hasNext(); ) {
submissionIds.add(iter.next());
iter.remove();
if (submissionIds.size() == MAX_BATCH_OF_IDS) {
if (submissionIds.size() == maxBatchId) {
break;
}
}

View File

@ -7,7 +7,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -26,20 +26,16 @@ import ca.uhn.fhir.batch2.api.RunOutcome;
import ca.uhn.fhir.batch2.api.StepExecutionDetails;
import ca.uhn.fhir.batch2.api.VoidModel;
import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.api.model.DeleteConflictList;
import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc;
import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
import ca.uhn.fhir.jpa.api.svc.IMdmClearHelperSvc;
import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
import ca.uhn.fhir.jpa.delete.DeleteConflictUtil;
import ca.uhn.fhir.jpa.model.dao.JpaPid;
import ca.uhn.fhir.mdm.dao.IMdmLinkDao;
import ca.uhn.fhir.mdm.interceptor.MdmStorageInterceptor;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId;
import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
import ca.uhn.fhir.rest.server.provider.ProviderConstants;
import ca.uhn.fhir.util.StopWatch;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
@ -52,6 +48,7 @@ import javax.annotation.Nonnull;
import java.util.List;
import java.util.concurrent.TimeUnit;
@SuppressWarnings("rawtypes")
public class MdmClearStep implements IJobStepWorker<MdmClearJobParameters, ResourceIdListWorkChunkJson, VoidModel> {
private static final Logger ourLog = LoggerFactory.getLogger(MdmClearStep.class);
@ -60,24 +57,23 @@ public class MdmClearStep implements IJobStepWorker<MdmClearJobParameters, Resou
@Autowired
HapiTransactionService myHapiTransactionService;
@Autowired
DaoRegistry myDaoRegistry;
@Autowired
IIdHelperService myIdHelperService;
@Autowired
FhirContext myFhirContext;
@Autowired
IMdmLinkDao myMdmLinkSvc;
@Autowired
private IMdmClearHelperSvc<? extends IResourcePersistentId<?>> myIMdmClearHelperSvc;
@Nonnull
@Override
public RunOutcome run(@Nonnull StepExecutionDetails<MdmClearJobParameters, ResourceIdListWorkChunkJson> theStepExecutionDetails, @Nonnull IJobDataSink<VoidModel> theDataSink) throws JobExecutionFailedException {
SystemRequestDetails requestDetails = new SystemRequestDetails();
requestDetails.setRetry(true);
requestDetails.setMaxRetries(100);
requestDetails.setRequestPartitionId(theStepExecutionDetails.getParameters().getRequestPartitionId());
TransactionDetails transactionDetails = new TransactionDetails();
myHapiTransactionService.execute(requestDetails, transactionDetails, buildJob(requestDetails, transactionDetails, theStepExecutionDetails));
myHapiTransactionService
.execute(requestDetails, transactionDetails, buildJob(requestDetails, transactionDetails, theStepExecutionDetails));
return new RunOutcome(theStepExecutionDetails.getData().size());
}
@ -101,9 +97,10 @@ public class MdmClearStep implements IJobStepWorker<MdmClearJobParameters, Resou
myChunkId = theStepExecutionDetails.getChunkId();
}
@SuppressWarnings("unchecked")
@Override
public Void doInTransaction(@Nonnull TransactionStatus theStatus) {
List<JpaPid> persistentIds = myData.getResourcePersistentIds(myIdHelperService);
List<? extends IResourcePersistentId> persistentIds = myData.getResourcePersistentIds(myIdHelperService);
if (persistentIds.isEmpty()) {
return null;
}
@ -121,25 +118,25 @@ public class MdmClearStep implements IJobStepWorker<MdmClearJobParameters, Resou
return null;
}
private void performWork(List<JpaPid> thePersistentIds) {
@SuppressWarnings({"unchecked", "rawtypes"})
private void performWork(List<? extends IResourcePersistentId> thePersistentIds) {
ourLog.info("Starting mdm clear work chunk with {} resources - Instance[{}] Chunk[{}]", thePersistentIds.size(), myInstanceId, myChunkId);
StopWatch sw = new StopWatch();
myMdmLinkSvc.deleteLinksWithAnyReferenceToPids(thePersistentIds);
ourLog.trace("Deleted {} mdm links in {}", thePersistentIds.size(), StopWatch.formatMillis(sw.getMillis()));
// We know the list is not empty, and that all resource types are the same, so just use the first one
String resourceName = myData.getResourceType(0);
IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(resourceName);
// use the expunge service to delete multiple resources at once efficiently
IDeleteExpungeSvc deleteExpungeSvc = myIMdmClearHelperSvc.getDeleteExpungeSvc();
int deletedRecords = deleteExpungeSvc.deleteExpunge(
thePersistentIds,
false,
null
);
DeleteConflictList conflicts = new DeleteConflictList();
dao.deletePidList(ProviderConstants.OPERATION_MDM_CLEAR, thePersistentIds, conflicts, myRequestDetails);
DeleteConflictUtil.validateDeleteConflictsEmptyOrThrowException(myFhirContext, conflicts);
ourLog.trace("Deleted {} golden resources in {}", thePersistentIds.size(), StopWatch.formatMillis(sw.getMillis()));
ourLog.trace("Deleted {} of {} golden resources in {}", deletedRecords, thePersistentIds.size(), StopWatch.formatMillis(sw.getMillis()));
dao.expunge(thePersistentIds, myRequestDetails);
ourLog.info("Finished removing {} golden resources in {} - {}/sec - Instance[{}] Chunk[{}]", thePersistentIds.size(), sw, sw.formatThroughput(thePersistentIds.size(), TimeUnit.SECONDS), myInstanceId, myChunkId);
ourLog.info("Finished removing {} of {} golden resources in {} - {}/sec - Instance[{}] Chunk[{}]", deletedRecords, thePersistentIds.size(), sw, sw.formatThroughput(thePersistentIds.size(), TimeUnit.SECONDS), myInstanceId, myChunkId);
if (ourClearCompletionCallbackForUnitTest != null) {
ourClearCompletionCallbackForUnitTest.run();

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -0,0 +1,8 @@
package ca.uhn.fhir.jpa.api.svc;
import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId;
public interface IMdmClearHelperSvc<T extends IResourcePersistentId<?>> {
IDeleteExpungeSvc<T> getDeleteExpungeSvc();
}

View File

@ -95,7 +95,6 @@ public class HapiTransactionService implements IHapiTransactionService {
return new ExecutionBuilder(null);
}
/**
* @deprecated Use {@link #withRequest(RequestDetails)} with fluent call instead
*/

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@ -7,7 +7,7 @@
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<packaging>pom</packaging>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<name>HAPI-FHIR</name>
<description>An open-source implementation of the FHIR specification in Java.</description>

View File

@ -7,7 +7,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir</artifactId>
<version>6.7.6-SNAPSHOT</version>
<version>6.7.8-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>