From 72a837cb103e3c5984ccb8f0115294d169854f0b Mon Sep 17 00:00:00 2001 From: jdar Date: Mon, 26 Aug 2024 10:21:43 -0700 Subject: [PATCH] pass the whole work chunk instead of just the ID --- .../delete/batch2/DeleteExpungeSvcImpl.java | 16 ++++++++------ .../dao/r4/FhirResourceDaoR4CreateTest.java | 2 +- .../r4/FhirResourceDaoR4QueryCountTest.java | 6 ++++- .../jpa/delete/DeleteExpungeSvcImplTest.java | 5 ++++- .../interceptor/MdmStorageInterceptor.java | 2 +- .../jobs/expunge/DeleteExpungeStep.java | 22 ++++++++++--------- .../fhir/mdm/batch2/clear/MdmClearStep.java | 11 +++++----- .../fhir/jpa/api/svc/IDeleteExpungeSvc.java | 4 +++- 8 files changed, 41 insertions(+), 27 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java index 4719879f83d..2e980f3e0be 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java @@ -19,10 +19,12 @@ */ package ca.uhn.fhir.jpa.delete.batch2; +import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.model.dao.JpaPid; import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.model.api.IModelJson; import jakarta.persistence.EntityManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,13 +50,16 @@ public class DeleteExpungeSvcImpl implements IDeleteExpungeSvc { } @Override - public int deleteExpunge( - List theJpaPids, boolean theCascade, Integer theCascadeMaxRounds, String theChunkId) { + public int deleteExpunge( + List theJpaPids, boolean theCascade, Integer theCascadeMaxRounds, PT theWorkChunk) { DeleteExpungeSqlBuilder.DeleteExpungeSqlResult sqlResult = myDeleteExpungeSqlBuilder.convertPidsToDeleteExpungeSql(theJpaPids, theCascade, theCascadeMaxRounds); List sqlList = sqlResult.getSqlStatements(); - String formattedChunkIdForLogMessage = theChunkId.isBlank() ? "" : "Chunk[" + theChunkId + "] - "; + String formattedChunkIdForLogMessage = ""; + if (theWorkChunk instanceof WorkChunk && !((WorkChunk) theWorkChunk).getId().isBlank()) { + formattedChunkIdForLogMessage = "Chunk[" + ((WorkChunk) theWorkChunk).getId() + "] - "; + } ourLog.debug("{}Executing {} delete expunge sql commands", formattedChunkIdForLogMessage, sqlList.size()); long totalDeleted = 0; @@ -63,10 +68,7 @@ public class DeleteExpungeSvcImpl implements IDeleteExpungeSvc { totalDeleted += myEntityManager.createNativeQuery(sql).executeUpdate(); } - ourLog.info( - "{}Delete expunge sql commands affected {} rows", - formattedChunkIdForLogMessage, - totalDeleted); + ourLog.info("{}Delete expunge sql commands affected {} rows", formattedChunkIdForLogMessage, totalDeleted); clearHibernateSearchIndex(theJpaPids); // TODO KHS instead of logging progress, produce result chunks that get aggregated into a delete expunge report diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4CreateTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4CreateTest.java index d32a6100556..97bf8f640a4 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4CreateTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4CreateTest.java @@ -1408,7 +1408,7 @@ public class FhirResourceDaoR4CreateTest extends BaseJpaR4Test { final List pidOrThrowException1 = List.of(pidOrThrowException); final TransactionTemplate transactionTemplate = new TransactionTemplate(getTxManager()); - transactionTemplate.execute(x -> myDeleteExpungeSvc.deleteExpunge(pidOrThrowException1, true, 10, "")); + transactionTemplate.execute(x -> myDeleteExpungeSvc.deleteExpunge(pidOrThrowException1, true, 10, null)); } } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java index 9d6150deb28..768205bf718 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java @@ -8,6 +8,7 @@ import ca.uhn.fhir.batch2.jobs.chunk.TypedPidJson; import ca.uhn.fhir.batch2.jobs.expunge.DeleteExpungeStep; import ca.uhn.fhir.batch2.jobs.reindex.ReindexJobParameters; import ca.uhn.fhir.batch2.jobs.reindex.ReindexStep; +import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.ValidationSupportContext; import ca.uhn.fhir.context.support.ValueSetExpansionOptions; @@ -853,9 +854,12 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test IJobDataSink sink = mock(IJobDataSink.class); + WorkChunk workChunk = mock(WorkChunk.class); + when(workChunk.getId()).thenReturn("chunk-id"); + // Test myCaptureQueriesListener.clear(); - RunOutcome outcome = myDeleteExpungeStep.doDeleteExpunge(new ResourceIdListWorkChunkJson(pids, null), sink, "instance-id", "chunk-id", false, null); + RunOutcome outcome = myDeleteExpungeStep.doDeleteExpunge(new ResourceIdListWorkChunkJson(pids, null), sink, "instance-id", workChunk, false, null); // Verify assertEquals(1, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/DeleteExpungeSvcImplTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/DeleteExpungeSvcImplTest.java index cbf53823984..4053a6c2a55 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/DeleteExpungeSvcImplTest.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/DeleteExpungeSvcImplTest.java @@ -1,5 +1,6 @@ package ca.uhn.fhir.jpa.delete; +import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; import ca.uhn.fhir.jpa.delete.batch2.DeleteExpungeSqlBuilder; import ca.uhn.fhir.jpa.delete.batch2.DeleteExpungeSvcImpl; @@ -48,8 +49,10 @@ public class DeleteExpungeSvcImplTest { logger.addAppender(myAppender); when(myDeleteExpungeSqlBuilder.convertPidsToDeleteExpungeSql(Collections.emptyList(), false, 1)).thenReturn(mock(DeleteExpungeSqlBuilder.DeleteExpungeSqlResult.class)); + WorkChunk workChunk = mock(WorkChunk.class); + when(workChunk.getId()).thenReturn("abc-123"); - myDeleteExpungeSvc.deleteExpunge(Collections.emptyList(), false, 1, "abc-123"); + myDeleteExpungeSvc.deleteExpunge(Collections.emptyList(), false, 1, workChunk); verify(myAppender, atLeastOnce()).doAppend(myLoggingEvent.capture()); List events = myLoggingEvent.getAllValues(); assertEquals(Level.INFO, events.get(0).getLevel()); diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/interceptor/MdmStorageInterceptor.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/interceptor/MdmStorageInterceptor.java index f4c85d124f0..95ddc795bd3 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/interceptor/MdmStorageInterceptor.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/interceptor/MdmStorageInterceptor.java @@ -352,7 +352,7 @@ public class MdmStorageInterceptor implements IMdmStorageInterceptor { @SuppressWarnings("unchecked") private int deleteExpungeGoldenResource(IResourcePersistentId theGoldenPid) { IDeleteExpungeSvc deleteExpungeSvc = myIMdmClearHelperSvc.getDeleteExpungeSvc(); - return deleteExpungeSvc.deleteExpunge(new ArrayList<>(Collections.singleton(theGoldenPid)), false, null, ""); + return deleteExpungeSvc.deleteExpunge(new ArrayList<>(Collections.singleton(theGoldenPid)), false, null, null); } private void forbidIfModifyingExternalEidOnTarget(IBaseResource theNewResource, IBaseResource theOldResource) { diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeStep.java index de0e6843b83..5e9e76443bd 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeStep.java @@ -26,6 +26,7 @@ import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.api.VoidModel; import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson; +import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; import ca.uhn.fhir.jpa.api.svc.IIdHelperService; import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; @@ -75,7 +76,7 @@ public class DeleteExpungeStep data, theDataSink, theStepExecutionDetails.getInstance().getInstanceId(), - theStepExecutionDetails.getChunkId(), + theStepExecutionDetails.getWorkChunk(), cascade, cascadeMaxRounds); } @@ -85,7 +86,7 @@ public class DeleteExpungeStep ResourceIdListWorkChunkJson theData, IJobDataSink theDataSink, String theInstanceId, - String theChunkId, + WorkChunk theWorkChunk, boolean theCascade, Integer theCascadeMaxRounds) { RequestDetails requestDetails = new SystemRequestDetails(); @@ -96,7 +97,7 @@ public class DeleteExpungeStep transactionDetails, theDataSink, theInstanceId, - theChunkId, + theWorkChunk, theCascade, theCascadeMaxRounds); myHapiTransactionService @@ -113,7 +114,7 @@ public class DeleteExpungeStep private final RequestDetails myRequestDetails; private final TransactionDetails myTransactionDetails; private final IJobDataSink myDataSink; - private final String myChunkId; + private final WorkChunk myWorkChunk; private final String myInstanceId; private final boolean myCascade; private final Integer myCascadeMaxRounds; @@ -125,7 +126,7 @@ public class DeleteExpungeStep TransactionDetails theTransactionDetails, IJobDataSink theDataSink, String theInstanceId, - String theChunkId, + WorkChunk theWorkChunk, boolean theCascade, Integer theCascadeMaxRounds) { myData = theData; @@ -133,7 +134,7 @@ public class DeleteExpungeStep myTransactionDetails = theTransactionDetails; myDataSink = theDataSink; myInstanceId = theInstanceId; - myChunkId = theChunkId; + myWorkChunk = theWorkChunk; myCascade = theCascade; myCascadeMaxRounds = theCascadeMaxRounds; } @@ -146,12 +147,13 @@ public class DeleteExpungeStep public Void doInTransaction(@Nonnull TransactionStatus theStatus) { List persistentIds = myData.getResourcePersistentIds(myIdHelperService); + String workChunkId = myWorkChunk.getId(); if (persistentIds.isEmpty()) { ourLog.info( "Starting delete expunge work chunk. There are no resources to delete expunge - Instance[{}] Chunk[{}]", myInstanceId, - myChunkId); + workChunkId); return null; } @@ -159,15 +161,15 @@ public class DeleteExpungeStep "Starting delete expunge work chunk with {} resources - Instance[{}] Chunk[{}]", persistentIds.size(), myInstanceId, - myChunkId); + workChunkId); - myRecordCount = myDeleteExpungeSvc.deleteExpunge(persistentIds, myCascade, myCascadeMaxRounds, myChunkId); + myRecordCount = myDeleteExpungeSvc.deleteExpunge(persistentIds, myCascade, myCascadeMaxRounds, myWorkChunk); ourLog.info( "Delete expunge finished deleting {} resources - Instance[{}] Chunk[{}]", myRecordCount, myInstanceId, - myChunkId); + workChunkId); return null; } diff --git a/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/clear/MdmClearStep.java b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/clear/MdmClearStep.java index d0b3b7d39ba..49f752d23fb 100644 --- a/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/clear/MdmClearStep.java +++ b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/clear/MdmClearStep.java @@ -26,6 +26,7 @@ import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.api.VoidModel; import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson; +import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; import ca.uhn.fhir.jpa.api.svc.IIdHelperService; import ca.uhn.fhir.jpa.api.svc.IMdmClearHelperSvc; @@ -96,7 +97,7 @@ public class MdmClearStep implements IJobStepWorker> { - int deleteExpunge(List thePersistentIds, boolean theCascade, Integer theCascadeMaxRounds, String theChunkid); + int deleteExpunge( + List thePersistentIds, boolean theCascade, Integer theCascadeMaxRounds, PT theWorkChunk); boolean isCascadeSupported(); }