From 27b52af892c9da4415c8c73da4b3d66eba2065d4 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Fri, 9 Jun 2023 11:52:18 -0400 Subject: [PATCH] added architecture for git expunge svc --- .../fhir/jpa/bulk/mdm/MDMClearHelperSvcImpl.java | 16 ++++++++++++++++ .../java/ca/uhn/fhir/jpa/config/JpaConfig.java | 7 +++++++ .../jpa/delete/batch2/DeleteExpungeSvcImpl.java | 4 ++-- .../fhir/jpa/mdm/config/MdmConsumerConfig.java | 5 ++++- .../ca/uhn/fhir/jpa/config/JpaEntityTest.java | 1 - .../fhir/batch2/coordinator/StepExecutor.java | 3 ++- .../src/main/java/ca/uhn/fhir/mdm/.gitkeep | 0 .../uhn/fhir/mdm/batch2/clear/MdmClearStep.java | 13 ++++++++++++- .../fhir/jpa/api/config/JpaStorageSettings.java | 2 ++ .../fhir/jpa/bulk/mdm/IMdmClearHelperSvc.java | 9 +++++++++ .../jpa/JpaModelScannerAndVerifier.java | 10 +++++++++- 11 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/mdm/MDMClearHelperSvcImpl.java create mode 100644 hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/.gitkeep create mode 100644 hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/mdm/IMdmClearHelperSvc.java diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/mdm/MDMClearHelperSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/mdm/MDMClearHelperSvcImpl.java new file mode 100644 index 00000000000..4eb0242ebff --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/mdm/MDMClearHelperSvcImpl.java @@ -0,0 +1,16 @@ +package ca.uhn.fhir.jpa.bulk.mdm; + +import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; +import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; +import org.springframework.beans.factory.annotation.Autowired; + +public class MDMClearHelperSvcImpl implements IMdmClearHelperSvc { + + @Autowired + IDeleteExpungeSvc myDeleteExpungeSvc; + + @Override + public IDeleteExpungeSvc> getDeleteExpungeService() { + return myDeleteExpungeSvc; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java index d3868db6f81..95ede8605d6 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java @@ -41,6 +41,8 @@ import ca.uhn.fhir.jpa.bulk.export.svc.BulkDataExportJobSchedulingHelperImpl; import ca.uhn.fhir.jpa.bulk.export.svc.BulkExportHelperService; import ca.uhn.fhir.jpa.bulk.imprt.api.IBulkDataImportSvc; import ca.uhn.fhir.jpa.bulk.imprt.svc.BulkDataImportSvcImpl; +import ca.uhn.fhir.jpa.bulk.mdm.IMdmClearHelperSvc; +import ca.uhn.fhir.jpa.bulk.mdm.MDMClearHelperSvcImpl; import ca.uhn.fhir.jpa.cache.IResourceVersionSvc; import ca.uhn.fhir.jpa.cache.ResourceVersionSvcDaoImpl; import ca.uhn.fhir.jpa.dao.DaoSearchParamProvider; @@ -831,4 +833,9 @@ public class JpaConfig { public ISearchUrlJobMaintenanceSvc searchUrlJobMaintenanceSvc(ResourceSearchUrlSvc theResourceSearchUrlSvc){ return new SearchUrlJobMaintenanceSvcImpl(theResourceSearchUrlSvc); } + + @Bean + public IMdmClearHelperSvc helperSvc() { + return new MDMClearHelperSvcImpl(); + } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java index 5bf0725652b..2a8e83fafbd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java @@ -49,10 +49,10 @@ public class DeleteExpungeSvcImpl implements IDeleteExpungeSvc { DeleteExpungeSqlBuilder.DeleteExpungeSqlResult sqlResult = myDeleteExpungeSqlBuilder.convertPidsToDeleteExpungeSql(theJpaPids, theCascade, theCascadeMaxRounds); List sqlList = sqlResult.getSqlStatements(); - ourLog.debug("Executing {} delete expunge sql commands", sqlList.size()); + ourLog.info("Executing {} delete expunge sql commands", sqlList.size()); long totalDeleted = 0; for (String sql : sqlList) { - ourLog.trace("Executing sql " + sql); + ourLog.info("Executing sql " + sql); totalDeleted += myEntityManager.createNativeQuery(sql).executeUpdate(); } diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java index 1593d5c27a4..95d9ccc329e 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java @@ -81,7 +81,10 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @Configuration -@Import({MdmCommonConfig.class, MdmBatch2Config.class}) +@Import({ + MdmCommonConfig.class, + MdmBatch2Config.class +}) public class MdmConsumerConfig { private static final Logger ourLog = Logs.getMdmTroubleshootingLog(); diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/config/JpaEntityTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/config/JpaEntityTest.java index aef5e4831d0..51d7fbfef30 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/config/JpaEntityTest.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/config/JpaEntityTest.java @@ -13,5 +13,4 @@ public class JpaEntityTest { ); } - } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java index 585e361bab6..40dac2d5ec0 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java @@ -70,10 +70,11 @@ public class StepExecutor { return false; } catch (Exception e) { if (theStepExecutionDetails.hasAssociatedWorkChunk()) { - ourLog.error("Failure executing job {} step {}, marking chunk {} as ERRORED", jobDefinitionId, targetStepId, chunkId, e); + ourLog.info("Failure executing job {} step {}, marking chunk {} as ERRORED. Retrying", jobDefinitionId, targetStepId, chunkId, e); WorkChunkErrorEvent parameters = new WorkChunkErrorEvent(chunkId, e.getMessage()); WorkChunkStatusEnum newStatus = myJobPersistence.onWorkChunkError(parameters); if (newStatus == WorkChunkStatusEnum.FAILED) { + ourLog.error("Failure executing job {} step {}, marking chunk {} as ERRORED. Retries expired", jobDefinitionId, targetStepId, chunkId, e); return false; } } else { diff --git a/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/.gitkeep b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/clear/MdmClearStep.java b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/clear/MdmClearStep.java index 993e6284d56..37bf7bedd02 100644 --- a/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/clear/MdmClearStep.java +++ b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/clear/MdmClearStep.java @@ -30,7 +30,9 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.api.model.DeleteConflictList; +import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; import ca.uhn.fhir.jpa.api.svc.IIdHelperService; +import ca.uhn.fhir.jpa.bulk.mdm.IMdmClearHelperSvc; import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; import ca.uhn.fhir.jpa.delete.DeleteConflictUtil; import ca.uhn.fhir.jpa.model.dao.JpaPid; @@ -68,6 +70,9 @@ public class MdmClearStep implements IJobStepWorker theStepExecutionDetails, @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { @@ -126,10 +131,16 @@ public class MdmClearStep implements IJobStepWorker dao = myDaoRegistry.getResourceDao(resourceName); DeleteConflictList conflicts = new DeleteConflictList(); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java index 570f89e7194..a0970956622 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java @@ -321,6 +321,8 @@ public class JpaStorageSettings extends StorageSettings { */ public JpaStorageSettings() { setMarkResourcesForReindexingUponSearchParameterChange(true); +// setReindexThreadCount(1); +// setExpungeThreadCount(1); setReindexThreadCount(Runtime.getRuntime().availableProcessors()); setExpungeThreadCount(Runtime.getRuntime().availableProcessors()); setBundleTypesAllowedForStorage(DEFAULT_BUNDLE_TYPES_ALLOWED_FOR_STORAGE); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/mdm/IMdmClearHelperSvc.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/mdm/IMdmClearHelperSvc.java new file mode 100644 index 00000000000..b3a8e36c258 --- /dev/null +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/mdm/IMdmClearHelperSvc.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.jpa.bulk.mdm; + +import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; +import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; + +public interface IMdmClearHelperSvc { + + IDeleteExpungeSvc> getDeleteExpungeService(); +} diff --git a/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/jpa/JpaModelScannerAndVerifier.java b/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/jpa/JpaModelScannerAndVerifier.java index de6732ed017..4f02bba2b7c 100644 --- a/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/jpa/JpaModelScannerAndVerifier.java +++ b/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/jpa/JpaModelScannerAndVerifier.java @@ -91,6 +91,9 @@ public class JpaModelScannerAndVerifier { "FK_SEARCHINC_SEARCH" ); private static Set ourReservedWords; + + + public JpaModelScannerAndVerifier() { super(); } @@ -264,7 +267,12 @@ public class JpaModelScannerAndVerifier { scanClassOrSuperclass(theNames, theClazz.getSuperclass(), true, columnNameToLength); } - private void scan(AnnotatedElement theAnnotatedElement, Set theNames, boolean theIsSuperClass, boolean theIsView) { + private void scan( + AnnotatedElement theAnnotatedElement, + Set theNames, + boolean theIsSuperClass, + boolean theIsView + ) { Table table = theAnnotatedElement.getAnnotation(Table.class); if (table != null) {