From bf495e2d92567f45b536068b49f70a41328bec85 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Mon, 27 Feb 2023 22:19:28 -0500 Subject: [PATCH] Don't merge yet: MegaScale API Changes 2 (#4544) * Megascale changes 2 * Cleanup * CLeanup * Sync master * Cleanup * Ongoing work * Move service * Fixes * Ongoing megascaler work * Work * Work * Fixes * Work * Test fixes * Work on remote services * Interceptor rework * Work on config refactor * Fixes * Docs tweaks * Address review comments * Test tweak * Test fixes * Try to fix tests * Test tweaks * Fix changelogs * Work on merging * Changes * Megascale fixes * Remove accidental commit * Address fixmes * Cleanup * Test fixes * Compile fix * Test fixes * Account for review comments * Bump HAPI FHIR version --- hapi-deployable-pom/pom.xml | 2 +- hapi-fhir-android/pom.xml | 2 +- hapi-fhir-base/pom.xml | 2 +- .../ca/uhn/fhir/interceptor/api/Pointcut.java | 47 ++- .../interceptor/model/RequestPartitionId.java | 12 +- .../java/ca/uhn/fhir/rest/api/Constants.java | 6 +- hapi-fhir-bom/pom.xml | 4 +- hapi-fhir-checkstyle/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-api/pom.xml | 6 +- hapi-fhir-cli/hapi-fhir-cli-app/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml | 2 +- hapi-fhir-cli/pom.xml | 2 +- hapi-fhir-client-okhttp/pom.xml | 2 +- hapi-fhir-client/pom.xml | 2 +- .../interceptor/LoggingInterceptor.java | 18 +- hapi-fhir-converter/pom.xml | 2 +- hapi-fhir-dist/pom.xml | 2 +- hapi-fhir-docs/pom.xml | 2 +- ...44-externally-stored-resource-jpa-api.yaml | 6 + ...544-hapi-transaction-service-adoption.yaml | 7 + .../fhir/docs/interceptors/interceptors.md | 3 +- hapi-fhir-jacoco/pom.xml | 2 +- hapi-fhir-jaxrsserver-base/pom.xml | 2 +- hapi-fhir-jpa/pom.xml | 2 +- hapi-fhir-jpaserver-base/pom.xml | 2 +- .../ca/uhn/fhir/jpa/config/JpaConfig.java | 14 +- .../ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java | 121 ++++--- .../fhir/jpa/dao/BaseHapiFhirResourceDao.java | 192 +++++++---- .../fhir/jpa/dao/BaseHapiFhirSystemDao.java | 39 ++- .../jpa/dao/JpaStorageResourceParser.java | 23 +- .../dao/expunge/ExpungeEverythingService.java | 169 +++++----- .../fhir/jpa/entity/BulkExportJobEntity.java | 8 +- .../fhir/jpa/entity/BulkImportJobEntity.java | 7 +- .../jpa/entity/ResourceReindexJobEntity.java | 5 +- .../java/ca/uhn/fhir/jpa/entity/Search.java | 31 +- .../esr/ExternallyStoredResourceAddress.java | 52 +++ ...nallyStoredResourceAddressMetadataKey.java | 39 +++ ...ternallyStoredResourceServiceRegistry.java | 69 ++++ .../esr/IExternallyStoredResourceService.java | 41 +++ .../ca/uhn/fhir/jpa/esr/package-info.java | 34 ++ .../tasks/HapiFhirJpaMigrationTasks.java | 41 ++- .../provider/JpaConformanceProviderDstu2.java | 2 +- .../search/DatabaseBackedPagingProvider.java | 16 +- .../search/PersistedJpaBundleProvider.java | 64 +++- .../PersistedJpaBundleProviderFactory.java | 16 +- ...istedJpaSearchFirstPageBundleProvider.java | 19 +- .../jpa/search/SearchCoordinatorSvcImpl.java | 119 +++---- .../jpa/search/SynchronousSearchSvcImpl.java | 11 +- .../jpa/search/builder/SearchBuilder.java | 2 +- .../StorageInterceptorHooksFacade.java | 8 +- .../builder/tasks/SearchContinuationTask.java | 9 +- .../jpa/search/builder/tasks/SearchTask.java | 69 ++-- .../cache/DatabaseSearchCacheSvcImpl.java | 1 - .../DatabaseSearchResultCacheSvcImpl.java | 80 +++-- .../search/cache/ISearchResultCacheSvc.java | 12 +- .../jpa/util/JpaHapiTransactionService.java | 45 +++ .../pom.xml | 2 +- hapi-fhir-jpaserver-ips/pom.xml | 2 +- hapi-fhir-jpaserver-mdm/pom.xml | 2 +- .../jpa/mdm/svc/MdmControllerSvcImpl.java | 30 +- .../mdm/provider/MdmProviderMatchR4Test.java | 2 +- .../jpa/mdm/svc/MdmControllerSvcImplTest.java | 16 +- hapi-fhir-jpaserver-model/pom.xml | 2 +- .../jpa/model/config/PartitionSettings.java | 38 ++- .../model/cross/IBasePersistedResource.java | 3 +- .../fhir/jpa/model/cross/IResourceLookup.java | 2 +- .../dialect/HapiSequenceStyleGenerator.java | 8 +- .../model/dialect/ISequenceValueMassager.java | 13 + .../jpa/model/entity/BaseHasResource.java | 9 +- .../entity/PartitionablePartitionId.java | 11 + .../model/entity/ResourceEncodingEnum.java | 11 +- .../model/entity/ResourceHistoryTable.java | 1 + .../fhir/jpa/model/entity/ResourceLink.java | 3 + .../fhir/jpa/model/entity/ResourceTable.java | 12 +- .../jpa/model/entity/StorageSettings.java | 4 + hapi-fhir-jpaserver-searchparam/pom.xml | 2 +- .../model/ReadPartitionIdRequestDetails.java | 72 +++- .../fhir/jpa/cache/IResourceVersionSvc.java | 4 +- .../fhir/jpa/cache/ResourceVersionMap.java | 4 +- .../partition/IRequestPartitionHelperSvc.java | 19 +- .../extractor/BaseSearchParamExtractor.java | 2 +- .../CrossPartitionReferenceDetails.java | 78 +++++ .../SearchParamExtractorService.java | 57 ++-- .../SearchParamExtractorServiceTest.java | 4 +- hapi-fhir-jpaserver-subscription/pom.xml | 2 +- .../SubscriptionTriggeringSvcImpl.java | 18 +- .../matching/DaoSubscriptionMatcherTest.java | 6 + .../module/BaseSubscriptionTest.java | 8 + .../SubscriptionMatchingSubscriberTest.java | 27 +- hapi-fhir-jpaserver-test-dstu2/pom.xml | 2 +- .../FhirResourceDaoDstu2ValidateTest.java | 4 +- .../provider/ResourceProviderDstu2Test.java | 4 +- .../search/SearchCoordinatorSvcImplTest.java | 139 ++++---- .../src/test}/resources/logback-test.xml | 0 hapi-fhir-jpaserver-test-dstu3/pom.xml | 2 +- .../dstu3/ResourceProviderDstu3Test.java | 21 +- ...rceProviderDstu3ValueSetVersionedTest.java | 8 +- .../src/test/resources/logback-test.xml | 46 +++ hapi-fhir-jpaserver-test-r4/pom.xml | 2 +- .../fhir/jpa/batch2/Batch2CoordinatorIT.java | 6 +- .../jpa/bulk/BulkDataExportProviderTest.java | 224 ++++++------- .../jpa/dao/r4/BasePartitioningR4Test.java | 6 +- ...hirResourceDaoR4ExternalReferenceTest.java | 20 +- .../r4/FhirResourceDaoR4SearchNoFtTest.java | 29 +- .../r4/FhirResourceDaoR4TerminologyTest.java | 4 +- .../dao/r4/FhirResourceDaoR4ValidateTest.java | 1 + .../fhir/jpa/dao/r4/FhirSystemDaoR4Test.java | 2 +- .../jpa/dao/r4/PartitioningSqlR4Test.java | 8 +- .../PartitioningInterceptorR4Test.java | 10 +- .../PatientIdPartitionInterceptorTest.java | 4 +- .../jpa/packages/JpaPackageCacheTest.java | 4 +- .../ca/uhn/fhir/jpa/packages/NpmR4Test.java | 56 ++-- .../provider/r4/PatientEverythingR4Test.java | 38 ++- .../r4/PatientMemberMatchOperationR4Test.java | 316 +++++++++--------- ...minologyServiceResourceProviderR4Test.java | 4 +- .../r4/ResourceProviderHasParamR4Test.java | 2 +- .../r4/ResourceProviderInterceptorR4Test.java | 19 +- ...ProviderR4ValueSetHSearchDisabledTest.java | 111 +----- ...rceProviderR4ValueSetNoVerCSNoVerTest.java | 4 +- ...ourceProviderR4ValueSetVerCSNoVerTest.java | 4 +- ...esourceProviderR4ValueSetVerCSVerTest.java | 8 +- .../ResourceReindexingSvcImplTest.java | 17 +- .../stresstest/GiantTransactionPerfTest.java | 17 +- .../ca/uhn/fhir/jpa/term/BaseTermR4Test.java | 8 +- .../jpa/term/TerminologySvcImplR4Test.java | 3 + .../src/test/resources/logback-test.xml | 46 +++ hapi-fhir-jpaserver-test-r4b/pom.xml | 2 +- .../src/test/resources/logback-test.xml | 46 +++ hapi-fhir-jpaserver-test-r5/pom.xml | 2 +- .../ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java | 3 + .../dao/r5/CrossPartitionReferencesTest.java | 247 ++++++++++++++ .../r5/ExternallyStoredResourceR5Test.java | 86 +++++ .../r5/ResourceProviderR5ValueSetTest.java | 2 +- ...sourceProviderR5ValueSetVersionedTest.java | 8 +- .../src/test/resources/logback-test.xml | 46 +++ hapi-fhir-jpaserver-test-utilities/pom.xml | 2 +- .../r4/BaseResourceProviderR4Test.java | 3 + .../ca/uhn/fhir/jpa/test/BaseJpaTest.java | 3 +- .../fhir/jpa/test/config/TestR5Config.java | 1 + hapi-fhir-jpaserver-uhnfhirtest/pom.xml | 2 +- hapi-fhir-server-mdm/pom.xml | 2 +- .../mdm/provider/MdmControllerHelper.java | 4 +- hapi-fhir-server-openapi/pom.xml | 2 +- hapi-fhir-server/pom.xml | 2 +- .../util/CompositeInterceptorBroadcaster.java | 1 + .../server/util/ICachedSearchDetails.java | 4 + .../hapi-fhir-caching-api/pom.xml | 2 +- .../hapi-fhir-caching-caffeine/pom.xml | 4 +- .../hapi-fhir-caching-guava/pom.xml | 2 +- .../hapi-fhir-caching-testing/pom.xml | 2 +- hapi-fhir-serviceloaders/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../hapi-fhir-spring-boot-samples/pom.xml | 2 +- .../hapi-fhir-spring-boot-starter/pom.xml | 2 +- hapi-fhir-spring-boot/pom.xml | 2 +- hapi-fhir-sql-migrate/pom.xml | 2 +- hapi-fhir-storage-batch2-jobs/pom.xml | 2 +- .../DeleteExpungeJobSubmitterImpl.java | 8 +- .../batch2/jobs/reindex/ReindexProvider.java | 5 +- .../jobs/reindex/ReindexProviderTest.java | 2 +- hapi-fhir-storage-batch2/pom.xml | 2 +- hapi-fhir-storage-cr/pom.xml | 2 +- .../src/test/resources/logback-test.xml | 15 + hapi-fhir-storage-mdm/pom.xml | 2 +- hapi-fhir-storage-test-utilities/pom.xml | 2 +- hapi-fhir-storage/pom.xml | 33 +- .../ca/uhn/fhir/jpa/api/IDaoRegistry.java | 2 +- .../fhir/jpa/api/dao/IFhirResourceDao.java | 33 +- .../jpa/api/model/DeleteMethodOutcome.java | 12 +- .../jpa/api/svc/ISearchCoordinatorSvc.java | 2 +- .../jpa/dao/BaseTransactionProcessor.java | 4 +- .../jpa/dao/tx/HapiTransactionService.java | 80 ++++- .../PatientIdPartitionInterceptor.java | 12 +- .../model/search/SearchRuntimeDetails.java | 2 +- .../jpa/model/search/SearchStatusEnum.java | 2 +- .../BaseRequestPartitionHelperSvc.java | 49 ++- .../jpa/util/BaseCaptureQueriesListener.java | 8 +- .../CircularQueueCaptureQueriesListener.java | 6 +- .../CurrentThreadCaptureQueriesListener.java | 2 +- .../java/ca/uhn/fhir/jpa/util/SqlQuery.java | 15 +- .../ca/uhn/fhir/jpa/util/SqlQueryList.java | 2 +- hapi-fhir-structures-dstu2.1/pom.xml | 2 +- hapi-fhir-structures-dstu2/pom.xml | 2 +- hapi-fhir-structures-dstu3/pom.xml | 2 +- .../uhn/fhir/rest/server/UpdateDstu3Test.java | 79 ++--- hapi-fhir-structures-hl7org-dstu2/pom.xml | 2 +- hapi-fhir-structures-r4/pom.xml | 2 +- hapi-fhir-structures-r4b/pom.xml | 2 +- hapi-fhir-structures-r5/pom.xml | 2 +- hapi-fhir-test-utilities/pom.xml | 2 +- hapi-fhir-testpage-overlay/pom.xml | 2 +- .../pom.xml | 2 +- hapi-fhir-validation-resources-dstu2/pom.xml | 2 +- hapi-fhir-validation-resources-dstu3/pom.xml | 2 +- hapi-fhir-validation-resources-r4/pom.xml | 2 +- hapi-fhir-validation-resources-r5/pom.xml | 2 +- hapi-fhir-validation/pom.xml | 2 +- hapi-tinder-plugin/pom.xml | 2 +- hapi-tinder-test/pom.xml | 2 +- pom.xml | 8 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- 206 files changed, 2721 insertions(+), 1378 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4544-externally-stored-resource-jpa-api.yaml create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4544-hapi-transaction-service-adoption.yaml create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/esr/ExternallyStoredResourceAddress.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/esr/ExternallyStoredResourceAddressMetadataKey.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/esr/ExternallyStoredResourceServiceRegistry.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/esr/IExternallyStoredResourceService.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/esr/package-info.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/JpaHapiTransactionService.java rename {hapi-fhir-base => hapi-fhir-jpaserver-searchparam}/src/main/java/ca/uhn/fhir/interceptor/model/ReadPartitionIdRequestDetails.java (51%) rename {hapi-fhir-storage => hapi-fhir-jpaserver-searchparam}/src/main/java/ca/uhn/fhir/jpa/partition/IRequestPartitionHelperSvc.java (81%) create mode 100644 hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/CrossPartitionReferenceDetails.java rename {hapi-fhir-jpaserver-test-utilities/src/main => hapi-fhir-jpaserver-test-dstu2/src/test}/resources/logback-test.xml (100%) create mode 100644 hapi-fhir-jpaserver-test-dstu3/src/test/resources/logback-test.xml create mode 100644 hapi-fhir-jpaserver-test-r4/src/test/resources/logback-test.xml create mode 100644 hapi-fhir-jpaserver-test-r4b/src/test/resources/logback-test.xml create mode 100644 hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/CrossPartitionReferencesTest.java create mode 100644 hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/ExternallyStoredResourceR5Test.java create mode 100644 hapi-fhir-jpaserver-test-r5/src/test/resources/logback-test.xml create mode 100644 hapi-fhir-storage-cr/src/test/resources/logback-test.xml rename {hapi-fhir-jpaserver-model => hapi-fhir-storage}/src/main/java/ca/uhn/fhir/jpa/api/IDaoRegistry.java (96%) rename {hapi-fhir-jpaserver-model => hapi-fhir-storage}/src/main/java/ca/uhn/fhir/jpa/model/search/SearchRuntimeDetails.java (99%) rename {hapi-fhir-jpaserver-model => hapi-fhir-storage}/src/main/java/ca/uhn/fhir/jpa/model/search/SearchStatusEnum.java (98%) rename {hapi-fhir-jpa => hapi-fhir-storage}/src/main/java/ca/uhn/fhir/jpa/util/BaseCaptureQueriesListener.java (93%) rename {hapi-fhir-jpa => hapi-fhir-storage}/src/main/java/ca/uhn/fhir/jpa/util/CircularQueueCaptureQueriesListener.java (98%) rename {hapi-fhir-jpa => hapi-fhir-storage}/src/main/java/ca/uhn/fhir/jpa/util/CurrentThreadCaptureQueriesListener.java (99%) rename {hapi-fhir-jpa => hapi-fhir-storage}/src/main/java/ca/uhn/fhir/jpa/util/SqlQuery.java (88%) rename {hapi-fhir-jpa => hapi-fhir-storage}/src/main/java/ca/uhn/fhir/jpa/util/SqlQueryList.java (97%) diff --git a/hapi-deployable-pom/pom.xml b/hapi-deployable-pom/pom.xml index 23caed22e13..85b3274978a 100644 --- a/hapi-deployable-pom/pom.xml +++ b/hapi-deployable-pom/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.5.2-SNAPSHOT + 6.5.3-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-android/pom.xml b/hapi-fhir-android/pom.xml index fbdd23757c7..c218688d43e 100644 --- a/hapi-fhir-android/pom.xml +++ b/hapi-fhir-android/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.5.2-SNAPSHOT + 6.5.3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-base/pom.xml b/hapi-fhir-base/pom.xml index bc4d947753c..8ca362f2370 100644 --- a/hapi-fhir-base/pom.xml +++ b/hapi-fhir-base/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.5.2-SNAPSHOT + 6.5.3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java index 70a6c80a4f3..dd92309a50e 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java @@ -628,6 +628,13 @@ public enum Pointcut implements IPointcut { * This method is called after all processing is completed for a request, but only if the * request completes normally (i.e. no exception is thrown). *

+ * This pointcut is called after the response has completely finished, meaning that the HTTP respsonse to the client + * may or may not have already completely been returned to the client by the time this pointcut is invoked. Use caution + * if you have timing-dependent logic, since there is no guarantee about whether the client will have already moved on + * by the time your method is invoked. If you need a guarantee that your method is invoked before returning to the + * client, consider using {@link #SERVER_OUTGOING_RESPONSE} instead. + *

+ *

* Hooks may accept the following parameters: *