From 24b3f0f30d211d9f680b4244b09a77b411e690db Mon Sep 17 00:00:00 2001 From: James Agnew Date: Thu, 27 May 2021 18:43:51 -0400 Subject: [PATCH] Enable mass ingestion mode (#2681) * Work on fixes * Work on counts * Enable mass ingestion mode * Add changelog * Test fix * Test fix * Test fixes * Fixes * Test fix * Test fix --- .../2681-enable-mass-ingestion-mode.yaml | 5 + .../ca/uhn/fhir/jpa/api/config/DaoConfig.java | 35 + .../ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java | 86 ++- .../fhir/jpa/dao/BaseHapiFhirResourceDao.java | 44 +- .../BaseHapiFhirResourceDaoObservation.java | 2 +- .../fhir/jpa/dao/BaseHapiFhirSystemDao.java | 2 + .../dstu3/FhirResourceDaoValueSetDstu3.java | 2 +- .../fhir/jpa/dao/index/IdHelperService.java | 25 +- .../jpa/dao/r4/FhirResourceDaoValueSetR4.java | 2 +- .../jpa/dao/r5/FhirResourceDaoValueSetR5.java | 2 +- .../uhn/fhir/jpa/util/MemoryCacheService.java | 21 +- .../ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java | 3 + .../r4/FhirResourceDaoR4QueryCountTest.java | 93 ++- .../FhirResourceDaoR4SearchOptimizedTest.java | 35 +- .../FhirResourceDaoSearchParameterR4Test.java | 2 +- .../jpa/dao/r4/PartitioningSqlR4Test.java | 6 +- .../stresstest/GiantTransactionPerfTest.java | 5 +- .../src/test/resources/r4/eob-bundle.json | 659 ++++++++++++++++++ .../fhir/jpa/model/entity/ResourceTable.java | 9 +- .../registry/ReadOnlySearchParamCache.java | 1 + .../registry/SearchParamRegistryImpl.java | 3 +- 21 files changed, 948 insertions(+), 94 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_5_0/2681-enable-mass-ingestion-mode.yaml create mode 100644 hapi-fhir-jpaserver-base/src/test/resources/r4/eob-bundle.json diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_5_0/2681-enable-mass-ingestion-mode.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_5_0/2681-enable-mass-ingestion-mode.yaml new file mode 100644 index 00000000000..8288c7967b5 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_5_0/2681-enable-mass-ingestion-mode.yaml @@ -0,0 +1,5 @@ +--- +type: add +issue: 2681 +title: "A new DaoConfig setting called Mass Ingestion Mode has been added. This mode enables rapid + data ingestion by skipping a number of unnecessary checks during backloading." diff --git a/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java index 1699dc78be1..4139eb34a64 100644 --- a/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java +++ b/hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java @@ -240,6 +240,7 @@ public class DaoConfig { * @since 5.5.0 */ private boolean myEnableTaskBulkExportJobExecution; + private boolean myMassIngestionMode; private boolean myAccountForDateIndexNulls; private boolean myTriggerSubscriptionsForNonVersioningChanges; @@ -2365,6 +2366,40 @@ public class DaoConfig { return myEnableTaskResourceReindexing; } + /** + * If this is enabled (disabled by default), Mass Ingestion Mode is enabled. In this mode, a number of + * runtime checks are disabled. This mode is designed for rapid backloading of data while the system is not + * being otherwise used. + * + * In this mode: + * + * - Tags/Profiles/Security Labels will not be updated on existing resources that already have them + * - Resources modification checks will be skipped in favour of a simple hash check + * - Extra resource ID caching is enabled + * + * @since 5.5.0 + */ + public void setMassIngestionMode(boolean theMassIngestionMode) { + myMassIngestionMode = theMassIngestionMode; + } + + /** + * If this is enabled (disabled by default), Mass Ingestion Mode is enabled. In this mode, a number of + * runtime checks are disabled. This mode is designed for rapid backloading of data while the system is not + * being otherwise used. + * + * In this mode: + * + * - Tags/Profiles/Security Labels will not be updated on existing resources that already have them + * - Resources modification checks will be skipped in favour of a simple hash check + * - Extra resource ID caching is enabled + * + * @since 5.5.0 + */ + public boolean isMassIngestionMode() { + return myMassIngestionMode; + } + /** * If this is enabled (this is the default), this server will attempt to run resource reindexing jobs. * Otherwise, this server will not. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java index 3a1c8df1a33..eabc69b082c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java @@ -387,10 +387,6 @@ public abstract class BaseHapiFhirDao extends BaseStora return myConfig; } - public void setConfig(DaoConfig theConfig) { - myConfig = theConfig; - } - @Override public FhirContext getContext() { return myContext; @@ -608,49 +604,56 @@ public abstract class BaseHapiFhirDao extends BaseStora } - Set allDefs = new HashSet<>(); - Set allTagsOld = getAllTagDefinitions(theEntity); - - if (theResource instanceof IResource) { - extractTagsHapi(theTransactionDetails, (IResource) theResource, theEntity, allDefs); - } else { - extractTagsRi(theTransactionDetails, (IAnyResource) theResource, theEntity, allDefs); + boolean skipUpdatingTags = false; + if (myConfig.isMassIngestionMode() && theEntity.isHasTags()) { + skipUpdatingTags = true; } - RuntimeResourceDefinition def = myContext.getResourceDefinition(theResource); - if (def.isStandardType() == false) { - String profile = def.getResourceProfile(""); - if (isNotBlank(profile)) { - TagDefinition profileDef = getTagOrNull(theTransactionDetails, TagTypeEnum.PROFILE, NS_JPA_PROFILE, profile, null); + if (!skipUpdatingTags) { + Set allDefs = new HashSet<>(); + Set allTagsOld = getAllTagDefinitions(theEntity); - ResourceTag tag = theEntity.addTag(profileDef); - allDefs.add(tag); - theEntity.setHasTags(true); - } - } - - Set allTagsNew = getAllTagDefinitions(theEntity); - Set allDefsPresent = new HashSet<>(); - allTagsNew.forEach(tag -> { - - // Don't keep duplicate tags - if (!allDefsPresent.add(tag.getTag())) { - theEntity.getTags().remove(tag); + if (theResource instanceof IResource) { + extractTagsHapi(theTransactionDetails, (IResource) theResource, theEntity, allDefs); + } else { + extractTagsRi(theTransactionDetails, (IAnyResource) theResource, theEntity, allDefs); } - // Drop any tags that have been removed - if (!allDefs.contains(tag)) { - if (shouldDroppedTagBeRemovedOnUpdate(theRequest, tag)) { - theEntity.getTags().remove(tag); + RuntimeResourceDefinition def = myContext.getResourceDefinition(theResource); + if (def.isStandardType() == false) { + String profile = def.getResourceProfile(""); + if (isNotBlank(profile)) { + TagDefinition profileDef = getTagOrNull(theTransactionDetails, TagTypeEnum.PROFILE, NS_JPA_PROFILE, profile, null); + + ResourceTag tag = theEntity.addTag(profileDef); + allDefs.add(tag); + theEntity.setHasTags(true); } } - }); + Set allTagsNew = getAllTagDefinitions(theEntity); + Set allDefsPresent = new HashSet<>(); + allTagsNew.forEach(tag -> { - if (!allTagsOld.equals(allTagsNew)) { - changed = true; + // Don't keep duplicate tags + if (!allDefsPresent.add(tag.getTag())) { + theEntity.getTags().remove(tag); + } + + // Drop any tags that have been removed + if (!allDefs.contains(tag)) { + if (shouldDroppedTagBeRemovedOnUpdate(theRequest, tag)) { + theEntity.getTags().remove(tag); + } + } + + }); + + if (!allTagsOld.equals(allTagsNew)) { + changed = true; + } + theEntity.setHasTags(!allTagsNew.isEmpty()); } - theEntity.setHasTags(!allTagsNew.isEmpty()); } else { theEntity.setHashSha256(null); @@ -661,6 +664,10 @@ public abstract class BaseHapiFhirDao extends BaseStora if (thePerformIndexing && changed == false) { if (theEntity.getId() == null) { changed = true; + } else if (myConfig.isMassIngestionMode()) { + + // Don't check existing - We'll rely on the SHA256 hash only + } else { ResourceHistoryTable currentHistoryVersion = theEntity.getCurrentVersionEntity(); if (currentHistoryVersion == null) { @@ -1719,6 +1726,11 @@ public abstract class BaseHapiFhirDao extends BaseStora ourDisableIncrementOnUpdateForUnitTest = theDisableIncrementOnUpdateForUnitTest; } + @VisibleForTesting + public void setDaoConfigForUnitTest(DaoConfig theDaoConfig) { + myConfig = theDaoConfig; + } + /** * Do not call this method outside of unit tests */ diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java index cd3491f0d89..666b5d88d7e 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java @@ -151,8 +151,6 @@ public abstract class BaseHapiFhirResourceDao extends B @Autowired protected PlatformTransactionManager myPlatformTransactionManager; - @Autowired - protected DaoConfig myDaoConfig; @Autowired(required = false) protected IFulltextSearchSvc mySearchDao; @Autowired @@ -236,7 +234,7 @@ public abstract class BaseHapiFhirResourceDao extends B } } - if (myDaoConfig.getResourceServerIdStrategy() == DaoConfig.IdStrategyEnum.UUID) { + if (getConfig().getResourceServerIdStrategy() == DaoConfig.IdStrategyEnum.UUID) { theResource.setId(UUID.randomUUID().toString()); theResource.setUserData(JpaConstants.RESOURCE_ID_SERVER_ASSIGNED, Boolean.TRUE); } @@ -304,7 +302,7 @@ public abstract class BaseHapiFhirResourceDao extends B createForcedIdIfNeeded(entity, theResource.getIdElement(), true); serverAssignedId = true; } else { - switch (myDaoConfig.getResourceClientIdStrategy()) { + switch (getConfig().getResourceClientIdStrategy()) { case NOT_ALLOWED: throw new ResourceNotFoundException( getContext().getLocalizer().getMessageSanitized(BaseHapiFhirResourceDao.class, "failedToCreateWithClientAssignedIdNotAllowed", theResource.getIdElement().getIdPart())); @@ -344,7 +342,7 @@ public abstract class BaseHapiFhirResourceDao extends B theResource.setId(entity.getIdDt()); if (serverAssignedId) { - switch (myDaoConfig.getResourceClientIdStrategy()) { + switch (getConfig().getResourceClientIdStrategy()) { case NOT_ALLOWED: case ALPHANUMERIC: break; @@ -550,7 +548,7 @@ public abstract class BaseHapiFhirResourceDao extends B Set resourceIds = myMatchResourceUrlService.search(paramMap, myResourceType, theRequest); if (resourceIds.size() > 1) { - if (!myDaoConfig.isAllowMultipleDelete()) { + if (!getConfig().isAllowMultipleDelete()) { throw new PreconditionFailedException(getContext().getLocalizer().getMessageSanitized(BaseHapiFhirDao.class, "transactionOperationWithMultipleMatchFailure", "DELETE", theUrl, resourceIds.size())); } } @@ -563,7 +561,7 @@ public abstract class BaseHapiFhirResourceDao extends B } private DeleteMethodOutcome deleteExpunge(String theUrl, RequestDetails theTheRequest, Set theResourceIds) { - if (!myDaoConfig.isExpungeEnabled() || !myDaoConfig.isDeleteExpungeEnabled()) { + if (!getConfig().isExpungeEnabled() || !getConfig().isDeleteExpungeEnabled()) { throw new MethodNotAllowedException("_expunge is not enabled on this server"); } @@ -643,7 +641,7 @@ public abstract class BaseHapiFhirResourceDao extends B } private void validateDeleteEnabled() { - if (!myDaoConfig.isDeleteEnabled()) { + if (!getConfig().isDeleteEnabled()) { String msg = getContext().getLocalizer().getMessage(BaseHapiFhirResourceDao.class, "deleteBlockedBecauseDisabled"); throw new PreconditionFailedException(msg); } @@ -764,7 +762,7 @@ public abstract class BaseHapiFhirResourceDao extends B } private void validateExpungeEnabled() { - if (!myDaoConfig.isExpungeEnabled()) { + if (!getConfig().isExpungeEnabled()) { throw new MethodNotAllowedException("$expunge is not enabled on this server"); } } @@ -870,7 +868,7 @@ public abstract class BaseHapiFhirResourceDao extends B return; } - if (myDaoConfig.isMarkResourcesForReindexingUponSearchParameterChange()) { + if (getConfig().isMarkResourcesForReindexingUponSearchParameterChange()) { String expression = defaultString(theExpression); @@ -1065,6 +1063,8 @@ public abstract class BaseHapiFhirResourceDao extends B @PostConstruct @Override public void start() { + assert getConfig() != null; + ourLog.debug("Starting resource DAO for type: {}", getResourceName()); myInstanceValidator = getApplicationContext().getBean(IInstanceValidatorModule.class); myTxTemplate = new TransactionTemplate(myPlatformTransactionManager); @@ -1349,7 +1349,7 @@ public abstract class BaseHapiFhirResourceDao extends B throw new MethodNotAllowedException("Searching with _contained mode enabled is not enabled on this server"); } - if (myDaoConfig.getIndexMissingFields() == DaoConfig.IndexEnabledEnum.DISABLED) { + if (getConfig().getIndexMissingFields() == DaoConfig.IndexEnabledEnum.DISABLED) { for (List> nextAnds : theParams.values()) { for (List nextOrs : nextAnds) { for (IQueryParameterType next : nextOrs) { @@ -1414,10 +1414,10 @@ public abstract class BaseHapiFhirResourceDao extends B notifyInterceptors(RestOperationTypeEnum.SEARCH_TYPE, requestDetails); if (theRequest.isSubRequest()) { - Integer max = myDaoConfig.getMaximumSearchResultCountInTransaction(); + Integer max = getConfig().getMaximumSearchResultCountInTransaction(); if (max != null) { Validate.inclusiveBetween(1, Integer.MAX_VALUE, max, "Maximum search result count in transaction ust be a positive integer"); - theParams.setLoadSynchronousUpTo(myDaoConfig.getMaximumSearchResultCountInTransaction()); + theParams.setLoadSynchronousUpTo(getConfig().getMaximumSearchResultCountInTransaction()); } } @@ -1447,7 +1447,7 @@ public abstract class BaseHapiFhirResourceDao extends B @Override public Set searchForIds(SearchParameterMap theParams, RequestDetails theRequest) { return myTransactionService.execute(theRequest, tx -> { - theParams.setLoadSynchronousUpTo(myDaoConfig.getInternalSynchronousSearchSize()); + theParams.setLoadSynchronousUpTo(getConfig().getInternalSynchronousSearchSize()); ISearchBuilder builder = mySearchBuilderFactory.newSearchBuilder(this, getResourceName(), getResourceType()); @@ -1619,7 +1619,12 @@ public abstract class BaseHapiFhirResourceDao extends B "Invalid resource ID[" + entity.getIdDt().toUnqualifiedVersionless() + "] of type[" + entity.getResourceType() + "] - Does not match expected [" + getResourceName() + "]"); } - IBaseResource oldResource = toResource(entity, false); + IBaseResource oldResource; + if (getConfig().isMassIngestionMode()) { + oldResource = null; + } else { + oldResource = toResource(entity, false); + } /* * Mark the entity as not deleted - This is also done in the actual updateInternal() @@ -1689,7 +1694,7 @@ public abstract class BaseHapiFhirResourceDao extends B // Validate that there are no resources pointing to the candidate that // would prevent deletion DeleteConflictList deleteConflicts = new DeleteConflictList(); - if (myDaoConfig.isEnforceReferentialIntegrityOnDelete()) { + if (getConfig().isEnforceReferentialIntegrityOnDelete()) { myDeleteConflictService.validateOkToDelete(deleteConflicts, entity, true, theRequest, new TransactionDetails()); } DeleteConflictService.validateDeleteConflictsEmptyOrThrowException(getContext(), deleteConflicts); @@ -1759,7 +1764,7 @@ public abstract class BaseHapiFhirResourceDao extends B private void validateGivenIdIsAppropriateToRetrieveResource(IIdType theId, BaseHasResource entity) { if (entity.getForcedId() != null) { - if (myDaoConfig.getResourceClientIdStrategy() != DaoConfig.ClientIdStrategyEnum.ANY) { + if (getConfig().getResourceClientIdStrategy() != DaoConfig.ClientIdStrategyEnum.ANY) { if (theId.isIdPartValidLong()) { // This means that the resource with the given numeric ID exists, but it has a "forced ID", meaning that // as far as the outside world is concerned, the given ID doesn't exist (it's just an internal pointer @@ -1782,11 +1787,6 @@ public abstract class BaseHapiFhirResourceDao extends B } } - @VisibleForTesting - public void setDaoConfig(DaoConfig theDaoConfig) { - myDaoConfig = theDaoConfig; - } - private static class IdChecker implements IValidatorModule { private final ValidationModeEnum myMode; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java index 52fe67f50a9..197b30a9e6c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java @@ -54,7 +54,7 @@ public abstract class BaseHapiFhirResourceDaoObservation extends B protected String getResourceName() { return null; } + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java index 968ba695414..b8492ac6fcf 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java @@ -82,7 +82,7 @@ public class FhirResourceDaoValueSetDstu3 extends BaseHapiFhirResourceDao new ResourcePersistentId(resolveResourceIdentity(theRequestPartitionId, theResourceType, theId).getResourceId())); + retVal = myMemoryCacheService.getThenPutAfterCommit(MemoryCacheService.CacheEnum.FORCED_ID_TO_PID, key, t -> { + List ids = Collections.singletonList(new IdType(theResourceType, theId)); + List resolvedIds = resolveResourcePersistentIdsWithCache(theRequestPartitionId, ids); + if (resolvedIds.isEmpty()) { + throw new ResourceNotFoundException(ids.get(0)); + } + return resolvedIds.get(0); + }); } } else { @@ -196,14 +204,14 @@ public class IdHelperService { } else { - String partitionIdStringForKey = RequestPartitionId.stringifyForKey(theRequestPartitionId); +// String partitionIdStringForKey = RequestPartitionId.stringifyForKey(theRequestPartitionId); for (Iterator idIterator = nextIds.iterator(); idIterator.hasNext(); ) { String nextId = idIterator.next(); - String key = partitionIdStringForKey + "/" + nextResourceType + "/" + nextId; - Long nextCachedPid = myMemoryCacheService.getIfPresent(MemoryCacheService.CacheEnum.PERSISTENT_ID, key); + String key = toForcedIdToPidKey(theRequestPartitionId, nextResourceType, nextId); + ResourcePersistentId nextCachedPid = myMemoryCacheService.getIfPresent(MemoryCacheService.CacheEnum.FORCED_ID_TO_PID, key); if (nextCachedPid != null) { idIterator.remove(); - retVal.add(new ResourcePersistentId(nextCachedPid)); + retVal.add(nextCachedPid); } } @@ -224,10 +232,11 @@ public class IdHelperService { for (Object[] nextView : views) { String forcedId = (String) nextView[0]; Long pid = (Long) nextView[1]; - retVal.add(new ResourcePersistentId(pid)); + ResourcePersistentId persistentId = new ResourcePersistentId(pid); + retVal.add(persistentId); - String key = partitionIdStringForKey + "/" + nextResourceType + "/" + forcedId; - myMemoryCacheService.put(MemoryCacheService.CacheEnum.PERSISTENT_ID, key, pid); + String key = toForcedIdToPidKey(theRequestPartitionId, nextResourceType, forcedId); + myMemoryCacheService.put(MemoryCacheService.CacheEnum.FORCED_ID_TO_PID, key, persistentId); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java index 0c01b644d52..43226b5f2ff 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java @@ -78,7 +78,7 @@ public class FhirResourceDaoValueSetR4 extends BaseHapiFhirResourceDao boolean theUpdateVersion, TransactionDetails theTransactionDetails, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { ResourceTable retVal = super.updateEntity(theRequestDetails, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theTransactionDetails, theForceUpdate, theCreateNewHistoryEntry); - if (myDaoConfig.isPreExpandValueSets() && !retVal.isUnchangedInCurrentOperation()) { + if (getConfig().isPreExpandValueSets() && !retVal.isUnchangedInCurrentOperation()) { if (retVal.getDeleted() == null) { ValueSet valueSet = (ValueSet) theResource; myTerminologySvc.storeTermValueSet(retVal, valueSet); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoValueSetR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoValueSetR5.java index b3cc6c1f331..652d0881457 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoValueSetR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoValueSetR5.java @@ -79,7 +79,7 @@ public class FhirResourceDaoValueSetR5 extends BaseHapiFhirResourceDao boolean theUpdateVersion, TransactionDetails theTransactionDetails, boolean theForceUpdate, boolean theCreateNewHistoryEntry) { ResourceTable retVal = super.updateEntity(theRequestDetails, theResource, theEntity, theDeletedTimestampOrNull, thePerformIndexing, theUpdateVersion, theTransactionDetails, theForceUpdate, theCreateNewHistoryEntry); - if (myDaoConfig.isPreExpandValueSets() && !retVal.isUnchangedInCurrentOperation()) { + if (getConfig().isPreExpandValueSets() && !retVal.isUnchangedInCurrentOperation()) { if (retVal.getDeleted() == null) { ValueSet valueSet = (ValueSet) theResource; myTerminologySvc.storeTermValueSet(retVal, org.hl7.fhir.convertors.conv40_50.ValueSet40_50.convertValueSet(valueSet)); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/MemoryCacheService.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/MemoryCacheService.java index 9c3d0b75f09..a91d4488f65 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/MemoryCacheService.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/MemoryCacheService.java @@ -64,25 +64,35 @@ public class MemoryCacheService { for (CacheEnum next : CacheEnum.values()) { long timeoutSeconds; + int maximumSize; + switch (next) { case CONCEPT_TRANSLATION: case CONCEPT_TRANSLATION_REVERSE: timeoutSeconds = myDaoConfig.getTranslationCachesExpireAfterWriteInMinutes() * 1000; + maximumSize = 10000; break; - case HISTORY_COUNT: - case TAG_DEFINITION: - case PERSISTENT_ID: - case RESOURCE_LOOKUP: case PID_TO_FORCED_ID: case FORCED_ID_TO_PID: case MATCH_URL: + timeoutSeconds = 60; + maximumSize = 10000; + if (myDaoConfig.isMassIngestionMode()) { + timeoutSeconds = 3000; + maximumSize = 100000; + } + break; + case HISTORY_COUNT: + case TAG_DEFINITION: + case RESOURCE_LOOKUP: case RESOURCE_CONDITIONAL_CREATE_VERSION: default: timeoutSeconds = 60; + maximumSize = 10000; break; } - Cache nextCache = Caffeine.newBuilder().expireAfterWrite(timeoutSeconds, TimeUnit.MINUTES).maximumSize(10000).build(); + Cache nextCache = Caffeine.newBuilder().expireAfterWrite(timeoutSeconds, TimeUnit.MINUTES).maximumSize(maximumSize).build(); myCaches.put(next, nextCache); } @@ -163,7 +173,6 @@ public class MemoryCacheService { public enum CacheEnum { TAG_DEFINITION(TagDefinitionCacheKey.class), - PERSISTENT_ID(String.class), RESOURCE_LOOKUP(String.class), FORCED_ID_TO_PID(String.class), PID_TO_FORCED_ID(Long.class), diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java index 9dd6bd2a64c..d48fca85979 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java @@ -79,6 +79,7 @@ import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc; import ca.uhn.fhir.jpa.term.api.ITermReadSvc; import ca.uhn.fhir.jpa.term.api.ITermReadSvcR4; +import ca.uhn.fhir.jpa.util.MemoryCacheService; import ca.uhn.fhir.jpa.util.ResourceCountCache; import ca.uhn.fhir.jpa.validation.ValidationSettings; import ca.uhn.fhir.parser.IParser; @@ -472,6 +473,8 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil @Autowired protected ITermConceptMapGroupElementTargetDao myTermConceptMapGroupElementTargetDao; @Autowired + protected MemoryCacheService myMemoryCacheService; + @Autowired protected ICacheWarmingSvc myCacheWarmingSvc; protected IServerInterceptor myInterceptor; @Autowired diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java index 94fe0c3e33f..a594d1e7388 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java @@ -1,10 +1,14 @@ package ca.uhn.fhir.jpa.dao.r4; +import ca.uhn.fhir.context.RuntimeSearchParam; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.model.HistoryCountModeEnum; +import ca.uhn.fhir.jpa.model.entity.ModelConfig; import ca.uhn.fhir.jpa.model.util.JpaConstants; +import ca.uhn.fhir.jpa.partition.SystemRequestDetails; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.SqlQuery; +import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.param.ReferenceParam; @@ -14,7 +18,11 @@ import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.CareTeam; import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Coverage; import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.ExplanationOfBenefit; import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Narrative; import org.hl7.fhir.r4.model.Observation; @@ -27,7 +35,11 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -48,6 +60,10 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test { myDaoConfig.setDeleteEnabled(new DaoConfig().isDeleteEnabled()); myDaoConfig.setMatchUrlCache(new DaoConfig().getMatchUrlCache()); myDaoConfig.setHistoryCountMode(DaoConfig.DEFAULT_HISTORY_COUNT_MODE); + myDaoConfig.setMassIngestionMode(new DaoConfig().isMassIngestionMode()); + myModelConfig.setAutoVersionReferenceAtPaths(new ModelConfig().getAutoVersionReferenceAtPaths()); + myModelConfig.setRespectVersionsForSearchIncludes(new ModelConfig().isRespectVersionsForSearchIncludes()); + myFhirCtx.getParserOptions().setStripVersionsFromReferences(true); } @BeforeEach @@ -537,7 +553,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test { myCaptureQueriesListener.clear(); assertEquals(1, myObservationDao.search(map).size().intValue()); // Resolve forced ID, Perform search, load result - assertEquals(3, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); + assertEquals(2, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); assertNoPartitionSelectors(); assertEquals(0, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); assertEquals(0, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); @@ -579,7 +595,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test { assertEquals(1, myObservationDao.search(map).size().intValue()); myCaptureQueriesListener.logAllQueriesForCurrentThread(); // Resolve forced ID, Perform search, load result - assertEquals(3, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); + assertEquals(2, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); assertEquals(0, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); assertEquals(0, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread()); @@ -830,9 +846,9 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test { myCaptureQueriesListener.logSelectQueriesForCurrentThread(); assertEquals(1, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); myCaptureQueriesListener.logInsertQueriesForCurrentThread(); - assertEquals(5, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); + assertEquals(4, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); myCaptureQueriesListener.logUpdateQueriesForCurrentThread(); - assertEquals(2, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); + assertEquals(1, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread()); // Pass 2 @@ -1484,4 +1500,73 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test { } + @Test + public void testMassIngestionMode_TransactionWithChanges() { + myDaoConfig.setDeleteEnabled(false); + myDaoConfig.setMatchUrlCache(true); + myDaoConfig.setMassIngestionMode(true); + myFhirCtx.getParserOptions().setStripVersionsFromReferences(false); + myModelConfig.setRespectVersionsForSearchIncludes(true); + myModelConfig.setAutoVersionReferenceAtPaths( + "ExplanationOfBenefit.patient", + "ExplanationOfBenefit.insurance.coverage" + ); + + Patient warmUpPt = new Patient(); + warmUpPt.getMeta().addProfile("http://foo"); + warmUpPt.setActive(true); + myPatientDao.create(warmUpPt); + + AtomicInteger ai = new AtomicInteger(0); + Supplier supplier = () -> { + BundleBuilder bb = new BundleBuilder(myFhirCtx); + + Coverage coverage = new Coverage(); + coverage.getMeta().addProfile("http://foo"); + coverage.setId(IdType.newRandomUuid()); + coverage.addIdentifier().setSystem("http://coverage").setValue("12345"); + coverage.setStatus(Coverage.CoverageStatus.ACTIVE); + coverage.setType(new CodeableConcept().addCoding(new Coding("http://coverage-type", "12345", null))); + bb.addTransactionUpdateEntry(coverage).conditional("Coverage?identifier=http://coverage|12345"); + + Patient patient = new Patient(); + patient.getMeta().addProfile("http://foo"); + patient.setId("Patient/PATIENT-A"); + patient.setActive(true); + patient.addName().setFamily("SMITH").addGiven("JAMES" + ai.incrementAndGet()); + bb.addTransactionUpdateEntry(patient); + + ExplanationOfBenefit eob = new ExplanationOfBenefit(); + eob.getMeta().addProfile("http://foo"); + eob.addIdentifier().setSystem("http://eob").setValue("12345"); + eob.addInsurance().setCoverage(new Reference(coverage.getId())); + eob.getPatient().setReference(patient.getId()); + eob.setCreatedElement(new DateTimeType("2021-01-01T12:12:12Z")); + bb.addTransactionUpdateEntry(eob).conditional("ExplanationOfBenefit?identifier=http://eob|12345"); + + return (Bundle) bb.getBundle(); + }; + + myCaptureQueriesListener.clear(); + mySystemDao.transaction(new SystemRequestDetails(), supplier.get()); +// myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertEquals(5, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); + assertEquals(13, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); + assertEquals(3, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); + assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread()); + + myCaptureQueriesListener.clear(); + mySystemDao.transaction(new SystemRequestDetails(), supplier.get()); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertEquals(11, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); + assertEquals(2, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); + assertEquals(3, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); + assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread()); + + // assertEquals(15, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); + // assertEquals(1, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); + // assertEquals(3, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); + // assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread()); + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java index c8510deb149..90497508a1a 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java @@ -1142,7 +1142,6 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { myPatientDao.create(pt).getId().getIdPartAsLong(); } - myCaptureQueriesListener.clear(); SearchParameterMap map = new SearchParameterMap(); map.add(Patient.SP_ORGANIZATION, new ReferenceOrListParam() @@ -1162,8 +1161,40 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { .map(t -> t.getSql(true, false)) .collect(Collectors.toList()); - // Forced ID resolution + // No resolution of the forced IDs since they should already be in the + // cache from the original write operation. So: + // 1 - perform the search + // 2 - load the results + assertEquals(2, queries.size()); + + // The search itself String resultingQueryNotFormatted = queries.get(0); + assertEquals(1, StringUtils.countMatches(resultingQueryNotFormatted, "Patient.managingOrganization"), resultingQueryNotFormatted); + assertThat(resultingQueryNotFormatted, matchesPattern(".*TARGET_RESOURCE_ID IN \\('[0-9]+','[0-9]+','[0-9]+','[0-9]+','[0-9]+'\\).*")); + + // Ensure that the search actually worked + assertEquals(5, search.size().intValue()); + + /* + * Now clear the caches and make sure the lookup works as expected + */ + + myMemoryCacheService.invalidateAllCaches(); + myCaptureQueriesListener.clear(); + search = myPatientDao.search(map); + + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + queries = myCaptureQueriesListener + .getSelectQueriesForCurrentThread() + .stream() + .map(t -> t.getSql(true, false)) + .collect(Collectors.toList()); + + // The first query is the forced ID resolution this time + assertEquals(3, queries.size()); + + // Forced ID resolution + resultingQueryNotFormatted = queries.get(0); assertThat(resultingQueryNotFormatted, containsString("RESOURCE_TYPE='Organization'")); assertThat(resultingQueryNotFormatted, containsString("FORCED_ID in ('ORG0' , 'ORG1' , 'ORG2' , 'ORG3' , 'ORG4')")); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoSearchParameterR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoSearchParameterR4Test.java index bb6555591fb..85011502e58 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoSearchParameterR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoSearchParameterR4Test.java @@ -36,7 +36,7 @@ public class FhirResourceDaoSearchParameterR4Test { myDao = new FhirResourceDaoSearchParameterR4(); myDao.setContext(myCtx); - myDao.setConfig(new DaoConfig()); + myDao.setDaoConfigForUnitTest(new DaoConfig()); myDao.setApplicationContext(myApplicationContext); myDao.start(); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java index fc00df065fb..7350f9c9761 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java @@ -2548,8 +2548,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test { ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); - assertEquals(1, StringUtils.countMatches(searchSql, "forcedid0_.PARTITION_ID in ('1')"), searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "and forcedid0_.RESOURCE_TYPE='Patient'"), searchSql); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IN ('1')"), searchSql); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID"), searchSql); // Same query, different partition @@ -2584,8 +2583,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "forcedid0_.PARTITION_ID is null"), searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "forcedid0_.RESOURCE_TYPE='Patient'"), searchSql); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL"), searchSql); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID"), searchSql); // Same query, different partition diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/stresstest/GiantTransactionPerfTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/stresstest/GiantTransactionPerfTest.java index 980ff9a30a0..83a6ac4392f 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/stresstest/GiantTransactionPerfTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/stresstest/GiantTransactionPerfTest.java @@ -176,6 +176,7 @@ public class GiantTransactionPerfTest { mySystemDao = new FhirSystemDaoR4(); mySystemDao.setTransactionProcessorForUnitTest(myTransactionProcessor); + mySystemDao.setDaoConfigForUnitTest(myDaoConfig); mySystemDao.start(); when(myAppCtx.getBean(eq(IInstanceValidatorModule.class))).thenReturn(myInstanceValidatorSvc); @@ -235,11 +236,10 @@ public class GiantTransactionPerfTest { myEobDao = new JpaResourceDao<>(); myEobDao.setContext(myCtx); - myEobDao.setConfig(myDaoConfig); + myEobDao.setDaoConfigForUnitTest(myDaoConfig); myEobDao.setResourceType(ExplanationOfBenefit.class); myEobDao.setApplicationContext(myAppCtx); myEobDao.setTransactionService(myHapiTransactionService); - myEobDao.setDaoConfig(myDaoConfig); myEobDao.setRequestPartitionHelperService(new MockRequestPartitionHelperSvc()); myEobDao.setEntityManager(myEntityManager); myEobDao.setSearchParamWithInlineReferencesExtractor(mySearchParamWithInlineReferencesExtractor); @@ -247,6 +247,7 @@ public class GiantTransactionPerfTest { myEobDao.setSearchParamRegistry(mySearchParamRegistry); myEobDao.setSearchParamPresenceSvc(mySearchParamPresenceSvc); myEobDao.setDaoSearchParamSynchronizer(myDaoSearchParamSynchronizer); + myEobDao.setDaoConfigForUnitTest(myDaoConfig); myEobDao.start(); myDaoRegistry.setResourceDaos(Lists.newArrayList(myEobDao)); diff --git a/hapi-fhir-jpaserver-base/src/test/resources/r4/eob-bundle.json b/hapi-fhir-jpaserver-base/src/test/resources/r4/eob-bundle.json new file mode 100644 index 00000000000..593fadab797 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/resources/r4/eob-bundle.json @@ -0,0 +1,659 @@ +{ + "resourceType": "Bundle", + "type": "transaction", + "entry": [{ + "resource": { + "resourceType": "ExplanationOfBenefit", + "identifier": [{ + "type": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType", + "code": "payerid" + }] + }, + "system": "https://hl7.org/fhir/sid/payerid", + "value": "37470269207" + }, { + "type": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType", + "code": "uc" + }] + }, + "system": "https://hl7.org/fhir/sid/claimid", + "value": "208676340" + }], + "status": "active", + "type": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/claim-type", + "code": "professional" + }] + }, + "use": "claim", + "patient": { + "reference": "Patient/7ba514fa-6ec2-3203-ef80-2d142a88bb1d" + }, + "billablePeriod": { + "start": "2021-01-06", + "end": "2021-01-06" + }, + "created": "2021-01-06T00:00:00-08:00", + "insurer": { + "reference": "Organization/4772bdd0-7f10-d3d1-458c-db66eb2b0d17" + }, + "provider": { + "reference": "Organization/330cfdc3-22d4-6946-bd37-4ac82e5b48c5" + }, + "related": [{ + "relationship": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/ex-relatedclaimrelationship", + "code": "prior" + }] + }, + "reference": { + "type": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType", + "code": "uc" + }] + }, + "value": "208676157" + } + }], + "payee": { + "type": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/payeetype", + "code": "provider" + }], + "text": "Claim paid to Provider" + }, + "party": { + "reference": "Organization/330cfdc3-22d4-6946-bd37-4ac82e5b48c5" + } + }, + "facility": { + "reference": "Location/48f91ece-c048-6b55-0e37-782630e9c4d0" + }, + "outcome": "complete", + "disposition": "PAID", + "careTeam": [{ + "sequence": 1, + "provider": { + "reference": "Practitioner/dbb4fe06-98c4-e8de-3eef-f1b42e07509b" + }, + "responsible": true, + "role": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBClaimCareTeamRole", + "code": "performing" + }] + } + }], + "supportingInfo": [{ + "sequence": 1, + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBSupportingInfoType", + "code": "clmrecvddate" + }] + }, + "timingDate": "2021-01-06" + }, { + "sequence": 2, + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBSupportingInfoType", + "code": "noncontracted" + }] + } + }], + "diagnosis": [{ + "sequence": 1, + "diagnosisCodeableConcept": { + "coding": [{ + "system": "http://hl7.org/fhir/sid/icd-10-cm", + "code": "R50.9" + }] + }, + "type": [{ + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/ex-diagnosistype", + "code": "principal" + }] + }] + }], + "procedure": [{ + "sequence": 1, + "date": "2021-01-06T00:00:00-08:00", + "procedureCodeableConcept": { + "coding": [{ + "system": "http://www.ama-assn.org/go/cpt", + "code": "99283", + "display": "Emergency department visit for moderate problem" + }], + "text": "EMERGENCY DEPARTMENT VISIT MODERATE SEVERITY" + } + }], + "insurance": [{ + "focal": true, + "coverage": { + "reference": "urn:uuid:f556ad3d-e1a3-46f2-91d5-fd3ff57c1cef" + } + }], + "item": [{ + "sequence": 1, + "diagnosisSequence": [1], + "procedureSequence": [1], + "productOrService": { + "coding": [{ + "system": "http://www.ama-assn.org/go/cpt", + "code": "99283", + "display": "Emergency department visit for moderate problem" + }], + "text": "EMERGENCY DEPARTMENT VISIT MODERATE SEVERITY" + }, + "servicedPeriod": { + "start": "2021-01-06", + "end": "2021-01-06" + }, + "locationCodeableConcept": { + "coding": [{ + "system": "https://www.cms.gov/Medicare/Coding/place-of-service-codes/Place_of_Service_Code_Set", + "code": "23" + }] + }, + "quantity": { + "value": -1, + "unit": "Units", + "system": "http://unitsofmeasure.org", + "code": "[arb'U]" + }, + "net": { + "value": -2000000.0 + }, + "adjudication": [{ + "category": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "submitted" + }] + }, + "amount": { + "value": -2000000.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "benefit" + }] + }, + "amount": { + "value": -2000000.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "copay" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "deductible" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "coinsurance" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "memberliability" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "noncovered" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "priorpayerpaid" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "paidtoprovider" + }] + }, + "amount": { + "value": -2000000.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBPayerAdjudicationStatus", + "code": "outofnetwork" + }] + }, + "amount": { + "value": -2000000.00, + "currency": "USD" + } + }] + }], + "total": [{ + "category": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "submitted" + }] + }, + "amount": { + "value": -2000000.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "benefit" + }] + }, + "amount": { + "value": -2000000.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "copay" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/adjudication", + "code": "deductible" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "coinsurance" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "memberliability" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "noncovered" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "priorpayerpaid" + }] + }, + "amount": { + "value": 0.00, + "currency": "USD" + } + }, { + "category": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudication", + "code": "paidtoprovider" + }] + }, + "amount": { + "value": -2000000.00, + "currency": "USD" + } + }], + "payment": { + "date": "2021-01-11", + "amount": { + "value": -2000000.00, + "currency": "USD" + } + } + }, + "request": { + "method": "PUT", + "url": "ExplanationOfBenefit?identifier=37470269207" + } + }, { + "resource": { + "resourceType": "Patient", + "id": "7ba514fa-6ec2-3203-ef80-2d142a88bb1d", + "meta": { + "lastUpdated": "2021-05-22", + "profile": ["http://hl7.org/fhir/us/core/StructureDefinition/us-core-patient"] + }, + "identifier": [{ + "type": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "MR" + }] + }, + "system": "https://foo.org/front-door", + "value": "A0A0A0A0" + }], + "name": [{ + "use": "usual", + "text": "Rhrg Tapymgr", + "family": "Tapymgr", + "given": ["Rhrg"] + }], + "gender": "male", + "birthDate": "1982-05-05", + "address": [{ + "use": "home", + "type": "postal", + "line": ["300 GAVEN ST"], + "city": "SAN FRANCISCO", + "state": "CA", + "postalCode": "94134-1113" + }] + }, + "request": { + "method": "PUT", + "url": "Patient/7ba514fa-6ec2-3203-ef80-2d142a88bb1d" + } + }, { + "fullUrl": "urn:uuid:f556ad3d-e1a3-46f2-91d5-fd3ff57c1cef", + "resource": { + "resourceType": "Coverage", + "meta": { + "lastUpdated": "2021-05-22", + "profile": ["http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Coverage"] + }, + "identifier": [{ + "type": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "FILL" + }] + }, + "system": "https://hl7.org/fhir/sid/coverageid", + "value": "A0A0A0A0-000088006" + }], + "status": "active", + "type": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode", + "code": "HMO", + "display": "health maintenance organization policy" + }], + "text": "HMO - HMO COMMERCIAL-HMO" + }, + "subscriberId": "110066672294", + "beneficiary": { + "reference": "Patient/7ba514fa-6ec2-3203-ef80-2d142a88bb1d" + }, + "relationship": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/subscriber-relationship", + "code": "self", + "display": "Self" + }], + "text": "The Beneficiary is the Subscriber" + }, + "period": { + "start": "2018-07-01" + }, + "class": [{ + "type": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/coverage-class", + "code": "group", + "display": "Group" + }], + "text": "An employee group" + }, + "value": "88006", + "name": "JEFFCO PAINTING & COATING JEFFCO PAINTING & COATING-HMO" + }] + }, + "request": { + "method": "PUT", + "url": "Coverage?identifier=A0A0A0A0-000088006" + } + }, { + "resource": { + "resourceType": "Organization", + "id": "330cfdc3-22d4-6946-bd37-4ac82e5b48c5", + "meta": { + "lastUpdated": "2021-05-22", + "profile": ["http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Organization"] + }, + "identifier": [{ + "type": { + "coding": [{ + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType", + "code": "npi" + }] + }, + "system": "http://hl7.org/fhir/sid/us-npi", + "value": "1649794157" + }, { + "type": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "TAX" + }] + }, + "system": "urn:oid:2.16.840.1.113883.4.4", + "value": "821883948" + }], + "active": true, + "type": [{ + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/organization-type", + "code": "prov" + }] + }], + "name": "OU MEDICAL CENTER", + "address": [{ + "use": "work", + "type": "physical", + "line": ["PO BOX 277362"], + "city": "ATLANTA", + "state": "GA", + "postalCode": "30384-9998", + "country": "USA" + }] + }, + "request": { + "method": "PUT", + "url": "Organization/330cfdc3-22d4-6946-bd37-4ac82e5b48c5" + } + }, { + "resource": { + "resourceType": "Organization", + "id": "4772bdd0-7f10-d3d1-458c-db66eb2b0d17", + "meta": { + "lastUpdated": "2021-05-22", + "profile": ["http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Organization"] + }, + "active": true, + "type": [{ + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/organization-type", + "code": "pay", + "display": "Payer" + }] + }], + "name": "FOO", + "telecom": [{ + "system": "phone", + "value": "1-800-000-0000", + "use": "work" + }], + "address": [{ + "use": "work", + "type": "postal", + "line": ["NATIONAL CLAIMS ADMINISTRATION NORTHERN CALIFORNIA", "PO Box 629028"], + "city": "El Dorado Hills", + "state": "CA", + "postalCode": "95762-9028" + }] + }, + "request": { + "method": "PUT", + "url": "Organization/4772bdd0-7f10-d3d1-458c-db66eb2b0d17" + } + }, { + "resource": { + "resourceType": "Practitioner", + "id": "dbb4fe06-98c4-e8de-3eef-f1b42e07509b", + "meta": { + "lastUpdated": "2021-05-22", + "profile": ["http://hl7.org/fhir/us/core/StructureDefinition/us-core-practitioner"] + }, + "identifier": [{ + "use": "usual", + "type": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "NPI" + }] + }, + "system": "http://hl7.org/fhir/sid/us-npi", + "value": "1649794157" + }], + "name": [{ + "use": "usual", + "text": "OU MEDICAL CENTER", + "family": "OU MEDICAL CENTER" + }], + "address": [{ + "use": "work", + "line": ["1200 EVERETT DRIVE"], + "city": "OKLAHOMA CITY", + "state": "OK", + "postalCode": "73104-5047" + }] + }, + "request": { + "method": "PUT", + "url": "Practitioner/dbb4fe06-98c4-e8de-3eef-f1b42e07509b" + } + }, { + "resource": { + "resourceType": "Location", + "id": "48f91ece-c048-6b55-0e37-782630e9c4d0", + "meta": { + "lastUpdated": "2021-05-22" + }, + "identifier": [{ + "use": "usual", + "type": { + "coding": [{ + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "NPI" + }] + }, + "value": "PIN12181422" + }], + "status": "active", + "name": "OU MED CTR - CHILDRENS HOSPITAL", + "mode": "kind", + "type": [{ + "coding": [{ + "system": "https://www.cms.gov/Medicare/Coding/place-of-service-codes/Place_of_Service_Code_Set", + "code": "99" + }] + }], + "address": { + "use": "work", + "type": "physical", + "line": ["1200 EVERETT DRIVE"], + "city": "OKLAHOMA CITY", + "state": "OK", + "postalCode": "73104-5047" + } + }, + "request": { + "method": "PUT", + "url": "Location/48f91ece-c048-6b55-0e37-782630e9c4d0" + } + }] +} diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceTable.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceTable.java index d6002a726db..6e29ec79991 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceTable.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceTable.java @@ -59,6 +59,7 @@ import static org.apache.commons.lang3.StringUtils.defaultString; @Index(name = "IDX_RES_TYPE", columnList = "RES_TYPE"), @Index(name = "IDX_INDEXSTATUS", columnList = "SP_INDEX_STATUS") }) +@NamedEntityGraph(name = "Resource.noJoins") public class ResourceTable extends BaseHasResource implements Serializable, IBasePersistedResource, IResourceLookup { public static final int RESTYPE_LEN = 40; private static final int MAX_LANGUAGE_LENGTH = 20; @@ -691,12 +692,14 @@ public class ResourceTable extends BaseHasResource implements Serializable, IBas @Override public IdDt getIdDt() { - if (getForcedId() == null) { + if (getTransientForcedId() != null) { + // Avoid a join query if possible + return new IdDt(getResourceType() + '/' + getTransientForcedId() + '/' + Constants.PARAM_HISTORY + '/' + getVersion()); + } else if (getForcedId() == null) { Long id = this.getResourceId(); return new IdDt(getResourceType() + '/' + id + '/' + Constants.PARAM_HISTORY + '/' + getVersion()); } else { - // Avoid a join query if possible - String forcedId = getTransientForcedId() != null ? getTransientForcedId() : getForcedId().getForcedId(); + String forcedId = getForcedId().getForcedId(); return new IdDt(getResourceType() + '/' + forcedId + '/' + Constants.PARAM_HISTORY + '/' + getVersion()); } } diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/ReadOnlySearchParamCache.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/ReadOnlySearchParamCache.java index d173693aae8..1b578384de2 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/ReadOnlySearchParamCache.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/ReadOnlySearchParamCache.java @@ -23,6 +23,7 @@ package ca.uhn.fhir.jpa.searchparam.registry; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.RuntimeResourceDefinition; import ca.uhn.fhir.context.RuntimeSearchParam; +import com.google.common.annotations.VisibleForTesting; import java.util.Collections; import java.util.HashMap; diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/SearchParamRegistryImpl.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/SearchParamRegistryImpl.java index a6cf42636aa..ab80d0784df 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/SearchParamRegistryImpl.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/SearchParamRegistryImpl.java @@ -45,8 +45,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.Nullable; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.ArrayList; @@ -313,4 +313,5 @@ public class SearchParamRegistryImpl implements ISearchParamRegistry, IResourceC public void resetForUnitTest() { handleInit(Collections.emptyList()); } + }