honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch '2161-refactor-empi' into gg_20201105-remove-person-references

This commit is contained in:
Tadgh 2020-11-17 09:28:37 -05:00
parent 676f6315e6 c50c4edea4
commit 805d5b1d6c
27 changed files with 1089 additions and 299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hapi-fhir-base/src/main/resources/ca/uhn/fhir/i18n/hapi-messages.properties
View File

@ -1,5 +1,8 @@
ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl.expansionRefersToUnknownCs=Unknown CodeSystem URI "{0}" referenced from ValueSet
ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl.valueSetNotYetExpanded=ValueSet "{0}" has not yet been pre-expanded. Performing in-memory expansion without parameters. Current status: {1} | {2}
ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl.valueSetNotYetExpanded_OffsetNotAllowed=ValueSet expansion can not combine "offset" with "ValueSet.compose.exclude" unless the ValueSet has been pre-expanded. ValueSet "{0}" must be pre-expanded for this operation to work.
# Core Library Messages
ca.uhn.fhir.context.FhirContext.unknownResourceName=Unknown resource name "{0}" (this name is not known in FHIR version "{1}")

5
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_2_0/2162-improve-vs-expansion-filtering.yaml Normal file
View File

@ -0,0 +1,5 @@
---
type: fix
issue: 2162
title: "When expanding a pre-expanded ValueSet using a filter, the filter was ignored and the pre-expansion was not used
resulting in an inefficient and potentially incorrect expansion. This has been corrected."

4
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_2_0/2164-package-description-longer-than-200-characters.yaml Normal file
View File

@ -0,0 +1,4 @@
---
type: fix
issue: 2164
title: "The JPA Package loader was failing if the package had a description longer than 200 characters. This has been fixed."

9
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_plain/rest_operations.md
View File

@ -389,7 +389,7 @@ Tags have very specific semantics, which may not be obvious simply by using the
## Accessing Tags in a Read / VRead / Search Method
Tags are stored within a resource object, in the Resource.meta element.
Tags are stored within a resource object, in the Resource.meta element. It is important to note that changing a resource's tags will not cause a version update to that resource.
In a server implementation, you may populate your tags into the returned resource(s) and HAPI will automatically place these tags into the response headers (for read/vread) or the bundle category tags (for search). The following example illustrates how to return tags from a server method. This example shows how to supply tags in a read method, but the same approach applies to vread and search operations as well.
@ -409,12 +409,17 @@ Within a [Type Create](#type_create) or [Instance Update](#instance_update) meth
Note that FHIR specifies that in an update method, any tags supplied by the client are copied to the newly saved version, as well as any tags the existing version had.
To work with tags in a create/update method, the pattern used in the read examples above is simply revered. In a server, the resource which is passed in will be populated with any tags that the client supplied:
To work with tags in a create/update method, the pattern used in the read examples above is simply reversed. In a server, the resource which is passed in will be populated with any tags that the client supplied:
```java
{{snippet:classpath:/ca/uhn/hapi/fhir/docs/RestfulPatientResourceProviderMore.java|createTags}}
```
## Removing Tags
In order to remove a tag, it does not suffice to remove it from the resource. Tags can be removed using the [Resource Operation Meta Delete](https://www.hl7.org/fhir/resource-operation-meta-delete.html), which takes a Parameter
definining which tags to delete.
# Handling _summary and _elements
The `_summary` and `_elements` parameters are automatically handled by the server, so no coding is required to make this work.

5
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptViewDao.java
View File

@ -1,6 +1,8 @@
package ca.uhn.fhir.jpa.dao.data;
import ca.uhn.fhir.jpa.entity.TermValueSetConceptView;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
@ -32,4 +34,7 @@ public interface ITermValueSetConceptViewDao extends JpaRepository<TermValueSetC
@Query("SELECT v FROM TermValueSetConceptView v WHERE v.myConceptValueSetPid = :pid AND v.myConceptOrder >= :from AND v.myConceptOrder < :to ORDER BY v.myConceptOrder")
List<TermValueSetConceptView> findByTermValueSetId(@Param("from") int theFrom, @Param("to") int theTo, @Param("pid") Long theValueSetId);
@Query("SELECT v FROM TermValueSetConceptView v WHERE v.myConceptValueSetPid = :pid AND v.myConceptDisplay LIKE :display ORDER BY v.myConceptOrder")
List<TermValueSetConceptView> findByTermValueSetId(@Param("pid") Long theValueSetId, @Param("display") String theDisplay);
}

3
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java
View File

@ -27,6 +27,7 @@ import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet;
import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao;
import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.util.JpaConstants;
import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
@ -221,7 +222,7 @@ public class FhirResourceDaoValueSetR4 extends BaseHapiFhirResourceDao<ValueSet>
private void addFilterIfPresent(String theFilter, ConceptSetComponent include) {
if (ElementUtil.isEmpty(include.getConcept())) {
if (isNotBlank(theFilter)) {
include.addFilter().setProperty("display").setOp(FilterOperator.EQUAL).setValue(theFilter);
include.addFilter().setProperty(JpaConstants.VALUESET_FILTER_DISPLAY).setOp(FilterOperator.EQUAL).setValue(theFilter);
}
}
}

1
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/tx/HapiTransactionService.java
View File

@ -94,7 +94,6 @@ public class HapiTransactionService {
}
}
}
/**

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java
View File

@ -40,7 +40,7 @@ import static org.apache.commons.lang3.StringUtils.length;
* bork up migration tasks.
*/
@Table(name = "TRM_VALUESET_CONCEPT", uniqueConstraints = {
@UniqueConstraint(name = "IDX_VS_CONCEPT_CS_CD", columnNames = {"VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL"}),
@UniqueConstraint(name = "IDX_VS_CONCEPT_CS_CODE", columnNames = {"VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL"}),
@UniqueConstraint(name = "IDX_VS_CONCEPT_ORDER", columnNames = {"VALUESET_PID", "VALUESET_ORDER"})
})
@Entity()

10
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java
View File

@ -217,10 +217,16 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
}
boolean currentVersion = updateCurrentVersionFlagForAllPackagesBasedOnNewIncomingVersion(thePackageId, packageVersionId);
String packageDesc;
if (npmPackage.description().length() > NpmPackageVersionEntity.PACKAGE_DESC_LENGTH) {
packageDesc = npmPackage.description().substring(0, NpmPackageVersionEntity.PACKAGE_DESC_LENGTH - 4) + "...";
} else {
packageDesc = npmPackage.description();
}
if (currentVersion) {
getProcessingMessages(npmPackage).add("Marking package " + thePackageId + "#" + thePackageVersionId + " as current version");
pkg.setCurrentVersionId(packageVersionId);
pkg.setDescription(npmPackage.description());
pkg.setDescription(packageDesc);
myPackageDao.save(pkg);
} else {
getProcessingMessages(npmPackage).add("Package " + thePackageId + "#" + thePackageVersionId + " is not the newest version");
@ -232,7 +238,7 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
packageVersion.setPackage(pkg);
packageVersion.setPackageBinary(persistedPackage);
packageVersion.setSavedTime(new Date());
packageVersion.setDescription(npmPackage.description());
packageVersion.setDescription(packageDesc);
packageVersion.setFhirVersionId(npmPackage.fhirVersion());
packageVersion.setFhirVersion(fhirVersion);
packageVersion.setCurrentVersion(currentVersion);

34
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java
View File

@ -37,8 +37,6 @@ import ca.uhn.fhir.jpa.entity.Search;
import ca.uhn.fhir.jpa.entity.SearchInclude;
import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
@ -47,6 +45,7 @@ import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.util.InterceptorUtil;
import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.api.Include;
import ca.uhn.fhir.rest.api.CacheControlDirective;
import ca.uhn.fhir.rest.api.Constants;
@ -468,7 +467,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
return txTemplate.execute(t -> {
// Load the results synchronously
final List<ResourcePersistentId> pids = new ArrayList<>();
@ -668,6 +667,12 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
myRequestPartitionHelperService = theRequestPartitionHelperService;
}
private boolean isWantCount(SearchParameterMap myParams, boolean wantOnlyCount) {
return wantOnlyCount ||
SearchTotalModeEnum.ACCURATE.equals(myParams.getSearchTotalMode()) ||
(myParams.getSearchTotalMode() == null && SearchTotalModeEnum.ACCURATE.equals(myDaoConfig.getDefaultTotalMode()));
}
/**
* A search task is a Callable task that runs in
* a thread pool to handle an individual search. One instance
@ -691,6 +696,8 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
private final ArrayList<ResourcePersistentId> myUnsyncedPids = new ArrayList<>();
private final RequestDetails myRequest;
private final RequestPartitionId myRequestPartitionId;
private final SearchRuntimeDetails mySearchRuntimeDetails;
private final Transaction myParentTransaction;
private Search mySearch;
private boolean myAbortRequested;
private int myCountSavedTotal = 0;
@ -699,8 +706,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
private boolean myAdditionalPrefetchThresholdsRemaining;
private List<ResourcePersistentId> myPreviouslyAddedResourcePids;
private Integer myMaxResultsToFetch;
private final SearchRuntimeDetails mySearchRuntimeDetails;
private final Transaction myParentTransaction;
/**
* Constructor
@ -1193,17 +1198,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
}
}
private boolean isWantCount(SearchParameterMap myParams, boolean wantOnlyCount) {
return wantOnlyCount ||
SearchTotalModeEnum.ACCURATE.equals(myParams.getSearchTotalMode()) ||
(myParams.getSearchTotalMode() == null && SearchTotalModeEnum.ACCURATE.equals(myDaoConfig.getDefaultTotalMode()));
}
private static boolean isWantOnlyCount(SearchParameterMap myParams) {
return SummaryEnum.COUNT.equals(myParams.getSummaryMode())
| INTEGER_0.equals(myParams.getCount());
}
public class SearchContinuationTask extends SearchTask {
public SearchContinuationTask(Search theSearch, IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequest, RequestPartitionId theRequestPartitionId) {
@ -1242,6 +1236,10 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
}
private static boolean isWantOnlyCount(SearchParameterMap myParams) {
return SummaryEnum.COUNT.equals(myParams.getSummaryMode())
| INTEGER_0.equals(myParams.getCount());
}
public static void populateSearchEntity(SearchParameterMap theParams, String theResourceType, String theSearchUuid, String theQueryString, Search theSearch) {
theSearch.setDeleted(false);
@ -1270,8 +1268,8 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
* Creates a {@link Pageable} using a start and end index
*/
@SuppressWarnings("WeakerAccess")
public static @Nullable
Pageable toPage(final int theFromIndex, int theToIndex) {
@Nullable
public static Pageable toPage(final int theFromIndex, int theToIndex) {
int pageSize = theToIndex - theFromIndex;
if (pageSize < 1) {
return null;

18
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/UriPredicateBuilder.java
View File

@ -77,6 +77,7 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
public Condition addPredicate(List<? extends IQueryParameterType> theUriOrParameterList, String theParamName, SearchFilterParser.CompareOperation theOperation, RequestDetails theRequestDetails) {
List<Condition> codePredicates = new ArrayList<>();
boolean predicateIsHash = false;
for (IQueryParameterType nextOr : theUriOrParameterList) {
if (nextOr instanceof UriParam) {
@ -141,8 +142,8 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
Condition uriPredicate = null;
if (theOperation == null || theOperation == SearchFilterParser.CompareOperation.eq) {
long hashUri = ResourceIndexedSearchParamUri.calculateHashUri(getPartitionSettings(), getRequestPartitionId(), getResourceType(), theParamName, value);
Condition hashPredicate = BinaryCondition.equalTo(myColumnHashUri, generatePlaceholder(hashUri));
codePredicates.add(hashPredicate);
uriPredicate = BinaryCondition.equalTo(myColumnHashUri, generatePlaceholder(hashUri));
predicateIsHash = true;
} else if (theOperation == SearchFilterParser.CompareOperation.ne) {
uriPredicate = BinaryCondition.notEqualTo(myColumnUri, generatePlaceholder(value));
} else if (theOperation == SearchFilterParser.CompareOperation.co) {
@ -164,11 +165,7 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
theOperation.toString()));
}
if (uriPredicate != null) {
long hashIdentity = BaseResourceIndexedSearchParam.calculateHashIdentity(getPartitionSettings(), getRequestPartitionId(), getResourceType(), theParamName);
BinaryCondition hashIdentityPredicate = BinaryCondition.equalTo(getColumnHashIdentity(), generatePlaceholder(hashIdentity));
codePredicates.add(ComboCondition.and(hashIdentityPredicate, uriPredicate));
}
codePredicates.add(uriPredicate);
}
} else {
@ -186,8 +183,11 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
}
ComboCondition orPredicate = ComboCondition.or(codePredicates.toArray(new Condition[0]));
Condition outerPredicate = combineWithHashIdentityPredicate(getResourceType(), theParamName, orPredicate);
return outerPredicate;
if (predicateIsHash) {
return orPredicate;
} else {
return combineWithHashIdentityPredicate(getResourceType(), theParamName, orPredicate);
}
}

445
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java
View File

@ -66,6 +66,7 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.sched.HapiJob;
import ca.uhn.fhir.jpa.model.sched.ISchedulerService;
import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition;
import ca.uhn.fhir.jpa.model.util.JpaConstants;
import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
@ -81,14 +82,17 @@ import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
import ca.uhn.fhir.util.CoverageIgnore;
import ca.uhn.fhir.util.FhirVersionIndependentConcept;
import ca.uhn.fhir.util.HapiExtensions;
import ca.uhn.fhir.util.StopWatch;
import ca.uhn.fhir.util.UrlUtil;
import ca.uhn.fhir.util.ValidateUtil;
import ca.uhn.fhir.util.FhirVersionIndependentConcept;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ArrayListMultimap;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.time.DateUtils;
@ -137,6 +141,7 @@ import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.interceptor.NoRollbackRuleAttribute;
import org.springframework.transaction.interceptor.RuleBasedTransactionAttribute;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.transaction.support.TransactionTemplate;
import javax.annotation.Nonnull;
@ -158,9 +163,9 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -169,8 +174,10 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
import static org.apache.commons.lang3.StringUtils.defaultString;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isEmpty;
@ -248,66 +255,64 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
return cs != null;
}
private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, TermConcept theConcept, boolean theAdd, AtomicInteger theCodeCounter, String theValueSetIncludeVersion) {
private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, TermConcept theConcept, boolean theAdd, String theValueSetIncludeVersion) {
String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri();
String code = theConcept.getCode();
String display = theConcept.getDisplay();
Collection<TermConceptDesignation> designations = theConcept.getDesignations();
if (StringUtils.isNotEmpty(theValueSetIncludeVersion)) {
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, theCodeCounter, codeSystem + "|" + theValueSetIncludeVersion, code, display);
return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem + "|" + theValueSetIncludeVersion, code, display);
} else {
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, theCodeCounter, codeSystem, code, display);
return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem, code, display);
}
}
private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, AtomicInteger theCodeCounter, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay) {
private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay) {
if (StringUtils.isNotEmpty(theCodeSystemVersion)) {
if (isNoneBlank(theCodeSystem, theCode)) {
if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, theDesignations);
theCodeCounter.incrementAndGet();
theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, null);
}
if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.excludeConcept(theCodeSystem + "|" + theCodeSystemVersion, theCode);
theCodeCounter.decrementAndGet();
}
}
} else {
if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations);
theCodeCounter.incrementAndGet();
theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, null);
}
if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
theCodeCounter.decrementAndGet();
}
}
}
private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, AtomicInteger theCodeCounter, String theCodeSystem, String theCode, String theDisplay) {
private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, String theCodeSystem, String theCode, String theDisplay) {
if (isNoneBlank(theCodeSystem, theCode)) {
if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations);
theCodeCounter.incrementAndGet();
return true;
}
if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
theCodeCounter.decrementAndGet();
return true;
}
}
return false;
}
private void addConceptsToList(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, String theSystem, List<CodeSystem.ConceptDefinitionComponent> theConcept, boolean theAdd, FhirVersionIndependentConcept theWantConceptOrNull) {
private void addConceptsToList(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, String theSystem, List<CodeSystem.ConceptDefinitionComponent> theConcept, boolean theAdd, @Nonnull ExpansionFilter theExpansionFilter) {
for (CodeSystem.ConceptDefinitionComponent next : theConcept) {
if (isNoneBlank(theSystem, next.getCode())) {
if (theWantConceptOrNull == null || theWantConceptOrNull.getCode().equals(next.getCode())) {
if (!theExpansionFilter.hasCode() || theExpansionFilter.getCode().equals(next.getCode())) {
addOrRemoveCode(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, next.getCode(), next.getDisplay());
}
}
addConceptsToList(theValueSetCodeAccumulator, theAddedCodes, theSystem, next.getConcept(), theAdd, theWantConceptOrNull);
addConceptsToList(theValueSetCodeAccumulator, theAddedCodes, theSystem, next.getConcept(), theAdd, theExpansionFilter);
}
}
@ -387,28 +392,14 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
deleteValueSetForResource(theResourceTable);
}
private ValueSet expandValueSetInMemory(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, FhirVersionIndependentConcept theWantConceptOrNull) {
int maxCapacity = myDaoConfig.getMaximumExpansionSize();
ValueSetExpansionComponentWithConceptAccumulator expansionComponent = new ValueSetExpansionComponentWithConceptAccumulator(myContext, maxCapacity);
expansionComponent.setIdentifier(UUID.randomUUID().toString());
expansionComponent.setTimestamp(new Date());
AtomicInteger codeCounter = new AtomicInteger(0);
expandValueSet(theExpansionOptions, theValueSetToExpand, expansionComponent, codeCounter, theWantConceptOrNull);
expansionComponent.setTotal(codeCounter.get());
ValueSet valueSet = new ValueSet();
valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
valueSet.setCompose(theValueSetToExpand.getCompose());
valueSet.setExpansion(expansionComponent);
return valueSet;
}
@Override
public List<FhirVersionIndependentConcept> expandValueSet(ValueSetExpansionOptions theExpansionOptions, String theValueSet) {
ExpansionFilter expansionFilter = ExpansionFilter.NO_FILTER;
return expandValueSet(theExpansionOptions, theValueSet, expansionFilter);
}
private List<FhirVersionIndependentConcept> expandValueSet(ValueSetExpansionOptions theExpansionOptions, String theValueSet, ExpansionFilter theExpansionFilter) {
// TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(theValueSet);
@ -416,7 +407,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
throwInvalidValueSet(theValueSet);
}
return expandValueSetAndReturnVersionIndependentConcepts(theExpansionOptions, valueSet, null);
return expandValueSetAndReturnVersionIndependentConcepts(theExpansionOptions, valueSet, theExpansionFilter);
}
@Override
@ -424,6 +415,45 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
public ValueSet expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand) {
ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null");
ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions);
int offset = expansionOptions.getOffset();
int count = expansionOptions.getCount();
ValueSetExpansionComponentWithConceptAccumulator accumulator = new ValueSetExpansionComponentWithConceptAccumulator(myContext, count);
accumulator.setHardExpansionMaximumSize(myDaoConfig.getMaximumExpansionSize());
accumulator.setSkipCountRemaining(offset);
accumulator.setIdentifier(UUID.randomUUID().toString());
accumulator.setTimestamp(new Date());
accumulator.setOffset(offset);
if (theExpansionOptions != null && isHibernateSearchEnabled()) {
accumulator.addParameter().setName("offset").setValue(new IntegerType(offset));
accumulator.addParameter().setName("count").setValue(new IntegerType(count));
}
ExpansionFilter filter = ExpansionFilter.NO_FILTER;
expandValueSetIntoAccumulator(theValueSetToExpand, theExpansionOptions, accumulator, filter, true);
if (accumulator.getTotalConcepts() != null) {
accumulator.setTotal(accumulator.getTotalConcepts());
}
ValueSet valueSet = new ValueSet();
valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
valueSet.setCompose(theValueSetToExpand.getCompose());
valueSet.setExpansion(accumulator);
for (String next : accumulator.getMessages()) {
valueSet.getMeta().addExtension()
.setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE)
.setValue(new StringType(next));
}
return valueSet;
}
private void expandValueSetIntoAccumulator(ValueSet theValueSetToExpand, ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theAccumulator, ExpansionFilter theFilter, boolean theAdd) {
Optional<TermValueSet> optionalTermValueSet;
if (theValueSetToExpand.hasUrl()) {
if (theValueSetToExpand.hasVersion()) {
@ -440,85 +470,87 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
optionalTermValueSet = Optional.empty();
}
/*
* ValueSet doesn't exist in pre-expansion database, so perform in-memory expansion
*/
if (!optionalTermValueSet.isPresent()) {
ourLog.debug("ValueSet is not present in terminology tables. Will perform in-memory expansion without parameters. {}", getValueSetInfo(theValueSetToExpand));
return expandValueSetInMemory(theExpansionOptions, theValueSetToExpand, null); // In-memory expansion.
}
TermValueSet termValueSet = optionalTermValueSet.get();
if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
ourLog.warn("{} is present in terminology tables but not ready for persistence-backed invocation of operation $expand. Will perform in-memory expansion without parameters. Current status: {} | {}",
getValueSetInfo(theValueSetToExpand), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
return expandValueSetInMemory(theExpansionOptions, theValueSetToExpand, null); // In-memory expansion.
}
ValueSet.ValueSetExpansionComponent expansionComponent = new ValueSet.ValueSetExpansionComponent();
expansionComponent.setIdentifier(UUID.randomUUID().toString());
expansionComponent.setTimestamp(new Date());
ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions);
int offset = expansionOptions.getOffset();
int count = expansionOptions.getCount();
populateExpansionComponent(expansionComponent, termValueSet, offset, count);
ValueSet valueSet = new ValueSet();
valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
valueSet.setCompose(theValueSetToExpand.getCompose());
valueSet.setExpansion(expansionComponent);
return valueSet;
}
private void populateExpansionComponent(ValueSet.ValueSetExpansionComponent theExpansionComponent, TermValueSet theTermValueSet, int theOffset, int theCount) {
int total = theTermValueSet.getTotalConcepts().intValue();
theExpansionComponent.setTotal(total);
theExpansionComponent.setOffset(theOffset);
theExpansionComponent.addParameter().setName("offset").setValue(new IntegerType(theOffset));
theExpansionComponent.addParameter().setName("count").setValue(new IntegerType(theCount));
if (theCount == 0) {
expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter);
return;
}
expandConcepts(theExpansionComponent, theTermValueSet, theOffset, theCount);
/*
* ValueSet exists in pre-expansion database, but pre-expansion is not yet complete so perform in-memory expansion
*/
TermValueSet termValueSet = optionalTermValueSet.get();
if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded", getValueSetInfo(theValueSetToExpand), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
theAccumulator.addMessage(msg);
expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter);
return;
}
/*
* ValueSet is pre-expanded in database so let's use that
*/
expandConcepts(theAccumulator, termValueSet, theFilter, theAdd);
}
private void expandConcepts(ValueSet.ValueSetExpansionComponent theExpansionComponent, TermValueSet theTermValueSet, int theOffset, int theCount) {
private void expandConcepts(IValueSetConceptAccumulator theAccumulator, TermValueSet theTermValueSet, ExpansionFilter theFilter, boolean theAdd) {
Integer offset = theAccumulator.getSkipCountRemaining();
offset = ObjectUtils.defaultIfNull(offset, 0);
offset = Math.min(offset, theTermValueSet.getTotalConcepts().intValue());
Integer count = theAccumulator.getCapacityRemaining();
count = defaultIfNull(count, myDaoConfig.getMaximumExpansionSize());
int conceptsExpanded = 0;
int designationsExpanded = 0;
int toIndex = theOffset + theCount;
Collection<TermValueSetConceptView> conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theOffset, toIndex, theTermValueSet.getId());
int toIndex = offset + count;
Collection<TermValueSetConceptView> conceptViews;
boolean wasFilteredResult = false;
if (!theFilter.getFilters().isEmpty() && JpaConstants.VALUESET_FILTER_DISPLAY.equals(theFilter.getFilters().get(0).getProperty()) && theFilter.getFilters().get(0).getOp() == ValueSet.FilterOperator.EQUAL) {
String displayValue = theFilter.getFilters().get(0).getValue().replace("%", "[%]") + "%";
conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theTermValueSet.getId(), displayValue);
wasFilteredResult = true;
} else {
conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId());
theAccumulator.consumeSkipCount(offset);
if (theAdd) {
theAccumulator.incrementOrDecrementTotalConcepts(true, theTermValueSet.getTotalConcepts().intValue());
}
}
if (conceptViews.isEmpty()) {
logConceptsExpanded("No concepts to expand. ", theTermValueSet, conceptsExpanded);
return;
}
Map<Long, ValueSet.ValueSetExpansionContainsComponent> pidToConcept = new HashMap<>();
Map<Long, FhirVersionIndependentConcept> pidToConcept = new LinkedHashMap<>();
ArrayListMultimap<Long, TermConceptDesignation> pidToDesignations = ArrayListMultimap.create();
for (TermValueSetConceptView conceptView : conceptViews) {
Long conceptPid = conceptView.getConceptPid();
ValueSet.ValueSetExpansionContainsComponent containsComponent;
if (!pidToConcept.containsKey(conceptPid)) {
containsComponent = theExpansionComponent.addContains();
containsComponent.setSystem(conceptView.getConceptSystemUrl());
containsComponent.setCode(conceptView.getConceptCode());
containsComponent.setDisplay(conceptView.getConceptDisplay());
pidToConcept.put(conceptPid, containsComponent);
} else {
containsComponent = pidToConcept.get(conceptPid);
String system = conceptView.getConceptSystemUrl();
String code = conceptView.getConceptCode();
String display = conceptView.getConceptDisplay();
FhirVersionIndependentConcept concept = new FhirVersionIndependentConcept(system, code, display);
pidToConcept.put(conceptPid, concept);
}
// TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to designations optional.
if (conceptView.getDesignationPid() != null) {
ValueSet.ConceptReferenceDesignationComponent designationComponent = containsComponent.addDesignation();
designationComponent.setLanguage(conceptView.getDesignationLang());
designationComponent.setUse(new Coding(
conceptView.getDesignationUseSystem(),
conceptView.getDesignationUseCode(),
conceptView.getDesignationUseDisplay()));
designationComponent.setValue(conceptView.getDesignationVal());
TermConceptDesignation designation = new TermConceptDesignation();
designation.setUseSystem(conceptView.getDesignationUseSystem());
designation.setUseCode(conceptView.getDesignationUseCode());
designation.setUseDisplay(conceptView.getDesignationUseDisplay());
designation.setValue(conceptView.getDesignationVal());
designation.setLanguage(conceptView.getDesignationLang());
pidToDesignations.put(conceptPid, designation);
if (++designationsExpanded % 250 == 0) {
logDesignationsExpanded("Expansion of designations in progress. ", theTermValueSet, designationsExpanded);
@ -530,6 +562,33 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
}
for (Long nextPid : pidToConcept.keySet()) {
FhirVersionIndependentConcept concept = pidToConcept.get(nextPid);
List<TermConceptDesignation> designations = pidToDesignations.get(nextPid);
String system = concept.getSystem();
String code = concept.getCode();
String display = concept.getDisplay();
if (theAdd) {
if (theAccumulator.getCapacityRemaining() != null) {
if (theAccumulator.getCapacityRemaining() == 0) {
break;
}
}
theAccumulator.includeConceptWithDesignations(system, code, display, designations);
} else {
boolean removed = theAccumulator.excludeConcept(system, code);
if (removed) {
theAccumulator.incrementOrDecrementTotalConcepts(false, 1);
}
}
}
if (wasFilteredResult && theAdd) {
theAccumulator.incrementOrDecrementTotalConcepts(true, pidToConcept.size());
}
logDesignationsExpanded("Finished expanding designations. ", theTermValueSet, designationsExpanded);
logConceptsExpanded("Finished expanding concepts. ", theTermValueSet, conceptsExpanded);
}
@ -549,25 +608,34 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
@Override
@Transactional(propagation = Propagation.REQUIRED)
public void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) {
expandValueSet(theExpansionOptions, theValueSetToExpand, theValueSetCodeAccumulator, new AtomicInteger(0), null);
expandValueSet(theExpansionOptions, theValueSetToExpand, theValueSetCodeAccumulator, ExpansionFilter.NO_FILTER);
}
@SuppressWarnings("ConstantConditions")
private void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator, AtomicInteger theCodeCounter, FhirVersionIndependentConcept theWantConceptOrNull) {
private void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator, @Nonnull ExpansionFilter theExpansionFilter) {
Set<String> addedCodes = new HashSet<>();
StopWatch sw = new StopWatch();
String valueSetInfo = getValueSetInfo(theValueSetToExpand);
ourLog.debug("Working with {}", valueSetInfo);
// Offset can't be combined with excludes
Integer skipCountRemaining = theValueSetCodeAccumulator.getSkipCountRemaining();
if (skipCountRemaining != null && skipCountRemaining > 0) {
if (theValueSetToExpand.getCompose().getExclude().size() > 0) {
String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded_OffsetNotAllowed", valueSetInfo);
throw new InvalidRequestException(msg);
}
}
// Handle includes
ourLog.debug("Handling includes");
for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getInclude()) {
for (int i = 0; ; i++) {
int queryIndex = i;
Boolean shouldContinue = myTxTemplate.execute(t -> {
Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> {
boolean add = true;
return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, include, add, theCodeCounter, queryIndex, theWantConceptOrNull);
return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, include, add, queryIndex, theExpansionFilter);
});
if (!shouldContinue) {
break;
@ -575,20 +643,15 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
}
// If the accumulator filled up, abort
if (theValueSetCodeAccumulator.getCapacityRemaining() != null && theValueSetCodeAccumulator.getCapacityRemaining() <= 0) {
String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myDaoConfig.getMaximumExpansionSize());
throw new ExpansionTooCostlyException(msg);
}
// Handle excludes
ourLog.debug("Handling excludes");
for (ValueSet.ConceptSetComponent exclude : theValueSetToExpand.getCompose().getExclude()) {
for (int i = 0; ; i++) {
int queryIndex = i;
Boolean shouldContinue = myTxTemplate.execute(t -> {
Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> {
boolean add = false;
return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, exclude, add, theCodeCounter, queryIndex, null);
ExpansionFilter expansionFilter = ExpansionFilter.NO_FILTER;
return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, exclude, add, queryIndex, expansionFilter);
});
if (!shouldContinue) {
break;
@ -603,47 +666,49 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
ourLog.debug("Done working with {} in {}ms", valueSetInfo, sw.getMillis());
}
/**
* Execute in a new transaction only if we aren't already in one. We do this because in some cases
* when performing a VS expansion we throw an {@link ExpansionTooCostlyException} and we don't want
* this to cause the TX to be marked a rollback prematurely.
*/
private <T> T executeInNewTransactionIfNeeded(Supplier<T> theAction) {
if (TransactionSynchronizationManager.isSynchronizationActive()) {
return theAction.get();
}
return myTxTemplate.execute(t->theAction.get());
}
private String getValueSetInfo(ValueSet theValueSet) {
StringBuilder sb = new StringBuilder();
boolean isIdentified = false;
sb
.append("ValueSet:");
if (theValueSet.hasId()) {
isIdentified = true;
sb
.append(" ValueSet.id[")
.append(theValueSet.getId())
.append("]");
}
if (theValueSet.hasUrl()) {
isIdentified = true;
sb
.append(" ValueSet.url[")
.append("ValueSet.url[")
.append(theValueSet.getUrl())
.append("]");
}
if (theValueSet.hasIdentifier()) {
} else if (theValueSet.hasId()) {
isIdentified = true;
sb
.append(" ValueSet.identifier[")
.append(theValueSet.getIdentifierFirstRep().getSystem())
.append("|")
.append(theValueSet.getIdentifierFirstRep().getValue())
.append("ValueSet.id[")
.append(theValueSet.getId())
.append("]");
}
if (!isIdentified) {
sb.append(" None of ValueSet.id, ValueSet.url, and ValueSet.identifier are provided.");
sb.append("Unidentified ValueSet");
}
return sb.toString();
}
protected List<FhirVersionIndependentConcept> expandValueSetAndReturnVersionIndependentConcepts(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpandR4, FhirVersionIndependentConcept theWantConceptOrNull) {
org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent expandedR4 = expandValueSetInMemory(theExpansionOptions, theValueSetToExpandR4, theWantConceptOrNull).getExpansion();
protected List<FhirVersionIndependentConcept> expandValueSetAndReturnVersionIndependentConcepts(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpandR4, @Nonnull ExpansionFilter theExpansionFilter) {
int maxCapacity = myDaoConfig.getMaximumExpansionSize();
ValueSetExpansionComponentWithConceptAccumulator accumulator = new ValueSetExpansionComponentWithConceptAccumulator(myContext, maxCapacity);
expandValueSet(theExpansionOptions, theValueSetToExpandR4, accumulator, theExpansionFilter);
ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
for (org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent nextContains : expandedR4.getContains()) {
for (org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent nextContains : accumulator.getContains()) {
retVal.add(new FhirVersionIndependentConcept(nextContains.getSystem(), nextContains.getCode(), nextContains.getDisplay(), nextContains.getVersion()));
}
return retVal;
@ -652,7 +717,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
/**
* @return Returns true if there are potentially more results to process.
*/
private Boolean expandValueSetHandleIncludeOrExclude(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, AtomicInteger theCodeCounter, int theQueryIndex, FhirVersionIndependentConcept theWantConceptOrNull) {
private Boolean expandValueSetHandleIncludeOrExclude(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter) {
String system = theIncludeOrExclude.getSystem();
boolean hasSystem = isNotBlank(system);
@ -660,7 +725,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
if (hasSystem) {
if (theWantConceptOrNull != null && theWantConceptOrNull.getSystem() != null && !system.equals(theWantConceptOrNull.getSystem())) {
if (theExpansionFilter.hasCode() && theExpansionFilter.getSystem() != null && !system.equals(theExpansionFilter.getSystem())) {
return false;
}
@ -669,13 +734,13 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system);
if (cs != null) {
return expandValueSetHandleIncludeOrExcludeUsingDatabase(theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, theAdd, theCodeCounter, theQueryIndex, theWantConceptOrNull, system, cs);
return expandValueSetHandleIncludeOrExcludeUsingDatabase(theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, theAdd, theQueryIndex, theExpansionFilter, system, cs);
} else {
if (theIncludeOrExclude.getConcept().size() > 0 && theWantConceptOrNull != null) {
if (defaultString(theIncludeOrExclude.getSystem()).equals(theWantConceptOrNull.getSystem())) {
if (theIncludeOrExclude.getConcept().stream().noneMatch(t -> t.getCode().equals(theWantConceptOrNull.getCode()))) {
if (theIncludeOrExclude.getConcept().size() > 0 && theExpansionFilter.hasCode()) {
if (defaultString(theIncludeOrExclude.getSystem()).equals(theExpansionFilter.getSystem())) {
if (theIncludeOrExclude.getConcept().stream().noneMatch(t -> t.getCode().equals(theExpansionFilter.getCode()))) {
return false;
}
}
@ -691,9 +756,9 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
// if someone creates a valueset that includes UCUM codes, since we don't have a CodeSystem resource for those
// but CommonCodeSystemsTerminologyService can validate individual codes.
List<FhirVersionIndependentConcept> includedConcepts = null;
if (theWantConceptOrNull != null) {
if (theExpansionFilter.hasCode()) {
includedConcepts = new ArrayList<>();
includedConcepts.add(theWantConceptOrNull);
includedConcepts.add(theExpansionFilter.toFhirVersionIndependentConcept());
} else if (!theIncludeOrExclude.getConcept().isEmpty()) {
includedConcepts = theIncludeOrExclude
.getConcept()
@ -737,7 +802,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
if (!theIncludeOrExclude.getConcept().isEmpty()) {
for (ValueSet.ConceptReferenceComponent next : theIncludeOrExclude.getConcept()) {
String nextCode = next.getCode();
if (theWantConceptOrNull == null || theWantConceptOrNull.getCode().equals(nextCode)) {
if (!theExpansionFilter.hasCode() || theExpansionFilter.getCode().equals(nextCode)) {
if (isNoneBlank(system, nextCode) && !theAddedCodes.contains(system + "|" + nextCode)) {
CodeSystem.ConceptDefinitionComponent code = findCode(codeSystemFromContext.getConcept(), nextCode);
@ -751,7 +816,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
} else {
List<CodeSystem.ConceptDefinitionComponent> concept = codeSystemFromContext.getConcept();
addConceptsToList(theValueSetCodeAccumulator, theAddedCodes, system, concept, theAdd, theWantConceptOrNull);
addConceptsToList(theValueSetCodeAccumulator, theAddedCodes, system, concept, theAdd, theExpansionFilter);
}
return false;
@ -760,44 +825,20 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
} else if (hasValueSet) {
for (CanonicalType nextValueSet : theIncludeOrExclude.getValueSet()) {
ourLog.debug("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), nextValueSet.getValueAsString());
String valueSetUrl = nextValueSet.getValueAsString();
ourLog.debug("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), valueSetUrl);
List<FhirVersionIndependentConcept> expanded = expandValueSet(theExpansionOptions, nextValueSet.getValueAsString());
Map<String, TermCodeSystem> uriToCodeSystem = new HashMap<>();
ExpansionFilter subExpansionFilter = new ExpansionFilter(theExpansionFilter, theIncludeOrExclude.getFilter(), theValueSetCodeAccumulator.getCapacityRemaining());
for (FhirVersionIndependentConcept nextConcept : expanded) {
if (theAdd) {
// TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
if (!uriToCodeSystem.containsKey(nextConcept.getSystem())) {
TermCodeSystem codeSystem = myCodeSystemDao.findByCodeSystemUri(nextConcept.getSystem());
uriToCodeSystem.put(nextConcept.getSystem(), codeSystem);
}
TermCodeSystem codeSystem = uriToCodeSystem.get(nextConcept.getSystem());
if (codeSystem != null) {
TermCodeSystemVersion termCodeSystemVersion;
if (nextConcept.getSystemVersion() != null) {
termCodeSystemVersion = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(codeSystem.getPid(), nextConcept.getSystemVersion());
} else {
termCodeSystemVersion = codeSystem.getCurrentVersion();
}
myConceptDao
.findByCodeSystemAndCode(termCodeSystemVersion, nextConcept.getCode())
.ifPresent(concept ->
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, theCodeCounter, nextConcept.getSystemVersion())
);
} else {
// This will happen if we're expanding against a built-in (part of FHIR) ValueSet that
// isn't actually in the database anywhere
Collection<TermConceptDesignation> emptyCollection = Collections.emptyList();
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, emptyCollection, theAdd, theCodeCounter, nextConcept.getSystem(), nextConcept.getSystemVersion(), nextConcept.getCode(), nextConcept.getDisplay());
}
}
if (isNoneBlank(nextConcept.getSystem(), nextConcept.getCode()) && !theAdd && theAddedCodes.remove(nextConcept.getSystem() + "|" + nextConcept.getCode())) {
theValueSetCodeAccumulator.excludeConcept(nextConcept.getSystem(), nextConcept.getCode());
}
ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(valueSetUrl);
if (valueSet == null) {
throw new ResourceNotFoundException("Unknown ValueSet: " + UrlUtil.escapeUrlParam(valueSetUrl));
}
expandValueSetIntoAccumulator(valueSet, theExpansionOptions, theValueSetCodeAccumulator, subExpansionFilter, theAdd);
}
return false;
@ -809,8 +850,12 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
private boolean isHibernateSearchEnabled() {
return myFulltextSearchSvc != null;
}
@Nonnull
private Boolean expandValueSetHandleIncludeOrExcludeUsingDatabase(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, AtomicInteger theCodeCounter, int theQueryIndex, FhirVersionIndependentConcept theWantConceptOrNull, String theSystem, TermCodeSystem theCs) {
private Boolean expandValueSetHandleIncludeOrExcludeUsingDatabase(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter, String theSystem, TermCodeSystem theCs) {
String includeOrExcludeVersion = theIncludeOrExclude.getVersion();
TermCodeSystemVersion csv;
if (isEmpty(includeOrExcludeVersion)) {
@ -824,8 +869,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
* If FullText searching is not enabled, we can handle only basic expansions
* since we're going to do it without the database.
*/
if (myFulltextSearchSvc == null) {
expandWithoutHibernateSearch(theValueSetCodeAccumulator, csv, theAddedCodes, theIncludeOrExclude, theSystem, theAdd, theCodeCounter);
if (!isHibernateSearchEnabled()) {
expandWithoutHibernateSearch(theValueSetCodeAccumulator, csv, theAddedCodes, theIncludeOrExclude, theSystem, theAdd);
return false;
}
@ -837,8 +882,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
bool.must(qb.keyword().onField("myCodeSystemVersionPid").matching(csv.getPid()).createQuery());
if (theWantConceptOrNull != null) {
bool.must(qb.keyword().onField("myCode").matching(theWantConceptOrNull.getCode()).createQuery());
if (theExpansionFilter.hasCode()) {
bool.must(qb.keyword().onField("myCode").matching(theExpansionFilter.getCode()).createQuery());
}
/*
@ -850,7 +895,12 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
} else {
codeSystemUrlAndVersion = theSystem;
}
handleFilters(bool, codeSystemUrlAndVersion, qb, theIncludeOrExclude);
for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) {
handleFilter(codeSystemUrlAndVersion, qb, bool, nextFilter);
}
for (ValueSet.ConceptSetFilterComponent nextFilter : theExpansionFilter.getFilters()) {
handleFilter(codeSystemUrlAndVersion, qb, bool, nextFilter);
}
Query luceneQuery = bool.createQuery();
@ -917,21 +967,22 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
StopWatch swForBatch = new StopWatch();
AtomicInteger countForBatch = new AtomicInteger(0);
List resultList = jpaQuery.getResultList();
List<?> resultList = jpaQuery.getResultList();
int resultsInBatch = resultList.size();
int firstResult = jpaQuery.getFirstResult();
int delta = 0;
for (Object next : resultList) {
count.incrementAndGet();
countForBatch.incrementAndGet();
TermConcept concept = (TermConcept) next;
try {
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, theCodeCounter, includeOrExcludeVersion);
} catch (ExpansionTooCostlyException e) {
return false;
boolean added = addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, includeOrExcludeVersion);
if (added) {
delta++;
}
}
ourLog.debug("Batch expansion for {} with starting index of {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), firstResult, countForBatch, swForBatch.getMillis());
theValueSetCodeAccumulator.incrementOrDecrementTotalConcepts(theAdd, delta);
if (resultsInBatch < maxResultsPerBatch) {
ourLog.debug("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis());
@ -959,14 +1010,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
}
private void handleFilters(BooleanJunction<?> theBool, String theCodeSystemIdentifier, QueryBuilder theQb, ValueSet.ConceptSetComponent theIncludeOrExclude) {
if (theIncludeOrExclude.getFilter().size() > 0) {
for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) {
handleFilter(theCodeSystemIdentifier, theQb, theBool, nextFilter);
}
}
}
private void handleFilter(String theCodeSystemIdentifier, QueryBuilder theQb, BooleanJunction<?> theBool, ValueSet.ConceptSetFilterComponent theFilter) {
if (isBlank(theFilter.getValue()) && theFilter.getOp() == null && isBlank(theFilter.getProperty())) {
return;
@ -1264,7 +1307,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
}
private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, TermCodeSystemVersion theVersion, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, AtomicInteger theCodeCounter) {
private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, TermCodeSystemVersion theVersion, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd) {
ourLog.trace("Hibernate search is not enabled");
if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) {
@ -1277,7 +1320,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
if (theInclude.getConcept().isEmpty()) {
for (TermConcept next : theVersion.getConcepts()) {
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, null, theAdd, theCodeCounter, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay());
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay());
}
}
@ -1285,7 +1328,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
if (!theSystem.equals(theInclude.getSystem()) && isNotBlank(theSystem)) {
continue;
}
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, null, theAdd, theCodeCounter, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay());
addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay());
}
@ -1447,14 +1490,13 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
@Nullable
private TermCodeSystemVersion getCurrentCodeSystemVersion(String theCodeSystemIdentifier) {
String myVersion = getVersionFromIdentifier(theCodeSystemIdentifier);
String key = theCodeSystemIdentifier;
TermCodeSystemVersion retVal = myCodeSystemCurrentVersionCache.get(key.toString(), t -> myTxTemplate.execute(tx -> {
String version = getVersionFromIdentifier(theCodeSystemIdentifier);
TermCodeSystemVersion retVal = myCodeSystemCurrentVersionCache.get(theCodeSystemIdentifier, t -> myTxTemplate.execute(tx -> {
TermCodeSystemVersion csv = null;
TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(getUrlFromIdentifier(theCodeSystemIdentifier));
if (cs != null) {
if (myVersion != null) {
csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), myVersion);
if (version != null) {
csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), version);
} else if (cs.getCurrentVersion() != null) {
csv = cs.getCurrentVersion();
}
@ -1642,7 +1684,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
*/
String conceptMapUrl = termConceptMap.getUrl();
String conceptMapVersion = termConceptMap.getVersion();
Optional<TermConceptMap> optionalExistingTermConceptMapByUrl = null;
Optional<TermConceptMap> optionalExistingTermConceptMapByUrl;
if (isBlank(conceptMapVersion)) {
optionalExistingTermConceptMapByUrl = myConceptMapDao.findTermConceptMapByUrlAndNullVersion(conceptMapUrl);
} else {
@ -1776,9 +1818,10 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
// We have a ValueSet to pre-expand.
try {
ValueSet valueSet = txTemplate.execute(t -> {
TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).get();
TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).orElseThrow(()->new IllegalStateException("Unknown VS ID: " + valueSetToExpand.getId()));
return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
});
assert valueSet != null;
ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator(valueSetToExpand, myValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao);
expandValueSet(null, valueSet, accumulator);
@ -2604,6 +2647,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
String systemVersion = theCodeSystemIdentifierType != null ? getVersionFromIdentifier(theCodeSystemIdentifierType.getValueAsString()): null;
if (theCodingType != null) {
Coding canonicalizedCoding = toCanonicalCoding(theCodingType);
assert canonicalizedCoding != null; // Shouldn't be null, since theCodingType isn't
code = canonicalizedCoding.getCode();
system = canonicalizedCoding.getSystem();
systemVersion = canonicalizedCoding.getVersion();
@ -2675,7 +2719,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
String code = theCode;
String version = theVersion;
String display = theDisplay;
if (haveCodeableConcept) {
@ -2689,7 +2732,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
code = nextCoding.getCode();
display = nextCoding.getDisplay();
CodeValidationResult nextValidation = codeSystemValidateCode(codeSystemUrl, version, code, display);
CodeValidationResult nextValidation = codeSystemValidateCode(codeSystemUrl, theVersion, code, display);
if (nextValidation.isOk() || i == codeableConcept.getCoding().size() - 1) {
return nextValidation;
}
@ -2705,7 +2748,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
display = coding.getDisplay();
}
return codeSystemValidateCode(codeSystemUrl, version, code, display);
return codeSystemValidateCode(codeSystemUrl, theVersion, code, display);
}

98
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ExpansionFilter.java Normal file
View File

@ -0,0 +1,98 @@
package ca.uhn.fhir.jpa.term;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2020 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.util.FhirVersionIndependentConcept;
import org.apache.commons.lang3.Validate;
import org.hl7.fhir.r4.model.ValueSet;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.List;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
class ExpansionFilter {
public static final ExpansionFilter NO_FILTER = new ExpansionFilter(null, null);
private final String myCode;
private final String mySystem;
private final List<ValueSet.ConceptSetFilterComponent> myFilters;
private final Integer myMaxCount;
/**
* Constructor
*/
ExpansionFilter(String theSystem, String theCode) {
this(theSystem, theCode, Collections.emptyList(), null);
}
/**
* Constructor
*/
ExpansionFilter(ExpansionFilter theExpansionFilter, List<ValueSet.ConceptSetFilterComponent> theFilters, Integer theMaxCount) {
this(theExpansionFilter.getSystem(), theExpansionFilter.getCode(), theFilters, theMaxCount);
}
/**
* Constructor
*/
ExpansionFilter(@Nullable String theSystem, @Nullable String theCode, @Nonnull List<ValueSet.ConceptSetFilterComponent> theFilters, Integer theMaxCount) {
Validate.isTrue(isNotBlank(theSystem) == isNotBlank(theCode));
Validate.notNull(theFilters);
mySystem = theSystem;
myCode = theCode;
myFilters = theFilters;
myMaxCount = theMaxCount;
}
public List<ValueSet.ConceptSetFilterComponent> getFilters() {
return myFilters;
}
boolean hasCode() {
return myCode != null;
}
String getCode() {
return myCode;
}
String getSystem() {
return mySystem;
}
/**
* Converts the system/code in this filter to a FhirVersionIndependentConcept. This method
* should not be called if {@link #hasCode()} returns <code>false</code>
*/
@Nonnull
public FhirVersionIndependentConcept toFhirVersionIndependentConcept() {
Validate.isTrue(hasCode());
return new FhirVersionIndependentConcept(mySystem, myCode);
}
public Integer getMaxCount() {
return myMaxCount;
}
}

27
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetConceptAccumulator.java
View File

@ -33,11 +33,36 @@ public interface IValueSetConceptAccumulator {
void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, @Nullable Collection<TermConceptDesignation> theDesignations);
void excludeConcept(String theSystem, String theCode);
/**
* @return Returns <code>true</code> if the code was actually present and was removed
*/
boolean excludeConcept(String theSystem, String theCode);
@Nullable
default Integer getCapacityRemaining() {
return null;
}
@Nullable
default Integer getSkipCountRemaining() {
return null;
}
@Nullable
default void consumeSkipCount(int theSkipCountToConsume) {
// nothing
}
/**
* Add or subtract from the total concept count (this is not necessarily the same thing as the number of concepts in
* the accumulator, since the <code>offset</code> and <code>count</code> parameters applied to the expansion can cause
* concepts to not actually be added.
*
* @param theAdd If <code>true</code>, increment. If <code>false</code>, decrement.
* @param theDelta The number of codes to add or subtract
*/
default void incrementOrDecrementTotalConcepts(boolean theAdd, int theDelta) {
// nothing
}
}

11
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImpl.java
View File

@ -55,6 +55,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
@ -157,7 +158,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
StopWatch stopwatch = new StopWatch();
int count = Math.min(1000, myDeferredConcepts.size());
ourLog.info("Saving {} deferred concepts...", count);
ourLog.debug("Saving {} deferred concepts...", count);
while (codeCount < count && myDeferredConcepts.size() > 0) {
TermConcept next = myDeferredConcepts.remove(0);
if(myCodeSystemVersionDao.findById(next.getCodeSystemVersion().getPid()).isPresent()) {
@ -174,8 +175,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
}
if (codeCount > 0) {
ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)",
codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount));
ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({} codes/sec)",
codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.formatThroughput(codeCount, TimeUnit.SECONDS));
}
if (codeCount == 0) {
@ -198,8 +199,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
}
if (relCount > 0) {
ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({}ms / entry)",
relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(relCount));
ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({} entries/sec)",
relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.formatThroughput(relCount, TimeUnit.SECONDS));
}
if ((myDeferredConcepts.size() + myConceptLinksToSaveLater.size()) == 0) {

5
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java
View File

@ -81,9 +81,9 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
}
@Override
public void excludeConcept(String theSystem, String theCode) {
public boolean excludeConcept(String theSystem, String theCode) {
if (isAnyBlank(theSystem, theCode)) {
return;
return false;
}
// Get existing entity so it can be deleted.
@ -114,6 +114,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
ourLog.info("Have excluded {} concepts from ValueSet[{}]", myConceptsExcluded, myTermValueSet.getUrl());
}
}
return false;
}
private TermValueSetConcept saveConcept(String theSystem, String theCode, String theDisplay) {

108
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithConceptAccumulator.java
View File

@ -25,19 +25,25 @@ import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
import ca.uhn.fhir.jpa.term.ex.ExpansionTooCostlyException;
import ca.uhn.fhir.model.api.annotation.Block;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.util.HapiExtensions;
import org.apache.commons.lang3.StringUtils;
import org.hl7.fhir.r4.model.StringType;
import org.hl7.fhir.r4.model.ValueSet;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@Block()
public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.ValueSetExpansionComponent implements IValueSetConceptAccumulator {
private final int myMaxCapacity;
private final FhirContext myContext;
private int myConceptsCount;
private int mySkipCountRemaining;
private int myHardExpansionMaximumSize;
private List<String> myMessages;
private int myAddedConcepts;
private Integer myTotalConcepts;
/**
* Constructor
@ -46,27 +52,40 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
* an {@link InternalErrorException}
*/
ValueSetExpansionComponentWithConceptAccumulator(FhirContext theContext, int theMaxCapacity) {
myContext = theContext;
myMaxCapacity = theMaxCapacity;
myConceptsCount = 0;
myContext = theContext;
}
@Nullable
@Nonnull
@Override
public Integer getCapacityRemaining() {
return myMaxCapacity - myConceptsCount;
return (myMaxCapacity - myAddedConcepts) + mySkipCountRemaining;
}
public List<String> getMessages() {
if (myMessages == null) {
return Collections.emptyList();
}
return Collections.unmodifiableList(myMessages);
}
@Override
public void addMessage(String theMessage) {
addExtension()
.setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE)
.setValue(new StringType(theMessage));
if (myMessages == null) {
myMessages = new ArrayList<>();
}
myMessages.add(theMessage);
}
@Override
public void includeConcept(String theSystem, String theCode, String theDisplay) {
if (mySkipCountRemaining > 0) {
mySkipCountRemaining--;
return;
}
incrementConceptsCount();
ValueSet.ValueSetExpansionContainsComponent contains = this.addContains();
setSystemAndVersion(theSystem, contains);
contains.setCode(theCode);
@ -75,7 +94,13 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
@Override
public void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, Collection<TermConceptDesignation> theDesignations) {
if (mySkipCountRemaining > 0) {
mySkipCountRemaining--;
return;
}
incrementConceptsCount();
ValueSet.ValueSetExpansionContainsComponent contains = this.addContains();
setSystemAndVersion(theSystem, contains);
contains.setCode(theCode);
@ -95,11 +120,22 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
}
@Override
public void excludeConcept(String theSystem, String theCode) {
public void consumeSkipCount(int theSkipCountToConsume) {
mySkipCountRemaining -= theSkipCountToConsume;
}
@Nullable
@Override
public Integer getSkipCountRemaining() {
return mySkipCountRemaining;
}
@Override
public boolean excludeConcept(String theSystem, String theCode) {
String excludeSystem;
String excludeSystemVersion;
int versionSeparator = theSystem.indexOf("|");
if(versionSeparator > -1) {
if (versionSeparator > -1) {
excludeSystemVersion = theSystem.substring(versionSeparator + 1);
excludeSystem = theSystem.substring(0, versionSeparator);
} else {
@ -107,22 +143,47 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
excludeSystemVersion = null;
}
if (excludeSystemVersion != null) {
this.getContains().removeIf(t ->
return this.getContains().removeIf(t ->
excludeSystem.equals(t.getSystem()) &&
theCode.equals(t.getCode()) &&
excludeSystemVersion.equals(t.getVersion()));
theCode.equals(t.getCode()) &&
excludeSystemVersion.equals(t.getVersion()));
} else {
this.getContains().removeIf(t ->
theSystem.equals(t.getSystem()) &&
theCode.equals(t.getCode()));
return this.getContains().removeIf(t ->
theSystem.equals(t.getSystem()) &&
theCode.equals(t.getCode()));
}
}
private void incrementConceptsCount() {
if (++myConceptsCount > myMaxCapacity) {
Integer capacityRemaining = getCapacityRemaining();
if (capacityRemaining == 0) {
String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myMaxCapacity);
throw new ExpansionTooCostlyException(msg);
}
if (myHardExpansionMaximumSize > 0 && myAddedConcepts > myHardExpansionMaximumSize) {
String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myHardExpansionMaximumSize);
throw new ExpansionTooCostlyException(msg);
}
myAddedConcepts++;
}
public Integer getTotalConcepts() {
return myTotalConcepts;
}
@Override
public void incrementOrDecrementTotalConcepts(boolean theAdd, int theDelta) {
int delta = theDelta;
if (!theAdd) {
delta = -delta;
}
if (myTotalConcepts == null) {
myTotalConcepts = delta;
} else {
myTotalConcepts = myTotalConcepts + delta;
}
}
private void setSystemAndVersion(String theSystemAndVersion, ValueSet.ValueSetExpansionContainsComponent myComponent) {
@ -130,11 +191,18 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
int versionSeparator = theSystemAndVersion.lastIndexOf('|');
if (versionSeparator != -1) {
myComponent.setVersion(theSystemAndVersion.substring(versionSeparator + 1));
myComponent.setSystem(theSystemAndVersion.substring(0,versionSeparator));
myComponent.setSystem(theSystemAndVersion.substring(0, versionSeparator));
} else {
myComponent.setSystem(theSystemAndVersion);
}
}
}
public void setSkipCountRemaining(int theSkipCountRemaining) {
mySkipCountRemaining = theSkipCountRemaining;
}
public void setHardExpansionMaximumSize(int theHardExpansionMaximumSize) {
myHardExpansionMaximumSize = theHardExpansionMaximumSize;
}
}

19
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java
View File

@ -32,7 +32,9 @@ import org.springframework.test.util.AopTestUtils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.fail;
@ -100,7 +102,7 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test {
@Test
public void testValidateQuestionnaireResponseWithValueSetIncludingCompleteCodeSystem() throws IOException {
public void testExpandLocalCodeSystemWithExplicitCodes() throws IOException {
CodeSystem cs = loadResourceFromClasspath(CodeSystem.class, "/dstu3/iar/CodeSystem-iar-citizenship-status.xml");
myCodeSystemDao.create(cs);
@ -110,18 +112,9 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test {
ValueSet expansion = myValueSetDao.expandByIdentifier("http://ccim.on.ca/fhir/iar/ValueSet/iar-citizenship-status", null);
ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
// Questionnaire q = loadResourceFromClasspath(Questionnaire.class,"/dstu3/iar/Questionnaire-iar-test.xml" );
// myQuestionnaireDao.create(q);
//
//
//
// Bundle bundleForValidation = loadResourceFromClasspath(Bundle.class, "/dstu3/iar/Bundle-for-validation.xml");
// try {
// MethodOutcome outcome = myBundleDao.validate(bundleForValidation, null, null, null, null, null, null);
// ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome.getOperationOutcome()));
// } catch (PreconditionFailedException e) {
// ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome()));
// }
assertThat(expansion.getExpansion().getContains().stream().map(t->t.getCode()).collect(Collectors.toList()), containsInAnyOrder(
"CDN", "PR", "TR", "REF", "UNK", "ASKU"
));
}

1
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java
View File

@ -1015,6 +1015,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
myAllergyIntoleranceDao.create(ai3, mySrd).getId().toUnqualifiedVersionless().getValue();
SearchParameterMap params;
params = new SearchParameterMap();
params.add(AllergyIntolerance.SP_CLINICAL_STATUS, new TokenParam(null, "active"));
assertThat(toUnqualifiedVersionlessIdValues(myAllergyIntoleranceDao.search(params)), containsInAnyOrder(id1));

31
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/JpaPackageCacheTest.java
View File

@ -1,7 +1,11 @@
package ca.uhn.fhir.jpa.packages;
import ca.uhn.fhir.jpa.dao.data.INpmPackageDao;
import ca.uhn.fhir.jpa.dao.data.INpmPackageVersionDao;
import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.util.JsonUtil;
import org.hl7.fhir.utilities.npm.IPackageCacheManager;
import org.hl7.fhir.utilities.npm.NpmPackage;
import org.junit.jupiter.api.Test;
@ -10,13 +14,21 @@ import org.springframework.beans.factory.annotation.Autowired;
import java.io.IOException;
import java.io.InputStream;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
public class JpaPackageCacheTest extends BaseJpaR4Test {
@Autowired
private IPackageCacheManager myPackageCacheManager;
private IHapiPackageCacheManager myPackageCacheManager;
@Autowired
private INpmPackageDao myPackageDao;
@Autowired
private INpmPackageVersionDao myPackageVersionDao;
@Test
public void testSavePackage() throws IOException {
@ -41,6 +53,23 @@ public class JpaPackageCacheTest extends BaseJpaR4Test {
}
@Test
public void testSavePackageWithLongDescription() throws IOException {
try (InputStream stream = IgInstallerDstu3Test.class.getResourceAsStream("/packages/package-davinci-cdex-0.2.0.tgz")) {
myPackageCacheManager.addPackageToCache("hl7.fhir.us.davinci-cdex", "0.2.0", stream, "hl7.fhir.us.davinci-cdex");
}
NpmPackage pkg;
pkg = myPackageCacheManager.loadPackage("hl7.fhir.us.davinci-cdex", null);
assertEquals("0.2.0", pkg.version());
assertEquals("This IG provides detailed guidance that helps implementers use FHIR-based interactions and resources relevant to support specific exchanges of clinical information between provider and payers (or ...", myPackageDao.findByPackageId("hl7.fhir.us.davinci-cdex").get().getDescription());
assertEquals("This IG provides detailed guidance that helps implementers use FHIR-based interactions and resources relevant to support specific exchanges of clinical information between provider and payers (or ...", myPackageVersionDao.findByPackageIdAndVersion("hl7.fhir.us.davinci-cdex", "0.2.0").get().getDescription());
}
@Test
public void testSavePackageCorrectFhirVersion() {

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java
View File

@ -168,7 +168,7 @@ public abstract class BaseResourceProviderR4Test extends BaseJpaR4Test {
confProvider.setSearchParamRegistry(ourSearchParamRegistry);
myFhirCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
myFhirCtx.getRestfulClientFactory().setSocketTimeout(5000000);
myFhirCtx.getRestfulClientFactory().setSocketTimeout(20000);
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
HttpClientBuilder builder = HttpClientBuilder.create();

251
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ValueSetLuceneDisabledTest.java Normal file
View File

@ -0,0 +1,251 @@
package ca.uhn.fhir.jpa.provider.r4;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet;
import ca.uhn.fhir.jpa.config.TestR4WithLuceneDisabledConfig;
import ca.uhn.fhir.jpa.dao.BaseJpaTest;
import ca.uhn.fhir.jpa.dao.dstu2.FhirResourceDaoDstu2SearchNoFtTest;
import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
import ca.uhn.fhir.jpa.subscription.match.config.WebsocketDispatcherConfig;
import ca.uhn.fhir.parser.IParser;
import ca.uhn.fhir.parser.StrictErrorHandler;
import ca.uhn.fhir.rest.api.EncodingEnum;
import ca.uhn.fhir.rest.client.api.IGenericClient;
import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum;
import ca.uhn.fhir.rest.server.RestfulServer;
import ca.uhn.fhir.rest.server.interceptor.CorsInterceptor;
import ca.uhn.fhir.rest.server.provider.ResourceProviderFactory;
import ca.uhn.fhir.test.utilities.JettyUtil;
import org.apache.commons.io.IOUtils;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.CodeSystem;
import org.hl7.fhir.r4.model.CodeableConcept;
import org.hl7.fhir.r4.model.Coding;
import org.hl7.fhir.r4.model.Parameters;
import org.hl7.fhir.r4.model.ValueSet;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallbackWithoutResult;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.web.context.ContextLoader;
import org.springframework.web.context.WebApplicationContext;
import org.springframework.web.context.support.AnnotationConfigWebApplicationContext;
import org.springframework.web.context.support.GenericWebApplicationContext;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.servlet.DispatcherServlet;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.junit.jupiter.api.Assertions.fail;
@ExtendWith(SpringExtension.class)
@ContextConfiguration(classes = {TestR4WithLuceneDisabledConfig.class})
public class ResourceProviderR4ValueSetLuceneDisabledTest extends BaseJpaTest {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceProviderR4ValueSetLuceneDisabledTest.class);
private static RestfulServer ourRestServer;
private static String ourServerBase;
private static Server ourServer;
private static DatabaseBackedPagingProvider ourPagingProvider;
@Autowired
private FhirContext myFhirCtx;
@Autowired
private PlatformTransactionManager myTxManager;
@Autowired
@Qualifier("myCodeSystemDaoR4")
private IFhirResourceDaoCodeSystem<CodeSystem, Coding, CodeableConcept> myCodeSystemDao;
@Autowired
@Qualifier("myValueSetDaoR4")
private IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> myValueSetDao;
@Autowired
@Qualifier("myResourceProvidersR4")
private ResourceProviderFactory myResourceProviders;
@Autowired
private ApplicationContext myAppCtx;
private IIdType myExtensionalCsId;
private IIdType myExtensionalVsId;
private IGenericClient myClient;
private void loadAndPersistCodeSystemAndValueSet() throws IOException {
loadAndPersistCodeSystem();
loadAndPersistValueSet();
}
private <T extends IBaseResource> T loadResourceFromClasspath(Class<T> type, String resourceName) throws IOException {
InputStream stream = FhirResourceDaoDstu2SearchNoFtTest.class.getResourceAsStream(resourceName);
if (stream == null) {
fail("Unable to load resource: " + resourceName);
}
String string = IOUtils.toString(stream, StandardCharsets.UTF_8);
IParser newJsonParser = EncodingEnum.detectEncodingNoDefault(string).newParser(myFhirCtx);
return newJsonParser.parseResource(type, string);
}
private void loadAndPersistCodeSystem() throws IOException {
CodeSystem codeSystem = loadResourceFromClasspath(CodeSystem.class, "/extensional-case-3-cs.xml");
codeSystem.setId("CodeSystem/cs");
persistCodeSystem(codeSystem);
}
private void persistCodeSystem(CodeSystem theCodeSystem) {
new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theStatus) {
myExtensionalCsId = myCodeSystemDao.create(theCodeSystem, mySrd).getId().toUnqualifiedVersionless();
}
});
myCodeSystemDao.readEntity(myExtensionalCsId, null).getId();
}
private void loadAndPersistValueSet() throws IOException {
ValueSet valueSet = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml");
valueSet.setId("ValueSet/vs");
persistValueSet(valueSet);
}
private void persistValueSet(ValueSet theValueSet) {
new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theStatus) {
myExtensionalVsId = myValueSetDao.create(theValueSet, mySrd).getId().toUnqualifiedVersionless();
}
});
myValueSetDao.readEntity(myExtensionalVsId, null).getId();
}
@Override
protected FhirContext getContext() {
return myFhirCtx;
}
@Override
protected PlatformTransactionManager getTxManager() {
return myTxManager;
}
@Test
public void testExpandById() throws Exception {
loadAndPersistCodeSystemAndValueSet();
Parameters respParam = myClient
.operation()
.onInstance(myExtensionalVsId)
.named("expand")
.withNoParameters(Parameters.class)
.execute();
ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource();
String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded);
ourLog.info(resp);
assertThat(resp, containsString("<ValueSet xmlns=\"http://hl7.org/fhir\">"));
assertThat(resp, containsString("<expansion>"));
assertThat(resp, containsString("<contains>"));
assertThat(resp, containsString("<system value=\"http://acme.org\"/>"));
assertThat(resp, containsString("<code value=\"8450-9\"/>"));
assertThat(resp, containsString("<display value=\"Systolic blood pressure--expiration\"/>"));
assertThat(resp, containsString("</contains>"));
assertThat(resp, containsString("<contains>"));
assertThat(resp, containsString("<system value=\"http://acme.org\"/>"));
assertThat(resp, containsString("<code value=\"11378-7\"/>"));
assertThat(resp, containsString("<display value=\"Systolic blood pressure at First encounter\"/>"));
assertThat(resp, containsString("</contains>"));
assertThat(resp, containsString("</expansion>"));
}
@BeforeEach
public void before() throws Exception {
myFhirCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
myFhirCtx.getRestfulClientFactory().setSocketTimeout(1200 * 1000);
myFhirCtx.setParserErrorHandler(new StrictErrorHandler());
if (ourServer == null) {
ourRestServer = new RestfulServer(myFhirCtx);
ourRestServer.registerProviders(myResourceProviders.createProviders());
ourRestServer.setDefaultResponseEncoding(EncodingEnum.XML);
ourPagingProvider = myAppCtx.getBean(DatabaseBackedPagingProvider.class);
Server server = new Server(0);
ServletContextHandler proxyHandler = new ServletContextHandler();
proxyHandler.setContextPath("/");
ServletHolder servletHolder = new ServletHolder();
servletHolder.setServlet(ourRestServer);
proxyHandler.addServlet(servletHolder, "/fhir/context/*");
GenericWebApplicationContext webApplicationContext = new GenericWebApplicationContext();
webApplicationContext.setParent(myAppCtx);
webApplicationContext.refresh();
proxyHandler.getServletContext().setAttribute(WebApplicationContext.ROOT_WEB_APPLICATION_CONTEXT_ATTRIBUTE, webApplicationContext);
DispatcherServlet dispatcherServlet = new DispatcherServlet();
// dispatcherServlet.setApplicationContext(webApplicationContext);
dispatcherServlet.setContextClass(AnnotationConfigWebApplicationContext.class);
ServletHolder subsServletHolder = new ServletHolder();
subsServletHolder.setServlet(dispatcherServlet);
subsServletHolder.setInitParameter(
ContextLoader.CONFIG_LOCATION_PARAM,
WebsocketDispatcherConfig.class.getName());
proxyHandler.addServlet(subsServletHolder, "/*");
// Register a CORS filter
CorsConfiguration config = new CorsConfiguration();
CorsInterceptor corsInterceptor = new CorsInterceptor(config);
config.addAllowedHeader("x-fhir-starter");
config.addAllowedHeader("Origin");
config.addAllowedHeader("Accept");
config.addAllowedHeader("X-Requested-With");
config.addAllowedHeader("Content-Type");
config.addAllowedHeader("Access-Control-Request-Method");
config.addAllowedHeader("Access-Control-Request-Headers");
config.addAllowedOrigin("*");
config.addExposedHeader("Location");
config.addExposedHeader("Content-Location");
config.setAllowedMethods(Arrays.asList("GET", "POST", "PUT", "DELETE", "OPTIONS"));
ourRestServer.registerInterceptor(corsInterceptor);
server.setHandler(proxyHandler);
JettyUtil.startServer(server);
int port = JettyUtil.getPortForStartedServer(server);
ourServerBase = "http://localhost:" + port + "/fhir/context";
myFhirCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
myFhirCtx.getRestfulClientFactory().setSocketTimeout(20000);
ourServer = server;
}
ourRestServer.setPagingProvider(ourPagingProvider);
myClient = myFhirCtx.newRestfulGenericClient(ourServerBase);
}
}

278
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4Test.java
View File

@ -10,8 +10,11 @@ import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
import ca.uhn.fhir.jpa.entity.TermValueSetConceptDesignation;
import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.util.JpaConstants;
import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
import ca.uhn.fhir.jpa.util.SqlQuery;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.util.HapiExtensions;
import com.google.common.collect.Lists;
import org.hl7.fhir.instance.model.api.IIdType;
@ -31,9 +34,12 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.empty;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@ -87,6 +93,260 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
assertEquals(24, expandedValueSet.getExpansion().getContains().size());
}
@Test
public void testExpandInline_IncludeCodeSystem_FilterOnDisplay_NoFilter() throws Exception {
loadAndPersistCodeSystemWithDesignations(HttpVerb.PUT);
ValueSet input = new ValueSet();
input.getCompose()
.addInclude()
.setSystem("http://acme.org");
ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
ourLog.info("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
assertEquals(24, expandedValueSet.getExpansion().getTotal());
}
@Test
public void testExpandInline_IncludeCodeSystem_FilterOnDisplay_ExactFilter() throws Exception {
loadAndPersistCodeSystemWithDesignations(HttpVerb.PUT);
ValueSet input = new ValueSet();
input.getCompose()
.addInclude()
.setSystem("http://acme.org")
.addFilter()
.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("Systolic blood pressure--inspiration");
ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
ourLog.info("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
assertEquals(1, expandedValueSet.getExpansion().getTotal());
assertThat(expandedValueSet.getExpansion().getContains().stream().map(t -> t.getDisplay()).collect(Collectors.toList()), containsInAnyOrder(
"Systolic blood pressure--inspiration"
));
}
@Test
public void testExpandInline_IncludeCodeSystem_FilterOnDisplay_LeftMatchFilter() throws Exception {
loadAndPersistCodeSystemWithDesignations(HttpVerb.PUT);
ValueSet input = new ValueSet();
input.getCompose()
.addInclude()
.setSystem("http://acme.org")
.addFilter()
.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("Systolic blood pressure 1");
ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
ourLog.info("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
assertEquals(3, expandedValueSet.getExpansion().getTotal());
assertThat(expandedValueSet.getExpansion().getContains().stream().map(t -> t.getDisplay()).collect(Collectors.toList()), containsInAnyOrder(
"Systolic blood pressure 1 hour minimum",
"Systolic blood pressure 1 hour mean",
"Systolic blood pressure 1 hour maximum"
));
}
@Test
public void testExpandInline_IncludePreExpandedValueSetByUri_FilterOnDisplay_LeftMatch_SelectAll() {
myDaoConfig.setPreExpandValueSets(true);
create100ConceptsCodeSystemAndValueSet();
ValueSet input = new ValueSet();
input.getCompose()
.addInclude()
.addValueSet("http://foo/vs")
.addFilter()
.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("display value 9");
myCaptureQueriesListener.clear();
ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
ourLog.debug("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
assertThat(toCodes(expandedValueSet).toString(), toCodes(expandedValueSet), contains(
"code9", "code90", "code91", "code92", "code93", "code94", "code95", "code96", "code97", "code98", "code99"
));
assertEquals(11, expandedValueSet.getExpansion().getContains().size(), toCodes(expandedValueSet).toString());
assertEquals(11, expandedValueSet.getExpansion().getTotal());
// Make sure we used the pre-expanded version
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
assertThat(lastSelectQuery, containsString("concept_display like 'display value 9%'"));
}
@Test
public void testExpandInline_IncludePreExpandedValueSetByUri_FilterOnDisplay_LeftMatch_SelectRange() {
myDaoConfig.setPreExpandValueSets(true);
create100ConceptsCodeSystemAndValueSet();
ValueSet input = new ValueSet();
input.getCompose()
.addInclude()
.addValueSet("http://foo/vs")
.addFilter()
.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("display value 9");
myCaptureQueriesListener.clear();
ValueSetExpansionOptions expansionOptions = new ValueSetExpansionOptions()
.setOffset(3)
.setCount(4);
ValueSet expandedValueSet = myTermSvc.expandValueSet(expansionOptions, input);
ourLog.debug("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
assertThat(toCodes(expandedValueSet).toString(), toCodes(expandedValueSet), contains(
"code92", "code93", "code94", "code95"
));
assertEquals(4, expandedValueSet.getExpansion().getContains().size(), toCodes(expandedValueSet).toString());
assertEquals(11, expandedValueSet.getExpansion().getTotal());
// Make sure we used the pre-expanded version
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
assertThat(lastSelectQuery, containsString("concept_display like 'display value 9%'"));
}
@Test
public void testExpandInline_IncludePreExpandedValueSetByUri_ExcludeCodes_FilterOnDisplay_LeftMatch_SelectAll() {
myDaoConfig.setPreExpandValueSets(true);
create100ConceptsCodeSystemAndValueSet();
ValueSet input = new ValueSet();
input.getCompose()
.addInclude()
.addValueSet("http://foo/vs")
.addFilter()
.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("display value 9");
input.getCompose()
.addExclude()
.addValueSet("http://foo/vs")
.addFilter()
.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("display value 90");
myCaptureQueriesListener.clear();
ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
ourLog.debug("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
assertThat(toCodes(expandedValueSet).toString(), toCodes(expandedValueSet), contains(
"code9", "code91", "code92", "code93", "code94", "code95", "code96", "code97", "code98", "code99"
));
assertEquals(10, expandedValueSet.getExpansion().getContains().size(), toCodes(expandedValueSet).toString());
assertEquals(10, expandedValueSet.getExpansion().getTotal());
// Make sure we used the pre-expanded version
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
assertThat(lastSelectQuery, containsString("concept_display like 'display value 90%'"));
}
@Test
public void testExpandInline_IncludePreExpandedValueSetByUri_ExcludeCodes_FilterOnDisplay_LeftMatch_SelectRange() {
myDaoConfig.setPreExpandValueSets(true);
create100ConceptsCodeSystemAndValueSet();
ValueSet input = new ValueSet();
input.getCompose()
.addInclude()
.addValueSet("http://foo/vs")
.addFilter()
.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("display value 9");
input.getCompose()
.addExclude()
.addValueSet("http://foo/vs")
.addFilter()
.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("display value 90");
myCaptureQueriesListener.clear();
ValueSetExpansionOptions options = new ValueSetExpansionOptions();
options.setOffset(3);
options.setCount(4);
try {
myTermSvc.expandValueSet(options, input);
fail();
} catch (InvalidRequestException e) {
assertEquals("ValueSet expansion can not combine \"offset\" with \"ValueSet.compose.exclude\" unless the ValueSet has been pre-expanded. ValueSet \"Unidentified ValueSet\" must be pre-expanded for this operation to work.", e.getMessage());
}
}
public void create100ConceptsCodeSystemAndValueSet() {
CodeSystem cs = new CodeSystem();
cs.setUrl("http://foo/cs");
cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
myCodeSystemDao.create(cs);
CustomTerminologySet additions = new CustomTerminologySet();
for (int i = 0; i < 100; i++) {
additions.addRootConcept("code" + i, "display value " + i);
}
myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", additions);
ValueSet vs = new ValueSet();
vs.setUrl("http://foo/vs");
vs.getCompose().addInclude().setSystem("http://foo/cs");
myValueSetDao.create(vs);
myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
}
@Test
public void testExpandInline_IncludeNonPreExpandedValueSetByUri_FilterOnDisplay_LeftMatch() {
myDaoConfig.setPreExpandValueSets(true);
create100ConceptsCodeSystemAndValueSet();
ValueSet input = new ValueSet();
input.getCompose()
.addInclude()
.addValueSet("http://foo/vs")
.addFilter()
.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("display value 9");
myCaptureQueriesListener.clear();
ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
ourLog.debug("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
assertThat(toCodes(expandedValueSet).toString(), toCodes(expandedValueSet), contains(
"code9", "code90", "code91", "code92", "code93", "code94", "code95", "code96", "code97", "code98", "code99"
));
assertEquals(11, expandedValueSet.getExpansion().getContains().size(), toCodes(expandedValueSet).toString());
assertEquals(11, expandedValueSet.getExpansion().getTotal(), toCodes(expandedValueSet).toString());
// Make sure we used the pre-expanded version
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
assertThat(lastSelectQuery, containsString("concept_display like 'display value 9%'"));
}
@Nonnull
public List<String> toCodes(ValueSet theExpandedValueSet) {
return theExpandedValueSet.getExpansion().getContains().stream().map(t -> t.getCode()).collect(Collectors.toList());
}
@SuppressWarnings("SpellCheckingInspection")
@Test
public void testExpandTermValueSetAndChildren() throws Exception {
@ -115,11 +375,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getTotal());
assertEquals(myDaoConfig.getPreExpandValueSetsDefaultOffset(), expandedValueSet.getExpansion().getOffset());
assertEquals(2, expandedValueSet.getExpansion().getParameter().size());
assertEquals("offset", expandedValueSet.getExpansion().getParameter().get(0).getName());
assertEquals(0, expandedValueSet.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue());
assertEquals("count", expandedValueSet.getExpansion().getParameter().get(1).getName());
assertEquals(1000, expandedValueSet.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue());
assertEquals(0, expandedValueSet.getExpansion().getParameter().size());
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getContains().size());
@ -337,11 +593,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getTotal());
assertEquals(myDaoConfig.getPreExpandValueSetsDefaultOffset(), expandedValueSet.getExpansion().getOffset());
assertEquals(2, expandedValueSet.getExpansion().getParameter().size());
assertEquals("offset", expandedValueSet.getExpansion().getParameter().get(0).getName());
assertEquals(0, expandedValueSet.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue());
assertEquals("count", expandedValueSet.getExpansion().getParameter().get(1).getName());
assertEquals(1000, expandedValueSet.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue());
assertEquals(0, expandedValueSet.getExpansion().getParameter().size());
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getContains().size());
@ -796,7 +1048,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
String encoded = myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome);
ourLog.info(encoded);
Extension extensionByUrl = outcome.getExpansion().getExtensionByUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE);
Extension extensionByUrl = outcome.getMeta().getExtensionByUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE);
assertEquals("Unknown CodeSystem URI \"http://unknown-system\" referenced from ValueSet", extensionByUrl.getValueAsPrimitive().getValueAsString());
}
@ -877,8 +1129,8 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
ValueSet.ConceptSetComponent include = vs.getCompose().addInclude();
include.setSystem(CS_URL);
try {
myTermSvc.expandValueSet(null, vs);
fail();
ValueSet expansion = myTermSvc.expandValueSet(null, vs);
fail("Expanded to " + expansion.getExpansion().getContains().size() + " but max was " + myDaoConfig.getMaximumExpansionSize());
} catch (InternalErrorException e) {
assertEquals("Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
}

BIN
hapi-fhir-jpaserver-base/src/test/resources/packages/package-davinci-cdex-0.2.0.tgz Normal file
View File

Binary file not shown.

12
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java
View File

@ -134,7 +134,7 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
init510_20200706_to_20200714();
Builder.BuilderWithTableName empiLink = version.onTable("MPI_LINK");
empiLink.addColumn("20200715.1", "VERSION").nonNullable().type(ColumnTypeEnum.STRING, EmpiLink.VERSION_LENGTH);
empiLink.addColumn("20200715.1", "VERSION").nonNullable().type(ColumnTypeEnum.STRING, 16);
empiLink.addColumn("20200715.2", "EID_MATCH").nullable().type(ColumnTypeEnum.BOOLEAN);
empiLink.addColumn("20200715.3", "NEW_PERSON").nullable().type(ColumnTypeEnum.BOOLEAN);
empiLink.addColumn("20200715.4", "VECTOR").nullable().type(ColumnTypeEnum.LONG);
@ -143,11 +143,11 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
init510_20200725();
//EMPI Target Type
empiLink.addColumn("20200727.1","TARGET_TYPE").nullable().type(ColumnTypeEnum.STRING, EmpiLink.TARGET_TYPE_LENGTH);
empiLink.addColumn("20200727.1","TARGET_TYPE").nullable().type(ColumnTypeEnum.STRING, 40);
//ConceptMap add version for search
Builder.BuilderWithTableName trmConceptMap = version.onTable("TRM_CONCEPT_MAP");
trmConceptMap.addColumn("20200910.1", "VER").nullable().type(ColumnTypeEnum.STRING, TermConceptMap.MAX_VER_LENGTH);
trmConceptMap.addColumn("20200910.1", "VER").nullable().type(ColumnTypeEnum.STRING, 200);
trmConceptMap.dropIndex("20200910.2", "IDX_CONCEPT_MAP_URL");
trmConceptMap.addIndex("20200910.3", "IDX_CONCEPT_MAP_URL").unique(true).withColumns("URL", "VER");
@ -155,15 +155,15 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
Builder.BuilderWithTableName trmCodeSystemVer = version.onTable("TRM_CODESYSTEM_VER");
trmCodeSystemVer.addIndex("20200923.1", "IDX_CODESYSTEM_AND_VER").unique(true).withColumns("CODESYSTEM_PID", "CS_VERSION_ID");
Builder.BuilderWithTableName trmValueSet = version.onTable("TRM_VALUESET");
trmValueSet.addColumn("20200923.2", "VER").nullable().type(ColumnTypeEnum.STRING, TermValueSet.MAX_VER_LENGTH);
trmValueSet.addColumn("20200923.2", "VER").nullable().type(ColumnTypeEnum.STRING, 200);
trmValueSet.dropIndex("20200923.3", "IDX_VALUESET_URL");
trmValueSet.addIndex("20200923.4", "IDX_VALUESET_URL").unique(true).withColumns("URL", "VER");
//Term ValueSet Component add system version
Builder.BuilderWithTableName trmValueSetComp = version.onTable("TRM_VALUESET_CONCEPT");
trmValueSetComp.addColumn("20201028.1", "SYSTEM_VER").nullable().type(ColumnTypeEnum.STRING, TermCodeSystemVersion.MAX_VERSION_LENGTH);
trmValueSetComp.addColumn("20201028.1", "SYSTEM_VER").nullable().type(ColumnTypeEnum.STRING, 200);
trmValueSetComp.dropIndex("20201028.2", "IDX_VS_CONCEPT_CS_CD");
trmValueSetComp.addIndex("20201028.3", "IDX_VS_CONCEPT_CS_CD").unique(true).withColumns("VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL");
trmValueSetComp.addIndex("20201028.3", "IDX_VS_CONCEPT_CS_CODE").unique(true).withColumns("VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL");
}
protected void init510_20200725() {

5
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/NpmPackageVersionEntity.java
View File

@ -54,6 +54,7 @@ import java.util.List;
public class NpmPackageVersionEntity {
public static final int VERSION_ID_LENGTH = 200;
public static final int PACKAGE_DESC_LENGTH = 200;
public static final int FHIR_VERSION_LENGTH = 10;
@SequenceGenerator(name = "SEQ_NPM_PACKVER", sequenceName = "SEQ_NPM_PACKVER")
@ -74,9 +75,9 @@ public class NpmPackageVersionEntity {
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "SAVED_TIME", nullable = false)
private Date mySavedTime;
@Column(name = "PKG_DESC", nullable = true, length = 200)
@Column(name = "PKG_DESC", nullable = true, length = PACKAGE_DESC_LENGTH)
private String myDescription;
@Column(name = "DESC_UPPER", nullable = true, length = 200)
@Column(name = "DESC_UPPER", nullable = true, length = PACKAGE_DESC_LENGTH)
private String myDescriptionUpper;
@Column(name = "CURRENT_VERSION", nullable = false)
private boolean myCurrentVersion;

1
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java
View File

@ -202,6 +202,7 @@ public class JpaConstants {
* URL for extension on a Phonetic String SearchParameter indicating that text values should be phonetically indexed with the named encoder
*/
public static final String EXT_SEARCHPARAM_PHONETIC_ENCODER = "http://hapifhir.io/fhir/StructureDefinition/searchparameter-phonetic-encoder";
public static final String VALUESET_FILTER_DISPLAY = "display";
/**
* Non-instantiable