diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/Constants.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/Constants.java index 99e4fad35ad..bca6f2a0198 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/Constants.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/Constants.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.rest.api; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,12 +21,7 @@ package ca.uhn.fhir.rest.api; */ import java.nio.charset.Charset; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; public class Constants { @@ -197,6 +192,10 @@ public class Constants { public static final String POWERED_BY_HEADER = "X-Powered-By"; public static final Charset CHARSET_US_ASCII; public static final String PARAM_PAGEID = "_pageId"; + /** + * This is provided for testing only! Use with caution as this property may change. + */ + public static final String TEST_SYSTEM_PROP_VALIDATION_RESOURCE_CACHES_MS = "TEST_SYSTEM_PROP_VALIDATION_RESOURCE_CACHES_MS"; static { CHARSET_UTF8 = Charset.forName(CHARSET_NAME_UTF8); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3.java index 8ee4904f3af..c4d6892b829 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3.java @@ -73,8 +73,10 @@ public class FhirResourceDaoDstu3 extends BaseHapiFhirRe @Override public MethodOutcome validate(T theResource, IIdType theId, String theRawResource, EncodingEnum theEncoding, ValidationModeEnum theMode, String theProfile, RequestDetails theRequestDetails) { - ActionRequestDetails requestDetails = new ActionRequestDetails(theRequestDetails, theResource, null, theId); - notifyInterceptors(RestOperationTypeEnum.VALIDATE, requestDetails); + if (theRequestDetails != null) { + ActionRequestDetails requestDetails = new ActionRequestDetails(theRequestDetails, theResource, null, theId); + notifyInterceptors(RestOperationTypeEnum.VALIDATE, requestDetails); + } if (theMode == ValidationModeEnum.DELETE) { if (theId == null || theId.hasIdPart() == false) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java index 2caab674724..ebfb44ed472 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.dao.dstu3; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,19 +20,22 @@ package ca.uhn.fhir.jpa.dao.dstu3; * #L% */ -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.isNotBlank; - -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem; +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem.LookupCodeResult; +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet; +import ca.uhn.fhir.jpa.util.LogicUtil; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.util.ElementUtil; import org.apache.commons.codec.binary.StringUtils; import org.hl7.fhir.dstu3.hapi.ctx.HapiWorkerContext; import org.hl7.fhir.dstu3.hapi.ctx.IValidationSupport; import org.hl7.fhir.dstu3.model.*; import org.hl7.fhir.dstu3.model.Enumerations.PublicationStatus; -import org.hl7.fhir.dstu3.model.ValueSet.*; +import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent; +import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent; +import org.hl7.fhir.dstu3.model.ValueSet.FilterOperator; +import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent; import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ValueSetExpansionOutcome; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IPrimitiveType; @@ -41,20 +44,18 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem.LookupCodeResult; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet; -import ca.uhn.fhir.jpa.util.LogicUtil; -import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.util.ElementUtil; +import java.util.Collections; +import java.util.List; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 implements IFhirResourceDaoValueSet { + private static final Logger ourLog = LoggerFactory.getLogger(FhirResourceDaoValueSetDstu3.class); @Autowired @Qualifier("myJpaValidationSupportChainDstu3") private IValidationSupport myValidationSupport; - @Autowired private IFhirResourceDaoCodeSystem myCodeSystemDao; @@ -69,21 +70,32 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 validateIncludes("include", theSource.getCompose().getInclude()); validateIncludes("exclude", theSource.getCompose().getExclude()); + /* + * If all of the code systems are supported by the HAPI FHIR terminology service, let's + * use that as it's more efficient. + */ + + boolean allSystemsAreSuppportedByTerminologyService = true; + for (ConceptSetComponent next : theSource.getCompose().getInclude()) { + if (!myTerminologySvc.supportsSystem(next.getSystem())) { + allSystemsAreSuppportedByTerminologyService = false; + } + } + for (ConceptSetComponent next : theSource.getCompose().getExclude()) { + if (!myTerminologySvc.supportsSystem(next.getSystem())) { + allSystemsAreSuppportedByTerminologyService = false; + } + } + if (allSystemsAreSuppportedByTerminologyService) { + return (ValueSet) myTerminologySvc.expandValueSet(theSource); + } + HapiWorkerContext workerContext = new HapiWorkerContext(getContext(), myValidationSupport); - ValueSetExpansionOutcome outcome = workerContext.expand(theSource, null); - ValueSet retVal = outcome.getValueset(); retVal.setStatus(PublicationStatus.ACTIVE); - return retVal; - // ValueSetExpansionComponent expansion = outcome.getValueset().getExpansion(); - // - // ValueSet retVal = new ValueSet(); - // retVal.getMeta().setLastUpdated(new Date()); - // retVal.setExpansion(expansion); - // return retVal; } private void validateIncludes(String name, List listToValidate) { @@ -185,8 +197,8 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 @Override public ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet.ValidateCodeResult validateCode(IPrimitiveType theValueSetIdentifier, IIdType theId, IPrimitiveType theCode, - IPrimitiveType theSystem, IPrimitiveType theDisplay, Coding theCoding, - CodeableConcept theCodeableConcept, RequestDetails theRequestDetails) { + IPrimitiveType theSystem, IPrimitiveType theDisplay, Coding theCoding, + CodeableConcept theCodeableConcept, RequestDetails theRequestDetails) { List valueSetIds = Collections.emptyList(); @@ -242,15 +254,12 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 } - - private static final Logger ourLog = LoggerFactory.getLogger(FhirResourceDaoValueSetDstu3.class); - private String toStringOrNull(IPrimitiveType thePrimitive) { return thePrimitive != null ? thePrimitive.getValue() : null; } private ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet.ValidateCodeResult validateCodeIsInContains(List contains, String theSystem, String theCode, - Coding theCoding, CodeableConcept theCodeableConcept) { + Coding theCoding, CodeableConcept theCodeableConcept) { for (ValueSetExpansionContainsComponent nextCode : contains) { ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet.ValidateCodeResult result = validateCodeIsInContains(nextCode.getContains(), theSystem, theCode, theCoding, theCodeableConcept); if (result != null) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java index 96656508902..6965832985b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java @@ -63,8 +63,25 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4 imple private ValueSet doExpand(ValueSet theSource) { - validateIncludes("include", theSource.getCompose().getInclude()); - validateIncludes("exclude", theSource.getCompose().getExclude()); + /* + * If all of the code systems are supported by the HAPI FHIR terminology service, let's + * use that as it's more efficient. + */ + + boolean allSystemsAreSuppportedByTerminologyService = true; + for (ConceptSetComponent next : theSource.getCompose().getInclude()) { + if (!isBlank(next.getSystem()) && !myTerminologySvc.supportsSystem(next.getSystem())) { + allSystemsAreSuppportedByTerminologyService = false; + } + } + for (ConceptSetComponent next : theSource.getCompose().getExclude()) { + if (!isBlank(next.getSystem()) && !myTerminologySvc.supportsSystem(next.getSystem())) { + allSystemsAreSuppportedByTerminologyService = false; + } + } + if (allSystemsAreSuppportedByTerminologyService) { + return myTerminologySvc.expandValueSet(theSource); + } HapiWorkerContext workerContext = new HapiWorkerContext(getContext(), myValidationSupport); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceHistoryTag.java_70782329243090 b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceHistoryTag.java_70782329243090 deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedCompositeStringUnique.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedCompositeStringUnique.java index ff8c6eae3b7..0da8a00bc3e 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedCompositeStringUnique.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedCompositeStringUnique.java @@ -32,7 +32,7 @@ import javax.persistence.*; }) public class ResourceIndexedCompositeStringUnique implements Comparable { - public static final int MAX_STRING_LENGTH = 150; + public static final int MAX_STRING_LENGTH = 200; public static final String IDX_IDXCMPSTRUNIQ_STRING = "IDX_IDXCMPSTRUNIQ_STRING"; public static final String IDX_IDXCMPSTRUNIQ_RESOURCE = "IDX_IDXCMPSTRUNIQ_RESOURCE"; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index f5b39ae2ea8..2f6e3f11bca 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -52,14 +52,9 @@ import org.hibernate.search.jpa.FullTextEntityManager; import org.hibernate.search.jpa.FullTextQuery; import org.hibernate.search.query.dsl.BooleanJunction; import org.hibernate.search.query.dsl.QueryBuilder; -import org.hibernate.search.query.dsl.TermMatchingContext; -import org.hibernate.search.query.dsl.TermTermination; import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.instance.model.api.IIdType; -import org.hl7.fhir.r4.model.CodeSystem; -import org.hl7.fhir.r4.model.Coding; -import org.hl7.fhir.r4.model.ConceptMap; -import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.*; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -84,6 +79,7 @@ import javax.persistence.TypedQuery; import javax.persistence.criteria.*; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -140,14 +136,16 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, private ApplicationContext myApplicationContext; /** - * @param theAdd If true, add the code. If false, remove the code. + * @param theAdd If true, add the code. If false, remove the code. + * @param theCodeCounter */ - private void addCodeIfNotAlreadyAdded(String theCodeSystem, ValueSet.ValueSetExpansionComponent theExpansionComponent, Set theAddedCodes, TermConcept theConcept, boolean theAdd) { + private void addCodeIfNotAlreadyAdded(ValueSet.ValueSetExpansionComponent theExpansionComponent, Set theAddedCodes, TermConcept theConcept, boolean theAdd, AtomicInteger theCodeCounter) { String code = theConcept.getCode(); if (theAdd && theAddedCodes.add(code)) { + String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri(); ValueSet.ValueSetExpansionContainsComponent contains = theExpansionComponent.addContains(); contains.setCode(code); - contains.setSystem(theCodeSystem); + contains.setSystem(codeSystem); contains.setDisplay(theConcept.getDisplay()); for (TermConceptDesignation nextDesignation : theConcept.getDesignations()) { contains @@ -158,10 +156,14 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, .setCode(nextDesignation.getUseCode()) .setDisplay(nextDesignation.getUseDisplay()); } + + theCodeCounter.incrementAndGet(); } if (!theAdd && theAddedCodes.remove(code)) { - removeCodeFromExpansion(theCodeSystem, code, theExpansionComponent); + String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri(); + removeCodeFromExpansion(codeSystem, code, theExpansionComponent); + theCodeCounter.decrementAndGet(); } } @@ -412,22 +414,33 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, @Override @Transactional(propagation = Propagation.REQUIRED) public ValueSet expandValueSet(ValueSet theValueSetToExpand) { + ValueSet.ValueSetExpansionComponent expansionComponent = new ValueSet.ValueSetExpansionComponent(); + expansionComponent.setIdentifier(UUID.randomUUID().toString()); + expansionComponent.setTimestamp(new Date()); + Set addedCodes = new HashSet<>(); + AtomicInteger codeCounter = new AtomicInteger(0); // Handle includes + ourLog.debug("Handling includes"); for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getInclude()) { boolean add = true; - expandValueSetHandleIncludeOrExclude(expansionComponent, addedCodes, include, add); + expandValueSetHandleIncludeOrExclude(expansionComponent, addedCodes, include, add, codeCounter); } // Handle excludes + ourLog.debug("Handling excludes"); for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getExclude()) { boolean add = false; - expandValueSetHandleIncludeOrExclude(expansionComponent, addedCodes, include, add); + expandValueSetHandleIncludeOrExclude(expansionComponent, addedCodes, include, add, codeCounter); } + expansionComponent.setTotal(codeCounter.get()); + ValueSet valueSet = new ValueSet(); + valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE); + valueSet.setCompose(theValueSetToExpand.getCompose()); valueSet.setExpansion(expansionComponent); return valueSet; } @@ -445,10 +458,13 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, return retVal; } - public void expandValueSetHandleIncludeOrExclude(ValueSet.ValueSetExpansionComponent theExpansionComponent, Set theAddedCodes, ValueSet.ConceptSetComponent include, boolean theAdd) { - String system = include.getSystem(); - if (isNotBlank(system)) { - ourLog.info("Starting expansion around code system: {}", system); + public void expandValueSetHandleIncludeOrExclude(ValueSet.ValueSetExpansionComponent theExpansionComponent, Set theAddedCodes, ValueSet.ConceptSetComponent theInclude, boolean theAdd, AtomicInteger theCodeCounter) { + String system = theInclude.getSystem(); + boolean hasSystem = isNotBlank(system); + boolean hasValueSet = theInclude.getValueSet().size() > 0; + + if (hasSystem) { + ourLog.info("Starting {} expansion around code system: {}", (theAdd ? "inclusion" : "exclusion"), system); TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system); if (cs != null) { @@ -463,9 +479,9 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, * Filters */ - if (include.getFilter().size() > 0) { + if (theInclude.getFilter().size() > 0) { - for (ValueSet.ConceptSetFilterComponent nextFilter : include.getFilter()) { + for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) { if (isBlank(nextFilter.getValue()) && nextFilter.getOp() == null && isBlank(nextFilter.getProperty())) { continue; } @@ -542,13 +558,13 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, * Include Concepts */ - List codes = include + List codes = theInclude .getConcept() .stream() .filter(Objects::nonNull) .map(ValueSet.ConceptReferenceComponent::getCode) .filter(StringUtils::isNotBlank) - .map(t->new Term("myCode", t)) + .map(t -> new Term("myCode", t)) .collect(Collectors.toList()); if (codes.size() > 0) { MultiPhraseQuery query = new MultiPhraseQuery(); @@ -564,19 +580,25 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, */ FullTextQuery jpaQuery = em.createFullTextQuery(luceneQuery, TermConcept.class); - jpaQuery.setMaxResults(1000); + int maxResult = 50000; + jpaQuery.setMaxResults(maxResult); StopWatch sw = new StopWatch(); + AtomicInteger count = new AtomicInteger(0); - @SuppressWarnings("unchecked") - List result = jpaQuery.getResultList(); - - ourLog.info("Expansion completed in {}ms", sw.getMillis()); - - for (TermConcept nextConcept : result) { - addCodeIfNotAlreadyAdded(system, theExpansionComponent, theAddedCodes, nextConcept, theAdd); + for (Object next : jpaQuery.getResultList()) { + count.incrementAndGet(); + TermConcept concept = (TermConcept) next; + addCodeIfNotAlreadyAdded(theExpansionComponent, theAddedCodes, concept, theAdd, theCodeCounter); } + + if (maxResult == count.get()) { + throw new InternalErrorException("Expansion fragment produced too many (>= " + maxResult + ") results"); + } + + ourLog.info("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis()); + } else { // No codesystem matching the URL found in the database @@ -585,8 +607,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, throw new InvalidRequestException("Unknown code system: " + system); } - if (include.getConcept().isEmpty() == false) { - for (ValueSet.ConceptReferenceComponent next : include.getConcept()) { + if (theInclude.getConcept().isEmpty() == false) { + for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) { String nextCode = next.getCode(); if (isNotBlank(nextCode) && !theAddedCodes.contains(nextCode)) { CodeSystem.ConceptDefinitionComponent code = findCode(codeSystemFromContext.getConcept(), nextCode); @@ -609,6 +631,25 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, } } + } else if (hasValueSet) { + for (CanonicalType nextValueSet : theInclude.getValueSet()) { + ourLog.info("Starting {} expansion around ValueSet URI: {}", (theAdd ? "inclusion" : "exclusion"), nextValueSet.getValueAsString()); + + List expanded = expandValueSet(nextValueSet.getValueAsString()); + for (VersionIndependentConcept nextConcept : expanded) { + if (theAdd) { + TermCodeSystem codeSystem = myCodeSystemDao.findByCodeSystemUri(nextConcept.getSystem()); + TermConcept concept = myConceptDao.findByCodeSystemAndCode(codeSystem.getCurrentVersion(), nextConcept.getCode()); + addCodeIfNotAlreadyAdded(theExpansionComponent, theAddedCodes, concept, theAdd, theCodeCounter); + } + if (!theAdd && theAddedCodes.remove(nextConcept.getCode())) { + removeCodeFromExpansion(nextConcept.getSystem(), nextConcept.getCode(), theExpansionComponent); + } + } + + } + } else { + throw new InvalidRequestException("ValueSet contains " + (theAdd ? "include" : "exclude") + " criteria with no system defined"); } } @@ -640,7 +681,10 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, if (theCode.equals(next.getCode())) { return next; } - findCode(next.getConcept(), theCode); + CodeSystem.ConceptDefinitionComponent val = findCode(next.getConcept(), theCode); + if (val != null) { + return val; + } } return null; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java index 8d4a6e056e5..5efbe1bbaf7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,6 +21,7 @@ package ca.uhn.fhir.jpa.term; */ import org.hl7.fhir.instance.hapi.validation.IValidationSupport; +import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.ConceptMap; @@ -80,6 +81,11 @@ public class HapiTerminologySvcDstu2 extends BaseHapiTerminologySvcImpl { return null; } + @Override + public IBaseResource expandValueSet(IBaseResource theValueSetToExpand) { + throw new UnsupportedOperationException(); + } + @Override public List expandValueSet(String theValueSet) { throw new UnsupportedOperationException(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java index d19a864aa2f..c5e880db83d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java @@ -166,6 +166,20 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen } } + @Override + public IBaseResource expandValueSet(IBaseResource theInput) { + ValueSet valueSetToExpand = (ValueSet) theInput; + + try { + org.hl7.fhir.r4.model.ValueSet valueSetToExpandR4; + valueSetToExpandR4 = VersionConvertor_30_40.convertValueSet(valueSetToExpand); + org.hl7.fhir.r4.model.ValueSet expandedR4 = super.expandValueSet(valueSetToExpandR4); + return VersionConvertor_30_40.convertValueSet(expandedR4); + } catch (FHIRException e) { + throw new InternalErrorException(e); + } + } + @Override public List expandValueSet(String theValueSet) { ValueSet vs = myValidationSupport.fetchResource(myContext, ValueSet.class, theValueSet); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java index c8602b643da..6130cd179de 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java @@ -39,9 +39,9 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -134,6 +134,13 @@ public class HapiTerminologySvcR4 extends BaseHapiTerminologySvcImpl implements return expandValueSetAndReturnVersionIndependentConcepts(vs); } + @Override + public IBaseResource expandValueSet(IBaseResource theInput) { + ValueSet valueSetToExpand = (ValueSet) theInput; + return super.expandValueSet(valueSetToExpand); + } + + @Override public ValueSetExpansionComponent expandValueSet(FhirContext theContext, ConceptSetComponent theInclude) { ValueSet valueSetToExpand = new ValueSet(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java index be8a1c81824..f4bca9c6000 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.term; import ca.uhn.fhir.jpa.entity.*; import ca.uhn.fhir.rest.api.server.RequestDetails; +import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.ConceptMap; import org.hl7.fhir.r4.model.ValueSet; @@ -35,6 +36,11 @@ public interface IHapiTerminologySvc { ValueSet expandValueSet(ValueSet theValueSetToExpand); + /** + * Version independent + */ + IBaseResource expandValueSet(IBaseResource theValueSetToExpand); + List expandValueSet(String theValueSet); TermConcept findCode(String theCodeSystem, String theCode); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java index 572d0a8c111..16e4368aceb 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java @@ -12,6 +12,7 @@ import ca.uhn.fhir.jpa.util.JpaConstants; import ca.uhn.fhir.jpa.util.LoggingRule; import ca.uhn.fhir.model.dstu2.resource.Bundle; import ca.uhn.fhir.model.dstu2.resource.Bundle.Entry; +import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.IRequestOperationCallback; import ca.uhn.fhir.rest.server.interceptor.IServerInterceptor; @@ -58,6 +59,10 @@ import static org.mockito.Mockito.when; public abstract class BaseJpaTest { + static { + System.setProperty(Constants.TEST_SYSTEM_PROP_VALIDATION_RESOURCE_CACHES_MS, "1000"); + } + protected static final String CM_URL = "http://example.com/my_concept_map"; protected static final String CS_URL = "http://example.com/my_code_system"; protected static final String CS_URL_2 = "http://example.com/my_code_system2"; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java index 87969b57c10..0ac2679fa62 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java @@ -1,39 +1,85 @@ package ca.uhn.fhir.jpa.dao.dstu3; -import static org.hamcrest.Matchers.containsString; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; - +import ca.uhn.fhir.rest.api.EncodingEnum; +import ca.uhn.fhir.rest.api.MethodOutcome; +import ca.uhn.fhir.rest.api.ValidationModeEnum; +import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException; +import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException; +import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; +import ca.uhn.fhir.util.StopWatch; +import ca.uhn.fhir.util.TestUtil; import org.apache.commons.io.IOUtils; import org.hl7.fhir.dstu3.model.*; import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent; import org.hl7.fhir.dstu3.model.Observation.ObservationStatus; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.Ignore; +import org.junit.Test; -import ca.uhn.fhir.util.StopWatch; -import ca.uhn.fhir.rest.api.*; -import ca.uhn.fhir.rest.server.exceptions.*; -import ca.uhn.fhir.util.TestUtil; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static ca.uhn.fhir.jpa.util.TestUtil.sleepAtLeast; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test { private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoDstu3ValidateTest.class); - @AfterClass - public static void afterClassClearContext() { - TestUtil.clearAllStaticFieldsForUnitTest(); + @Test + public void testValidateChangedQuestionnaire() { + Questionnaire q = new Questionnaire(); + q.setId("QUEST"); + q.addItem().setLinkId("A").setType(Questionnaire.QuestionnaireItemType.STRING).setRequired(true); + myQuestionnaireDao.update(q); + + try { + QuestionnaireResponse qr = new QuestionnaireResponse(); + qr.setStatus(QuestionnaireResponse.QuestionnaireResponseStatus.COMPLETED); + qr.getQuestionnaire().setReference("Questionnaire/QUEST"); + qr.addItem().setLinkId("A").addAnswer().setValue(new StringType("AAA")); + + MethodOutcome results = myQuestionnaireResponseDao.validate(qr, null, null, null, null, null, null); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(results.getOperationOutcome())); + } catch (PreconditionFailedException e) { + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome())); + fail(e.toString()); + } + + + q = new Questionnaire(); + q.setId("QUEST"); + q.addItem().setLinkId("B").setType(Questionnaire.QuestionnaireItemType.STRING).setRequired(true); + myQuestionnaireDao.update(q); + + QuestionnaireResponse qr = new QuestionnaireResponse(); + qr.setStatus(QuestionnaireResponse.QuestionnaireResponseStatus.COMPLETED); + qr.getQuestionnaire().setReference("Questionnaire/QUEST"); + qr.addItem().setLinkId("A").addAnswer().setValue(new StringType("AAA")); + + MethodOutcome results = myQuestionnaireResponseDao.validate(qr, null, null, null, null, null, null); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(results.getOperationOutcome())); + + sleepAtLeast(2500); + try { + myQuestionnaireResponseDao.validate(qr, null, null, null, null, null, null); + fail(); + } catch (PreconditionFailedException e) { + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome())); + // good + } + } @Test public void testValidateStructureDefinition() throws Exception { String input = IOUtils.toString(getClass().getResourceAsStream("/sd-david-dhtest7.json"), StandardCharsets.UTF_8); StructureDefinition sd = myFhirCtx.newJsonParser().parseResource(StructureDefinition.class, input); - - + + ourLog.info("Starting validation"); try { myStructureDefinitionDao.validate(sd, null, null, null, ValidationModeEnum.UPDATE, null, mySrd); @@ -41,7 +87,7 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test { ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome())); } ourLog.info("Done validation"); - + StopWatch sw = new StopWatch(); ourLog.info("Starting validation"); try { @@ -52,13 +98,13 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test { ourLog.info("Done validation in {}ms", sw.getMillis()); } - + @Test public void testValidateDocument() throws Exception { String input = IOUtils.toString(getClass().getResourceAsStream("/document-bundle-dstu3.json"), StandardCharsets.UTF_8); Bundle document = myFhirCtx.newJsonParser().parseResource(Bundle.class, input); - - + + ourLog.info("Starting validation"); try { MethodOutcome outcome = myBundleDao.validate(document, null, null, null, ValidationModeEnum.CREATE, null, mySrd); @@ -66,7 +112,7 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test { ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome())); } ourLog.info("Done validation"); - + // ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome.getOperationOutcome())); } @@ -125,24 +171,24 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test { MethodOutcome outcome = null; ValidationModeEnum mode = ValidationModeEnum.CREATE; switch (enc) { - case JSON: - encoded = myFhirCtx.newJsonParser().encodeResourceToString(input); - try { - myObservationDao.validate(input, null, encoded, EncodingEnum.JSON, mode, null, mySrd); - fail(); - } catch (PreconditionFailedException e) { - return (OperationOutcome) e.getOperationOutcome(); - } - break; - case XML: - encoded = myFhirCtx.newXmlParser().encodeResourceToString(input); - try { - myObservationDao.validate(input, null, encoded, EncodingEnum.XML, mode, null, mySrd); - fail(); - } catch (PreconditionFailedException e) { - return (OperationOutcome) e.getOperationOutcome(); - } - break; + case JSON: + encoded = myFhirCtx.newJsonParser().encodeResourceToString(input); + try { + myObservationDao.validate(input, null, encoded, EncodingEnum.JSON, mode, null, mySrd); + fail(); + } catch (PreconditionFailedException e) { + return (OperationOutcome) e.getOperationOutcome(); + } + break; + case XML: + encoded = myFhirCtx.newXmlParser().encodeResourceToString(input); + try { + myObservationDao.validate(input, null, encoded, EncodingEnum.XML, mode, null, mySrd); + fail(); + } catch (PreconditionFailedException e) { + return (OperationOutcome) e.getOperationOutcome(); + } + break; } throw new IllegalStateException(); // shouldn't get here @@ -283,18 +329,18 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test { } return retVal; } - + /** * Format has changed, this is out of date */ @Test @Ignore public void testValidateNewQuestionnaireFormat() throws Exception { - String input =IOUtils.toString(FhirResourceDaoDstu3ValidateTest.class.getResourceAsStream("/questionnaire_dstu3.xml")); + String input = IOUtils.toString(FhirResourceDaoDstu3ValidateTest.class.getResourceAsStream("/questionnaire_dstu3.xml")); try { - MethodOutcome results = myQuestionnaireDao.validate(null, null, input, EncodingEnum.XML, ValidationModeEnum.UPDATE, null, mySrd); - OperationOutcome oo = (OperationOutcome) results.getOperationOutcome(); - ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + MethodOutcome results = myQuestionnaireDao.validate(null, null, input, EncodingEnum.XML, ValidationModeEnum.UPDATE, null, mySrd); + OperationOutcome oo = (OperationOutcome) results.getOperationOutcome(); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); } catch (PreconditionFailedException e) { // this is a failure of the test ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome())); @@ -302,4 +348,9 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test { } } + @AfterClass + public static void afterClassClearContext() { + TestUtil.clearAllStaticFieldsForUnitTest(); + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java index bce25c41bbf..44903bdf921 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java @@ -561,6 +561,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { } @Test + @Ignore public void testExpandWithNoResultsInLocalValueSet1() { createLocalCsAndVs(); @@ -609,30 +610,54 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { public void testExpandWithSystemAndCodesAndFilterKeywordInLocalValueSet() { createLocalCsAndVs(); - ValueSet vs = new ValueSet(); - ConceptSetComponent include = vs.getCompose().addInclude(); - include.setSystem(URL_MY_CODE_SYSTEM); - include.addConcept().setCode("A"); + { + ValueSet vs = new ValueSet(); + ConceptSetComponent include = vs.getCompose().addInclude(); + include.setSystem(URL_MY_CODE_SYSTEM); + include.addConcept().setCode("AAA"); - include.addFilter().setProperty("display").setOp(FilterOperator.EQUAL).setValue("AAA"); + include.addFilter().setProperty("display").setOp(FilterOperator.EQUAL).setValue("AAA"); - ValueSet result = myValueSetDao.expand(vs, null); + ValueSet result = myValueSetDao.expand(vs, null); - // Technically it's not valid to expand a ValueSet with both includes and filters so the - // result fails validation because of the input.. we're being permissive by allowing both - // though, so we won't validate the input - result.setCompose(new ValueSetComposeComponent()); + // Technically it's not valid to expand a ValueSet with both includes and filters so the + // result fails validation because of the input.. we're being permissive by allowing both + // though, so we won't validate the input + result.setCompose(new ValueSetComposeComponent()); - logAndValidateValueSet(result); + logAndValidateValueSet(result); - ArrayList codes = toCodesContains(result.getExpansion().getContains()); - assertThat(codes, containsInAnyOrder("A", "AAA")); + ArrayList codes = toCodesContains(result.getExpansion().getContains()); + assertThat(codes, containsInAnyOrder("AAA")); - int idx = codes.indexOf("AAA"); - assertEquals("AAA", result.getExpansion().getContains().get(idx).getCode()); - assertEquals("Code AAA", result.getExpansion().getContains().get(idx).getDisplay()); - assertEquals(URL_MY_CODE_SYSTEM, result.getExpansion().getContains().get(idx).getSystem()); - // + int idx = codes.indexOf("AAA"); + assertEquals("AAA", result.getExpansion().getContains().get(idx).getCode()); + assertEquals("Code AAA", result.getExpansion().getContains().get(idx).getDisplay()); + assertEquals(URL_MY_CODE_SYSTEM, result.getExpansion().getContains().get(idx).getSystem()); + } + + // Now with a disjunction + { + ValueSet vs = new ValueSet(); + ConceptSetComponent include = vs.getCompose().addInclude(); + include.setSystem(URL_MY_CODE_SYSTEM); + include.addConcept().setCode("A"); + + include.addFilter().setProperty("display").setOp(FilterOperator.EQUAL).setValue("AAA"); + + ValueSet result = myValueSetDao.expand(vs, null); + + // Technically it's not valid to expand a ValueSet with both includes and filters so the + // result fails validation because of the input.. we're being permissive by allowing both + // though, so we won't validate the input + result.setCompose(new ValueSetComposeComponent()); + + logAndValidateValueSet(result); + + ArrayList codes = toCodesContains(result.getExpansion().getContains()); + assertThat(codes, empty()); + + } } @Test diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java index 26ced1d0dfa..0214b47221a 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java @@ -32,6 +32,8 @@ import org.springframework.transaction.annotation.Transactional; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import static ca.uhn.fhir.jpa.dao.dstu3.FhirResourceDaoDstu3TerminologyTest.URL_MY_CODE_SYSTEM; import static ca.uhn.fhir.jpa.dao.dstu3.FhirResourceDaoDstu3TerminologyTest.URL_MY_VALUE_SET; @@ -96,6 +98,39 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3 createLocalVs(codeSystem); } + + public void createLoincSystemWithSomeCodes() { + runInTransaction(() -> { + CodeSystem codeSystem = new CodeSystem(); + codeSystem.setUrl(CS_URL); + codeSystem.setContent(CodeSystemContentMode.NOTPRESENT); + IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified(); + + ResourceTable table = myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalArgumentException::new); + + TermCodeSystemVersion cs = new TermCodeSystemVersion(); + cs.setResource(table); + + TermConcept code; + code = new TermConcept(cs, "50015-7"); + code.addPropertyString("SYSTEM", "Bld/Bone mar^Donor"); + cs.getConcepts().add(code); + + code = new TermConcept(cs, "43343-3"); + code.addPropertyString("SYSTEM", "Ser"); + code.addPropertyString("HELLO", "12345-1"); + cs.getConcepts().add(code); + + code = new TermConcept(cs, "43343-4"); + code.addPropertyString("SYSTEM", "Ser"); + code.addPropertyString("HELLO", "12345-2"); + cs.getConcepts().add(code); + + myTermSvc.storeNewCodeSystemVersion(table.getId(), CS_URL, "SYSTEM NAME", cs); + }); + } + + private void createLocalVs(CodeSystem codeSystem) { myLocalVs = new ValueSet(); myLocalVs.setUrl(URL_MY_VALUE_SET); @@ -132,6 +167,71 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3 myLocalValueSetId = myValueSetDao.create(myLocalVs, mySrd).getId().toUnqualifiedVersionless(); } + + @Test + public void testExpandValueSetPropertySearchWithRegexExcludeUsingOr() { + createLoincSystemWithSomeCodes(); + + List codes; + ValueSet vs; + ValueSet outcome; + ValueSet.ConceptSetComponent exclude; + + // Include + vs = new ValueSet(); + vs.getCompose() + .addInclude() + .setSystem(CS_URL); + + + exclude = vs.getCompose().addExclude(); + exclude.setSystem(CS_URL); + exclude + .addFilter() + .setProperty("HELLO") + .setOp(ValueSet.FilterOperator.REGEX) + .setValue("12345-1|12345-2"); + + IIdType vsId = ourClient.create().resource(vs).execute().getId(); + outcome = (ValueSet) ourClient.operation().onInstance(vsId).named("expand").withNoParameters(Parameters.class).execute().getParameter().get(0).getResource(); + codes = toCodesContains(outcome.getExpansion().getContains()); + ourLog.info("** Got codes: {}", codes); + assertThat(codes, containsInAnyOrder("50015-7")); + + + assertEquals(1, outcome.getCompose().getInclude().size()); + assertEquals(1, outcome.getCompose().getExclude().size()); + assertEquals(1, outcome.getExpansion().getTotal()); + + } + + + @Test + public void testExpandValueSetPropertySearchWithRegexExcludeNoFilter() { + createLoincSystemWithSomeCodes(); + + List codes; + ValueSet vs; + ValueSet outcome; + ValueSet.ConceptSetComponent exclude; + + // Include + vs = new ValueSet(); + vs.getCompose() + .addInclude() + .setSystem(CS_URL); + + + exclude = vs.getCompose().addExclude(); + exclude.setSystem(CS_URL); + + IIdType vsId = ourClient.create().resource(vs).execute().getId(); + outcome = (ValueSet) ourClient.operation().onInstance(vsId).named("expand").withNoParameters(Parameters.class).execute().getParameter().get(0).getResource(); + codes = toCodesContains(outcome.getExpansion().getContains()); + assertThat(codes, empty()); + } + + @Test public void testExpandById() throws IOException { //@formatter:off @@ -611,4 +711,15 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3 return codeSystem; } + + public static List toCodesContains(List theContains) { + List retVal = new ArrayList<>(); + + for (ValueSet.ValueSetExpansionContainsComponent next : theContains) { + retVal.add(next.getCode()); + } + + return retVal; + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java index 5880ea6bf93..14e8ddc9fb3 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java @@ -591,7 +591,7 @@ public class TerminologySvcImplDstu3Test extends BaseJpaDstu3Test { } } - private List toCodesContains(List theContains) { + public static List toCodesContains(List theContains) { List retVal = new ArrayList<>(); for (ValueSet.ValueSetExpansionContainsComponent next : theContains) { diff --git a/hapi-fhir-structures-dstu3/pom.xml b/hapi-fhir-structures-dstu3/pom.xml index a36ee3530f5..a0c1baac547 100644 --- a/hapi-fhir-structures-dstu3/pom.xml +++ b/hapi-fhir-structures-dstu3/pom.xml @@ -168,6 +168,14 @@ --> + + + com.github.ben-manes.caffeine + caffeine + true + + + org.xmlunit diff --git a/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/hapi/ctx/HapiWorkerContext.java b/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/hapi/ctx/HapiWorkerContext.java index cbc74d9ea9e..4c1f394d465 100644 --- a/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/hapi/ctx/HapiWorkerContext.java +++ b/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/hapi/ctx/HapiWorkerContext.java @@ -1,10 +1,14 @@ package org.hl7.fhir.dstu3.hapi.ctx; import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.util.CoverageIgnore; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.time.DateUtils; import org.hl7.fhir.dstu3.context.IWorkerContext; import org.hl7.fhir.dstu3.formats.IParser; import org.hl7.fhir.dstu3.formats.ParserType; @@ -22,13 +26,15 @@ import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; import java.util.*; +import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander, ValueSetExpanderFactory { private final FhirContext myCtx; - private Map myFetchedResourceCache = new HashMap(); + private final Cache myFetchedResourceCache; + private IValidationSupport myValidationSupport; private ExpansionProfile myExpansionProfile; @@ -37,6 +43,12 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander Validate.notNull(theValidationSupport, "theValidationSupport must not be null"); myCtx = theCtx; myValidationSupport = theValidationSupport; + + long timeoutMillis = 10 * DateUtils.MILLIS_PER_SECOND; + if (System.getProperties().containsKey(Constants.TEST_SYSTEM_PROP_VALIDATION_RESOURCE_CACHES_MS)) { + timeoutMillis = Long.parseLong(System.getProperty(Constants.TEST_SYSTEM_PROP_VALIDATION_RESOURCE_CACHES_MS)); + } + myFetchedResourceCache = Caffeine.newBuilder().expireAfterWrite(timeoutMillis, TimeUnit.MILLISECONDS).build(); } @Override @@ -92,13 +104,9 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander return null; } else { @SuppressWarnings("unchecked") - T retVal = (T) myFetchedResourceCache.get(theUri); - if (retVal == null) { - retVal = myValidationSupport.fetchResource(myCtx, theClass, theUri); - if (retVal != null) { - myFetchedResourceCache.put(theUri, retVal); - } - } + T retVal = (T) myFetchedResourceCache.get(theUri, t->{ + return myValidationSupport.fetchResource(myCtx, theClass, theUri); + }); return retVal; } } diff --git a/hapi-fhir-structures-r4/pom.xml b/hapi-fhir-structures-r4/pom.xml index 538acaa1fc8..39a8077b6db 100644 --- a/hapi-fhir-structures-r4/pom.xml +++ b/hapi-fhir-structures-r4/pom.xml @@ -50,6 +50,13 @@ true + + + com.github.ben-manes.caffeine + caffeine + true + + diff --git a/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java index 7f7a3a975ad..8bbdf87b3a8 100644 --- a/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java +++ b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java @@ -1,11 +1,14 @@ package org.hl7.fhir.r4.hapi.ctx; import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.context.RuntimeResourceDefinition; +import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.util.CoverageIgnore; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.time.DateUtils; import org.fhir.ucum.UcumService; import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.exceptions.TerminologyServiceException; @@ -28,13 +31,14 @@ import org.hl7.fhir.utilities.TranslationServices; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; import java.util.*; +import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander, ValueSetExpanderFactory { private final FhirContext myCtx; - private Map myFetchedResourceCache = new HashMap(); + private final Cache myFetchedResourceCache; private IValidationSupport myValidationSupport; private ExpansionProfile myExpansionProfile; @@ -43,6 +47,13 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander Validate.notNull(theValidationSupport, "theValidationSupport must not be null"); myCtx = theCtx; myValidationSupport = theValidationSupport; + + long timeoutMillis = 10 * DateUtils.MILLIS_PER_SECOND; + if (System.getProperties().containsKey(ca.uhn.fhir.rest.api.Constants.TEST_SYSTEM_PROP_VALIDATION_RESOURCE_CACHES_MS)) { + timeoutMillis = Long.parseLong(System.getProperty(Constants.TEST_SYSTEM_PROP_VALIDATION_RESOURCE_CACHES_MS)); + } + + myFetchedResourceCache = Caffeine.newBuilder().expireAfterWrite(timeoutMillis, TimeUnit.MILLISECONDS).build(); } @Override @@ -206,9 +217,9 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander ValueSetExpansionOutcome expandedValueSet = null; - /* - * The following valueset is a special case, since the BCP codesystem is very difficult to expand - */ + /* + * The following valueset is a special case, since the BCP codesystem is very difficult to expand + */ if (theVs != null && "http://hl7.org/fhir/ValueSet/languages".equals(theVs.getId())) { ValueSet expansion = new ValueSet(); for (ConceptSetComponent nextInclude : theVs.getCompose().getInclude()) { @@ -338,13 +349,9 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander return null; } else { @SuppressWarnings("unchecked") - T retVal = (T) myFetchedResourceCache.get(theUri); - if (retVal == null) { - retVal = myValidationSupport.fetchResource(myCtx, theClass, theUri); - if (retVal != null) { - myFetchedResourceCache.put(theUri, (Resource) retVal); - } - } + T retVal = (T) myFetchedResourceCache.get(theUri, t -> { + return myValidationSupport.fetchResource(myCtx, theClass, theUri); + }); return retVal; } } diff --git a/hapi-fhir-validation/src/main/java/org/hl7/fhir/dstu3/hapi/validation/FhirInstanceValidator.java b/hapi-fhir-validation/src/main/java/org/hl7/fhir/dstu3/hapi/validation/FhirInstanceValidator.java index 15ffcf08901..85394a2c15e 100644 --- a/hapi-fhir-validation/src/main/java/org/hl7/fhir/dstu3/hapi/validation/FhirInstanceValidator.java +++ b/hapi-fhir-validation/src/main/java/org/hl7/fhir/dstu3/hapi/validation/FhirInstanceValidator.java @@ -2,6 +2,7 @@ package org.hl7.fhir.dstu3.hapi.validation; import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.EncodingEnum; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.validation.IValidationContext; @@ -15,6 +16,7 @@ import com.google.gson.JsonObject; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.commons.lang3.time.DateUtils; import org.fhir.ucum.UcumService; import org.hl7.fhir.convertors.VersionConvertor_30_40; import org.hl7.fhir.dstu3.hapi.ctx.HapiWorkerContext; @@ -272,14 +274,24 @@ public class FhirInstanceValidator extends BaseValidatorBridge implements IValid private final HapiWorkerContext myWrap; private final VersionConvertor_30_40 myConverter; private volatile List myAllStructures; - private LoadingCache myFetchResourceCache - = Caffeine.newBuilder() - .expireAfterWrite(10, TimeUnit.SECONDS) + private LoadingCache myFetchResourceCache; + + public WorkerContextWrapper(HapiWorkerContext theWorkerContext) { + myWrap = theWorkerContext; + myConverter = new VersionConvertor_30_40(); + + long timeoutMillis = 10 * DateUtils.MILLIS_PER_SECOND; + if (System.getProperties().containsKey(ca.uhn.fhir.rest.api.Constants.TEST_SYSTEM_PROP_VALIDATION_RESOURCE_CACHES_MS)) { + timeoutMillis = Long.parseLong(System.getProperty(Constants.TEST_SYSTEM_PROP_VALIDATION_RESOURCE_CACHES_MS)); + } + + myFetchResourceCache = Caffeine.newBuilder() + .expireAfterWrite(timeoutMillis, TimeUnit.MILLISECONDS) .maximumSize(10000) .build(new CacheLoader() { @Override - public org.hl7.fhir.r4.model.Resource load(FhirInstanceValidator.ResourceKey key) throws Exception { - org.hl7.fhir.dstu3.model.Resource fetched; + public org.hl7.fhir.r4.model.Resource load(ResourceKey key) throws Exception { + Resource fetched; switch (key.getResourceName()) { case "StructureDefinition": fetched = myWrap.fetchResource(StructureDefinition.class, key.getUri()); @@ -308,10 +320,6 @@ public class FhirInstanceValidator extends BaseValidatorBridge implements IValid } } }); - - public WorkerContextWrapper(HapiWorkerContext theWorkerContext) { - myWrap = theWorkerContext; - myConverter = new VersionConvertor_30_40(); } @Override diff --git a/pom.xml b/pom.xml index fe024fdedee..05412d538cb 100644 --- a/pom.xml +++ b/pom.xml @@ -465,12 +465,13 @@ Ana Maria Radu Cerner Corporation - - jbalbien - alinleonard Alin Leonard + Cerner Corporation + + + jbalbien diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 93d4a1cff59..f0f54a83f09 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -297,6 +297,11 @@ The maximum length for codes in the JPA server terminology service have been increased to 500 in order to better accomodate code systems with very long codes. + + A bug in the DSTU3 validator was fixed where validation resources such as StructureDefinitions + and Questionnaires were cached in a cache that never expired, leading to validations against + stale versions of resources. +