From fee01668001d54bbf86b4340d2dd4b930e3ada89 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Tue, 28 Aug 2018 05:26:06 -0400 Subject: [PATCH] Work on removing validstion cache --- .../jpa/dao/dstu3/FhirResourceDaoDstu3.java | 6 +- .../dstu3/FhirResourceDaoValueSetDstu3.java | 75 ++++++------ .../jpa/dao/r4/FhirResourceDaoValueSetR4.java | 21 +++- .../ResourceHistoryTag.java_70782329243090 | 0 .../jpa/term/BaseHapiTerminologySvcImpl.java | 67 +++++++---- .../jpa/term/HapiTerminologySvcDstu2.java | 10 +- .../jpa/term/HapiTerminologySvcDstu3.java | 14 +++ .../fhir/jpa/term/HapiTerminologySvcR4.java | 11 +- .../fhir/jpa/term/IHapiTerminologySvc.java | 6 + .../FhirResourceDaoDstu3ValidateTest.java | 42 +++++++ .../ResourceProviderDstu3ValueSetTest.java | 111 ++++++++++++++++++ .../jpa/term/TerminologySvcImplDstu3Test.java | 2 +- hapi-fhir-structures-dstu3/pom.xml | 8 ++ .../dstu3/hapi/ctx/HapiWorkerContext.java | 17 +-- hapi-fhir-structures-r4/pom.xml | 7 ++ .../fhir/r4/hapi/ctx/HapiWorkerContext.java | 23 ++-- 16 files changed, 334 insertions(+), 86 deletions(-) delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceHistoryTag.java_70782329243090 diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3.java index 8ee4904f3af..c4d6892b829 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3.java @@ -73,8 +73,10 @@ public class FhirResourceDaoDstu3 extends BaseHapiFhirRe @Override public MethodOutcome validate(T theResource, IIdType theId, String theRawResource, EncodingEnum theEncoding, ValidationModeEnum theMode, String theProfile, RequestDetails theRequestDetails) { - ActionRequestDetails requestDetails = new ActionRequestDetails(theRequestDetails, theResource, null, theId); - notifyInterceptors(RestOperationTypeEnum.VALIDATE, requestDetails); + if (theRequestDetails != null) { + ActionRequestDetails requestDetails = new ActionRequestDetails(theRequestDetails, theResource, null, theId); + notifyInterceptors(RestOperationTypeEnum.VALIDATE, requestDetails); + } if (theMode == ValidationModeEnum.DELETE) { if (theId == null || theId.hasIdPart() == false) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java index 2caab674724..ebfb44ed472 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.dao.dstu3; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,19 +20,22 @@ package ca.uhn.fhir.jpa.dao.dstu3; * #L% */ -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.isNotBlank; - -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem; +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem.LookupCodeResult; +import ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet; +import ca.uhn.fhir.jpa.util.LogicUtil; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.util.ElementUtil; import org.apache.commons.codec.binary.StringUtils; import org.hl7.fhir.dstu3.hapi.ctx.HapiWorkerContext; import org.hl7.fhir.dstu3.hapi.ctx.IValidationSupport; import org.hl7.fhir.dstu3.model.*; import org.hl7.fhir.dstu3.model.Enumerations.PublicationStatus; -import org.hl7.fhir.dstu3.model.ValueSet.*; +import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent; +import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent; +import org.hl7.fhir.dstu3.model.ValueSet.FilterOperator; +import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent; import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ValueSetExpansionOutcome; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IPrimitiveType; @@ -41,20 +44,18 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem.LookupCodeResult; -import ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet; -import ca.uhn.fhir.jpa.util.LogicUtil; -import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.util.ElementUtil; +import java.util.Collections; +import java.util.List; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 implements IFhirResourceDaoValueSet { + private static final Logger ourLog = LoggerFactory.getLogger(FhirResourceDaoValueSetDstu3.class); @Autowired @Qualifier("myJpaValidationSupportChainDstu3") private IValidationSupport myValidationSupport; - @Autowired private IFhirResourceDaoCodeSystem myCodeSystemDao; @@ -69,21 +70,32 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 validateIncludes("include", theSource.getCompose().getInclude()); validateIncludes("exclude", theSource.getCompose().getExclude()); + /* + * If all of the code systems are supported by the HAPI FHIR terminology service, let's + * use that as it's more efficient. + */ + + boolean allSystemsAreSuppportedByTerminologyService = true; + for (ConceptSetComponent next : theSource.getCompose().getInclude()) { + if (!myTerminologySvc.supportsSystem(next.getSystem())) { + allSystemsAreSuppportedByTerminologyService = false; + } + } + for (ConceptSetComponent next : theSource.getCompose().getExclude()) { + if (!myTerminologySvc.supportsSystem(next.getSystem())) { + allSystemsAreSuppportedByTerminologyService = false; + } + } + if (allSystemsAreSuppportedByTerminologyService) { + return (ValueSet) myTerminologySvc.expandValueSet(theSource); + } + HapiWorkerContext workerContext = new HapiWorkerContext(getContext(), myValidationSupport); - ValueSetExpansionOutcome outcome = workerContext.expand(theSource, null); - ValueSet retVal = outcome.getValueset(); retVal.setStatus(PublicationStatus.ACTIVE); - return retVal; - // ValueSetExpansionComponent expansion = outcome.getValueset().getExpansion(); - // - // ValueSet retVal = new ValueSet(); - // retVal.getMeta().setLastUpdated(new Date()); - // retVal.setExpansion(expansion); - // return retVal; } private void validateIncludes(String name, List listToValidate) { @@ -185,8 +197,8 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 @Override public ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet.ValidateCodeResult validateCode(IPrimitiveType theValueSetIdentifier, IIdType theId, IPrimitiveType theCode, - IPrimitiveType theSystem, IPrimitiveType theDisplay, Coding theCoding, - CodeableConcept theCodeableConcept, RequestDetails theRequestDetails) { + IPrimitiveType theSystem, IPrimitiveType theDisplay, Coding theCoding, + CodeableConcept theCodeableConcept, RequestDetails theRequestDetails) { List valueSetIds = Collections.emptyList(); @@ -242,15 +254,12 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 } - - private static final Logger ourLog = LoggerFactory.getLogger(FhirResourceDaoValueSetDstu3.class); - private String toStringOrNull(IPrimitiveType thePrimitive) { return thePrimitive != null ? thePrimitive.getValue() : null; } private ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet.ValidateCodeResult validateCodeIsInContains(List contains, String theSystem, String theCode, - Coding theCoding, CodeableConcept theCodeableConcept) { + Coding theCoding, CodeableConcept theCodeableConcept) { for (ValueSetExpansionContainsComponent nextCode : contains) { ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet.ValidateCodeResult result = validateCodeIsInContains(nextCode.getContains(), theSystem, theCode, theCoding, theCodeableConcept); if (result != null) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java index 96656508902..a0023fd786a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java @@ -63,8 +63,25 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4 imple private ValueSet doExpand(ValueSet theSource) { - validateIncludes("include", theSource.getCompose().getInclude()); - validateIncludes("exclude", theSource.getCompose().getExclude()); + /* + * If all of the code systems are supported by the HAPI FHIR terminology service, let's + * use that as it's more efficient. + */ + + boolean allSystemsAreSuppportedByTerminologyService = true; + for (ConceptSetComponent next : theSource.getCompose().getInclude()) { + if (!myTerminologySvc.supportsSystem(next.getSystem())) { + allSystemsAreSuppportedByTerminologyService = false; + } + } + for (ConceptSetComponent next : theSource.getCompose().getExclude()) { + if (!myTerminologySvc.supportsSystem(next.getSystem())) { + allSystemsAreSuppportedByTerminologyService = false; + } + } + if (allSystemsAreSuppportedByTerminologyService) { + return myTerminologySvc.expandValueSet(theSource); + } HapiWorkerContext workerContext = new HapiWorkerContext(getContext(), myValidationSupport); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceHistoryTag.java_70782329243090 b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceHistoryTag.java_70782329243090 deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index f5b39ae2ea8..966dbc53957 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -52,8 +52,6 @@ import org.hibernate.search.jpa.FullTextEntityManager; import org.hibernate.search.jpa.FullTextQuery; import org.hibernate.search.query.dsl.BooleanJunction; import org.hibernate.search.query.dsl.QueryBuilder; -import org.hibernate.search.query.dsl.TermMatchingContext; -import org.hibernate.search.query.dsl.TermTermination; import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.CodeSystem; @@ -84,6 +82,7 @@ import javax.persistence.TypedQuery; import javax.persistence.criteria.*; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -141,8 +140,9 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, /** * @param theAdd If true, add the code. If false, remove the code. + * @param theCodeCounter */ - private void addCodeIfNotAlreadyAdded(String theCodeSystem, ValueSet.ValueSetExpansionComponent theExpansionComponent, Set theAddedCodes, TermConcept theConcept, boolean theAdd) { + private void addCodeIfNotAlreadyAdded(String theCodeSystem, ValueSet.ValueSetExpansionComponent theExpansionComponent, Set theAddedCodes, TermConcept theConcept, boolean theAdd, AtomicInteger theCodeCounter) { String code = theConcept.getCode(); if (theAdd && theAddedCodes.add(code)) { ValueSet.ValueSetExpansionContainsComponent contains = theExpansionComponent.addContains(); @@ -158,10 +158,13 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, .setCode(nextDesignation.getUseCode()) .setDisplay(nextDesignation.getUseDisplay()); } + + theCodeCounter.incrementAndGet(); } if (!theAdd && theAddedCodes.remove(code)) { removeCodeFromExpansion(theCodeSystem, code, theExpansionComponent); + theCodeCounter.decrementAndGet(); } } @@ -412,22 +415,32 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, @Override @Transactional(propagation = Propagation.REQUIRED) public ValueSet expandValueSet(ValueSet theValueSetToExpand) { + ValueSet.ValueSetExpansionComponent expansionComponent = new ValueSet.ValueSetExpansionComponent(); + expansionComponent.setIdentifier(UUID.randomUUID().toString()); + expansionComponent.setTimestamp(new Date()); + Set addedCodes = new HashSet<>(); + AtomicInteger codeCounter = new AtomicInteger(0); // Handle includes + ourLog.debug("Handling includes"); for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getInclude()) { boolean add = true; - expandValueSetHandleIncludeOrExclude(expansionComponent, addedCodes, include, add); + expandValueSetHandleIncludeOrExclude(expansionComponent, addedCodes, include, add, codeCounter); } // Handle excludes + ourLog.debug("Handling excludes"); for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getExclude()) { boolean add = false; - expandValueSetHandleIncludeOrExclude(expansionComponent, addedCodes, include, add); + expandValueSetHandleIncludeOrExclude(expansionComponent, addedCodes, include, add, codeCounter); } + expansionComponent.setTotal(codeCounter.get()); + ValueSet valueSet = new ValueSet(); + valueSet.setCompose(theValueSetToExpand.getCompose()); valueSet.setExpansion(expansionComponent); return valueSet; } @@ -445,10 +458,10 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, return retVal; } - public void expandValueSetHandleIncludeOrExclude(ValueSet.ValueSetExpansionComponent theExpansionComponent, Set theAddedCodes, ValueSet.ConceptSetComponent include, boolean theAdd) { - String system = include.getSystem(); + public void expandValueSetHandleIncludeOrExclude(ValueSet.ValueSetExpansionComponent theExpansionComponent, Set theAddedCodes, ValueSet.ConceptSetComponent theInclude, boolean theAdd, AtomicInteger theCodeCounter) { + String system = theInclude.getSystem(); + ourLog.info("Starting {} expansion around code system: {}", (theAdd ? "inclusion" : "exclusion"), system); if (isNotBlank(system)) { - ourLog.info("Starting expansion around code system: {}", system); TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system); if (cs != null) { @@ -463,9 +476,9 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, * Filters */ - if (include.getFilter().size() > 0) { + if (theInclude.getFilter().size() > 0) { - for (ValueSet.ConceptSetFilterComponent nextFilter : include.getFilter()) { + for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) { if (isBlank(nextFilter.getValue()) && nextFilter.getOp() == null && isBlank(nextFilter.getProperty())) { continue; } @@ -542,13 +555,13 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, * Include Concepts */ - List codes = include + List codes = theInclude .getConcept() .stream() .filter(Objects::nonNull) .map(ValueSet.ConceptReferenceComponent::getCode) .filter(StringUtils::isNotBlank) - .map(t->new Term("myCode", t)) + .map(t -> new Term("myCode", t)) .collect(Collectors.toList()); if (codes.size() > 0) { MultiPhraseQuery query = new MultiPhraseQuery(); @@ -564,19 +577,25 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, */ FullTextQuery jpaQuery = em.createFullTextQuery(luceneQuery, TermConcept.class); - jpaQuery.setMaxResults(1000); + int maxResult = 50000; + jpaQuery.setMaxResults(maxResult); StopWatch sw = new StopWatch(); + AtomicInteger count = new AtomicInteger(0); - @SuppressWarnings("unchecked") - List result = jpaQuery.getResultList(); - - ourLog.info("Expansion completed in {}ms", sw.getMillis()); - - for (TermConcept nextConcept : result) { - addCodeIfNotAlreadyAdded(system, theExpansionComponent, theAddedCodes, nextConcept, theAdd); + for (Object next : jpaQuery.getResultList()) { + count.incrementAndGet(); + TermConcept concept = (TermConcept) next; + addCodeIfNotAlreadyAdded(system, theExpansionComponent, theAddedCodes, concept, theAdd, theCodeCounter); } + + if (maxResult == count.get()) { + throw new InternalErrorException("Expansion fragment produced too many (>= " + maxResult + ") results"); + } + + ourLog.info("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis()); + } else { // No codesystem matching the URL found in the database @@ -585,8 +604,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, throw new InvalidRequestException("Unknown code system: " + system); } - if (include.getConcept().isEmpty() == false) { - for (ValueSet.ConceptReferenceComponent next : include.getConcept()) { + if (theInclude.getConcept().isEmpty() == false) { + for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) { String nextCode = next.getCode(); if (isNotBlank(nextCode) && !theAddedCodes.contains(nextCode)) { CodeSystem.ConceptDefinitionComponent code = findCode(codeSystemFromContext.getConcept(), nextCode); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java index 8d4a6e056e5..5efbe1bbaf7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,6 +21,7 @@ package ca.uhn.fhir.jpa.term; */ import org.hl7.fhir.instance.hapi.validation.IValidationSupport; +import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.ConceptMap; @@ -80,6 +81,11 @@ public class HapiTerminologySvcDstu2 extends BaseHapiTerminologySvcImpl { return null; } + @Override + public IBaseResource expandValueSet(IBaseResource theValueSetToExpand) { + throw new UnsupportedOperationException(); + } + @Override public List expandValueSet(String theValueSet) { throw new UnsupportedOperationException(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java index d19a864aa2f..c5e880db83d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java @@ -166,6 +166,20 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen } } + @Override + public IBaseResource expandValueSet(IBaseResource theInput) { + ValueSet valueSetToExpand = (ValueSet) theInput; + + try { + org.hl7.fhir.r4.model.ValueSet valueSetToExpandR4; + valueSetToExpandR4 = VersionConvertor_30_40.convertValueSet(valueSetToExpand); + org.hl7.fhir.r4.model.ValueSet expandedR4 = super.expandValueSet(valueSetToExpandR4); + return VersionConvertor_30_40.convertValueSet(expandedR4); + } catch (FHIRException e) { + throw new InternalErrorException(e); + } + } + @Override public List expandValueSet(String theValueSet) { ValueSet vs = myValidationSupport.fetchResource(myContext, ValueSet.class, theValueSet); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java index c8602b643da..6130cd179de 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java @@ -39,9 +39,9 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -134,6 +134,13 @@ public class HapiTerminologySvcR4 extends BaseHapiTerminologySvcImpl implements return expandValueSetAndReturnVersionIndependentConcepts(vs); } + @Override + public IBaseResource expandValueSet(IBaseResource theInput) { + ValueSet valueSetToExpand = (ValueSet) theInput; + return super.expandValueSet(valueSetToExpand); + } + + @Override public ValueSetExpansionComponent expandValueSet(FhirContext theContext, ConceptSetComponent theInclude) { ValueSet valueSetToExpand = new ValueSet(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java index be8a1c81824..f4bca9c6000 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.term; import ca.uhn.fhir.jpa.entity.*; import ca.uhn.fhir.rest.api.server.RequestDetails; +import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.ConceptMap; import org.hl7.fhir.r4.model.ValueSet; @@ -35,6 +36,11 @@ public interface IHapiTerminologySvc { ValueSet expandValueSet(ValueSet theValueSetToExpand); + /** + * Version independent + */ + IBaseResource expandValueSet(IBaseResource theValueSetToExpand); + List expandValueSet(String theValueSet); TermConcept findCode(String theCodeSystem, String theCode); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java index 87969b57c10..6fc96e7fa73 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java @@ -28,6 +28,48 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test { TestUtil.clearAllStaticFieldsForUnitTest(); } + @Test + public void testValidateChangedQuestionnaire() { + Questionnaire q = new Questionnaire(); + q.setId("QUEST"); + q.addItem().setLinkId("A").setType(Questionnaire.QuestionnaireItemType.STRING).setRequired(true); + myQuestionnaireDao.update(q); + + try { + QuestionnaireResponse qr = new QuestionnaireResponse(); + qr.setStatus(QuestionnaireResponse.QuestionnaireResponseStatus.COMPLETED); + qr.getQuestionnaire().setReference("Questionnaire/QUEST"); + qr.addItem().setLinkId("A").addAnswer().setValue(new StringType("AAA")); + + MethodOutcome results = myQuestionnaireResponseDao.validate(qr, null, null, null, null, null, null); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(results.getOperationOutcome())); + } catch (PreconditionFailedException e) { + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome())); + fail(e.toString()); + } + + + q = new Questionnaire(); + q.setId("QUEST"); + q.addItem().setLinkId("B").setType(Questionnaire.QuestionnaireItemType.STRING).setRequired(true); + myQuestionnaireDao.update(q); + + try { + QuestionnaireResponse qr = new QuestionnaireResponse(); + qr.setStatus(QuestionnaireResponse.QuestionnaireResponseStatus.COMPLETED); + qr.getQuestionnaire().setReference("Questionnaire/QUEST"); + qr.addItem().setLinkId("A").addAnswer().setValue(new StringType("AAA")); + + MethodOutcome results = myQuestionnaireResponseDao.validate(qr, null, null, null, null, null, null); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(results.getOperationOutcome())); + fail(); + } catch (PreconditionFailedException e) { + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome())); + // good + } + + } + @Test public void testValidateStructureDefinition() throws Exception { String input = IOUtils.toString(getClass().getResourceAsStream("/sd-david-dhtest7.json"), StandardCharsets.UTF_8); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java index 26ced1d0dfa..0214b47221a 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java @@ -32,6 +32,8 @@ import org.springframework.transaction.annotation.Transactional; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import static ca.uhn.fhir.jpa.dao.dstu3.FhirResourceDaoDstu3TerminologyTest.URL_MY_CODE_SYSTEM; import static ca.uhn.fhir.jpa.dao.dstu3.FhirResourceDaoDstu3TerminologyTest.URL_MY_VALUE_SET; @@ -96,6 +98,39 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3 createLocalVs(codeSystem); } + + public void createLoincSystemWithSomeCodes() { + runInTransaction(() -> { + CodeSystem codeSystem = new CodeSystem(); + codeSystem.setUrl(CS_URL); + codeSystem.setContent(CodeSystemContentMode.NOTPRESENT); + IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified(); + + ResourceTable table = myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalArgumentException::new); + + TermCodeSystemVersion cs = new TermCodeSystemVersion(); + cs.setResource(table); + + TermConcept code; + code = new TermConcept(cs, "50015-7"); + code.addPropertyString("SYSTEM", "Bld/Bone mar^Donor"); + cs.getConcepts().add(code); + + code = new TermConcept(cs, "43343-3"); + code.addPropertyString("SYSTEM", "Ser"); + code.addPropertyString("HELLO", "12345-1"); + cs.getConcepts().add(code); + + code = new TermConcept(cs, "43343-4"); + code.addPropertyString("SYSTEM", "Ser"); + code.addPropertyString("HELLO", "12345-2"); + cs.getConcepts().add(code); + + myTermSvc.storeNewCodeSystemVersion(table.getId(), CS_URL, "SYSTEM NAME", cs); + }); + } + + private void createLocalVs(CodeSystem codeSystem) { myLocalVs = new ValueSet(); myLocalVs.setUrl(URL_MY_VALUE_SET); @@ -132,6 +167,71 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3 myLocalValueSetId = myValueSetDao.create(myLocalVs, mySrd).getId().toUnqualifiedVersionless(); } + + @Test + public void testExpandValueSetPropertySearchWithRegexExcludeUsingOr() { + createLoincSystemWithSomeCodes(); + + List codes; + ValueSet vs; + ValueSet outcome; + ValueSet.ConceptSetComponent exclude; + + // Include + vs = new ValueSet(); + vs.getCompose() + .addInclude() + .setSystem(CS_URL); + + + exclude = vs.getCompose().addExclude(); + exclude.setSystem(CS_URL); + exclude + .addFilter() + .setProperty("HELLO") + .setOp(ValueSet.FilterOperator.REGEX) + .setValue("12345-1|12345-2"); + + IIdType vsId = ourClient.create().resource(vs).execute().getId(); + outcome = (ValueSet) ourClient.operation().onInstance(vsId).named("expand").withNoParameters(Parameters.class).execute().getParameter().get(0).getResource(); + codes = toCodesContains(outcome.getExpansion().getContains()); + ourLog.info("** Got codes: {}", codes); + assertThat(codes, containsInAnyOrder("50015-7")); + + + assertEquals(1, outcome.getCompose().getInclude().size()); + assertEquals(1, outcome.getCompose().getExclude().size()); + assertEquals(1, outcome.getExpansion().getTotal()); + + } + + + @Test + public void testExpandValueSetPropertySearchWithRegexExcludeNoFilter() { + createLoincSystemWithSomeCodes(); + + List codes; + ValueSet vs; + ValueSet outcome; + ValueSet.ConceptSetComponent exclude; + + // Include + vs = new ValueSet(); + vs.getCompose() + .addInclude() + .setSystem(CS_URL); + + + exclude = vs.getCompose().addExclude(); + exclude.setSystem(CS_URL); + + IIdType vsId = ourClient.create().resource(vs).execute().getId(); + outcome = (ValueSet) ourClient.operation().onInstance(vsId).named("expand").withNoParameters(Parameters.class).execute().getParameter().get(0).getResource(); + codes = toCodesContains(outcome.getExpansion().getContains()); + assertThat(codes, empty()); + } + + @Test public void testExpandById() throws IOException { //@formatter:off @@ -611,4 +711,15 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3 return codeSystem; } + + public static List toCodesContains(List theContains) { + List retVal = new ArrayList<>(); + + for (ValueSet.ValueSetExpansionContainsComponent next : theContains) { + retVal.add(next.getCode()); + } + + return retVal; + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java index 5880ea6bf93..14e8ddc9fb3 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java @@ -591,7 +591,7 @@ public class TerminologySvcImplDstu3Test extends BaseJpaDstu3Test { } } - private List toCodesContains(List theContains) { + public static List toCodesContains(List theContains) { List retVal = new ArrayList<>(); for (ValueSet.ValueSetExpansionContainsComponent next : theContains) { diff --git a/hapi-fhir-structures-dstu3/pom.xml b/hapi-fhir-structures-dstu3/pom.xml index a36ee3530f5..a0c1baac547 100644 --- a/hapi-fhir-structures-dstu3/pom.xml +++ b/hapi-fhir-structures-dstu3/pom.xml @@ -168,6 +168,14 @@ --> + + + com.github.ben-manes.caffeine + caffeine + true + + + org.xmlunit diff --git a/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/hapi/ctx/HapiWorkerContext.java b/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/hapi/ctx/HapiWorkerContext.java index cbc74d9ea9e..64a5880645b 100644 --- a/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/hapi/ctx/HapiWorkerContext.java +++ b/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/hapi/ctx/HapiWorkerContext.java @@ -4,6 +4,8 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.util.CoverageIgnore; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.commons.lang3.Validate; import org.hl7.fhir.dstu3.context.IWorkerContext; import org.hl7.fhir.dstu3.formats.IParser; @@ -22,13 +24,15 @@ import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; import java.util.*; +import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander, ValueSetExpanderFactory { private final FhirContext myCtx; - private Map myFetchedResourceCache = new HashMap(); + private final Cache myFetchedResourceCache; + private IValidationSupport myValidationSupport; private ExpansionProfile myExpansionProfile; @@ -37,6 +41,7 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander Validate.notNull(theValidationSupport, "theValidationSupport must not be null"); myCtx = theCtx; myValidationSupport = theValidationSupport; + myFetchedResourceCache = Caffeine.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build(); } @Override @@ -92,13 +97,9 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander return null; } else { @SuppressWarnings("unchecked") - T retVal = (T) myFetchedResourceCache.get(theUri); - if (retVal == null) { - retVal = myValidationSupport.fetchResource(myCtx, theClass, theUri); - if (retVal != null) { - myFetchedResourceCache.put(theUri, retVal); - } - } + T retVal = (T) myFetchedResourceCache.get(theUri, t->{ + return myValidationSupport.fetchResource(myCtx, theClass, theUri); + }); return retVal; } } diff --git a/hapi-fhir-structures-r4/pom.xml b/hapi-fhir-structures-r4/pom.xml index 538acaa1fc8..39a8077b6db 100644 --- a/hapi-fhir-structures-r4/pom.xml +++ b/hapi-fhir-structures-r4/pom.xml @@ -50,6 +50,13 @@ true + + + com.github.ben-manes.caffeine + caffeine + true + + diff --git a/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java index 7f7a3a975ad..8a3b11f3983 100644 --- a/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java +++ b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java @@ -1,10 +1,11 @@ package org.hl7.fhir.r4.hapi.ctx; import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.context.RuntimeResourceDefinition; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.util.CoverageIgnore; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.commons.lang3.Validate; import org.fhir.ucum.UcumService; import org.hl7.fhir.exceptions.FHIRException; @@ -28,13 +29,14 @@ import org.hl7.fhir.utilities.TranslationServices; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; import java.util.*; +import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander, ValueSetExpanderFactory { private final FhirContext myCtx; - private Map myFetchedResourceCache = new HashMap(); + private final Cache myFetchedResourceCache; private IValidationSupport myValidationSupport; private ExpansionProfile myExpansionProfile; @@ -43,6 +45,7 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander Validate.notNull(theValidationSupport, "theValidationSupport must not be null"); myCtx = theCtx; myValidationSupport = theValidationSupport; + myFetchedResourceCache = Caffeine.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build(); } @Override @@ -206,9 +209,9 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander ValueSetExpansionOutcome expandedValueSet = null; - /* - * The following valueset is a special case, since the BCP codesystem is very difficult to expand - */ + /* + * The following valueset is a special case, since the BCP codesystem is very difficult to expand + */ if (theVs != null && "http://hl7.org/fhir/ValueSet/languages".equals(theVs.getId())) { ValueSet expansion = new ValueSet(); for (ConceptSetComponent nextInclude : theVs.getCompose().getInclude()) { @@ -338,13 +341,9 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander return null; } else { @SuppressWarnings("unchecked") - T retVal = (T) myFetchedResourceCache.get(theUri); - if (retVal == null) { - retVal = myValidationSupport.fetchResource(myCtx, theClass, theUri); - if (retVal != null) { - myFetchedResourceCache.put(theUri, (Resource) retVal); - } - } + T retVal = (T) myFetchedResourceCache.get(theUri, t -> { + return myValidationSupport.fetchResource(myCtx, theClass, theUri); + }); return retVal; } }