honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow unknown code systems during validation (#2523) 

* Allow unknown code systems during validation

* Add changelog

* Test fix
This commit is contained in:
James Agnew 2021-04-05 20:59:59 -04:00 committed by GitHub
parent b617c7690d
commit fcffb04c7b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 189 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_4_0/2523-allow-validation-with-unknown-code-system.yaml Normal file
View File

@ -0,0 +1,6 @@
---
type: add
issue: 2523
title: "A new Validation Support Module has been added called UnknownCodeSystemWarningValidationSupport. This module
allows validation to produce a warning but not an error if a code being validated references
an unknown code system."

9
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/validation/validation_support_modules.md
View File

@ -134,6 +134,15 @@ This module will invoke the following operations on the remote terminology serve
* **POST [base]/CodeSystem/$validate-code** – Validate codes in fields where no specific ValueSet is bound
* **POST [base]/ValueSet/$validate-code** – Validate codes in fields where a specific ValueSet is bound
# UnknownCodeSystemWarningValidationSupport
[JavaDoc](/hapi-fhir/apidocs/hapi-fhir-validation/org/hl7/fhir/common/hapi/validation/support/UnknownCodeSystemWarningValidationSupport.html) / [Source](https://github.com/jamesagnew/hapi-fhir/blob/ja_20200218_validation_api_changes/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/UnknownCodeSystemWarningValidationSupport.java)
This validation support module may be placed at the end of a ValidationSupportChain in order to configure the validator to generate a warning if a resource being validated contains an unknown code system.
Note that this module must also be activated by calling [setAllowNonExistentCodeSystem(true)](/hapi-fhir/apidocs/hapi-fhir-validation/org/hl7/fhir/common/hapi/validation/support/UnknownCodeSystemWarningValidationSupport.html#setAllowNonExistentCodeSystem(boolean)) in order to specify that unknown code systems should be allowed.
# Recipes
The IValidationSupport instance passed to the FhirInstanceValidator will often resemble the chain shown in the diagram below. In this diagram:

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java
View File

@ -130,6 +130,7 @@ import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.server.interceptor.consent.IConsentContextServices;
import ca.uhn.fhir.rest.server.interceptor.partition.RequestTenantPartitionInterceptor;
import org.hibernate.jpa.HibernatePersistenceProvider;
import org.hl7.fhir.common.hapi.validation.support.UnknownCodeSystemWarningValidationSupport;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.utilities.graphql.IGraphQLStorageServices;
import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
@ -227,7 +228,6 @@ public abstract class BaseConfig {
this.searchCoordQueueCapacity = searchCoordQueueCapacity;
}
@Bean
public BatchConfigurer batchConfigurer() {
return new NonPersistedBatchConfigurer();
@ -834,6 +834,11 @@ public abstract class BaseConfig {
return new JpaResourceLoader();
}
@Bean
public UnknownCodeSystemWarningValidationSupport unknownCodeSystemWarningValidationSupport() {
return new UnknownCodeSystemWarningValidationSupport(fhirContext());
}
public static void configureEntityManagerFactory(LocalContainerEntityManagerFactoryBean theFactory, FhirContext theCtx) {
theFactory.setJpaDialect(hibernateJpaDialect(theCtx.getLocalizer()));
theFactory.setPackagesToScan("ca.uhn.fhir.jpa.model.entity", "ca.uhn.fhir.jpa.entity");

5
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java
View File

@ -2184,11 +2184,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
retVal = createFailureCodeValidationResult(theCodeSystem, theCode, append);
}
if (retVal == null) {
String append = " - Unable to expand ValueSet[" + theValueSetUrl + "]";
retVal = createFailureCodeValidationResult(theCodeSystem, theCode, append);
}
return retVal;
}

6
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/JpaValidationSupportChain.java
View File

@ -26,8 +26,9 @@ import ca.uhn.fhir.jpa.packages.NpmJpaValidationSupport;
import ca.uhn.fhir.jpa.term.api.ITermConceptMappingSvc;
import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService;
import org.hl7.fhir.common.hapi.validation.support.SnapshotGeneratingValidationSupport;
import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport;
import org.hl7.fhir.common.hapi.validation.support.SnapshotGeneratingValidationSupport;
import org.hl7.fhir.common.hapi.validation.support.UnknownCodeSystemWarningValidationSupport;
import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
@ -52,6 +53,8 @@ public class JpaValidationSupportChain extends ValidationSupportChain {
private NpmJpaValidationSupport myNpmJpaValidationSupport;
@Autowired
private ITermConceptMappingSvc myConceptMappingSvc;
@Autowired
private UnknownCodeSystemWarningValidationSupport myUnknownCodeSystemWarningValidationSupport;
public JpaValidationSupportChain(FhirContext theFhirContext) {
myFhirContext = theFhirContext;
@ -77,6 +80,7 @@ public class JpaValidationSupportChain extends ValidationSupportChain {
addValidationSupport(myNpmJpaValidationSupport);
addValidationSupport(new CommonCodeSystemsTerminologyService(myFhirContext));
addValidationSupport(myConceptMappingSvc);
addValidationSupport(myUnknownCodeSystemWarningValidationSupport);
}
}

159
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java
View File

@ -9,7 +9,6 @@ import ca.uhn.fhir.jpa.entity.TermValueSet;
import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl;
import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
@ -28,6 +27,7 @@ import ca.uhn.fhir.util.OperationOutcomeUtil;
import ca.uhn.fhir.util.StopWatch;
import ca.uhn.fhir.validation.IValidatorModule;
import org.apache.commons.io.IOUtils;
import org.hl7.fhir.common.hapi.validation.support.UnknownCodeSystemWarningValidationSupport;
import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator;
import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
import org.hl7.fhir.instance.model.api.IBaseResource;
@ -69,12 +69,10 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.util.AopTestUtils;
import org.springframework.transaction.PlatformTransactionManager;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Locale;
import java.util.stream.Collectors;
import static org.awaitility.Awaitility.await;
@ -96,18 +94,111 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
@Autowired
private ITermReadSvc myTermReadSvc;
@Autowired
private ITermCodeSystemStorageSvc myTermCodeSystemStorageSvcc;
@Autowired
private DaoRegistry myDaoRegistry;
@Autowired
private JpaValidationSupportChain myJpaValidationSupportChain;
@Autowired
private PlatformTransactionManager myTransactionManager;
@Autowired
private ValidationSettings myValidationSettings;
@Autowired
private UnknownCodeSystemWarningValidationSupport myUnknownCodeSystemWarningValidationSupport;
@AfterEach
public void after() {
FhirInstanceValidator val = AopTestUtils.getTargetObject(myValidatorModule);
val.setBestPracticeWarningLevel(IResourceValidator.BestPracticeWarningLevel.Warning);
myDaoConfig.setAllowExternalReferences(new DaoConfig().isAllowExternalReferences());
myDaoConfig.setMaximumExpansionSize(DaoConfig.DEFAULT_MAX_EXPANSION_SIZE);
myDaoConfig.setPreExpandValueSets(new DaoConfig().isPreExpandValueSets());
BaseTermReadSvcImpl.setInvokeOnNextCallForUnitTest(null);
myValidationSettings.setLocalReferenceValidationDefaultPolicy(IResourceValidator.ReferenceValidationPolicy.IGNORE);
myFhirCtx.setParserErrorHandler(new StrictErrorHandler());
myUnknownCodeSystemWarningValidationSupport.setAllowNonExistentCodeSystem(UnknownCodeSystemWarningValidationSupport.ALLOW_NON_EXISTENT_CODE_SYSTEM_DEFAULT);
}
/**
* By default an unknown code system should fail vaildation
*/
@Test
public void testValidateCodeInValueSetWithUnknownCodeSystem() {
public void testValidateCodeInValueSetWithUnknownCodeSystem_FailValidation() {
createStructureDefWithBindingToUnknownCs();
Observation obs = new Observation();
obs.getMeta().addProfile("http://sd");
obs.getText().setDivAsString("<div>Hello</div>");
obs.getText().setStatus(Narrative.NarrativeStatus.GENERATED);
obs.getCategoryFirstRep().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/observation-category").setCode("vital-signs");
obs.getCode().setText("hello");
obs.setSubject(new Reference("Patient/123"));
obs.addPerformer(new Reference("Practitioner/123"));
obs.setEffective(DateTimeType.now());
obs.setStatus(ObservationStatus.FINAL);
OperationOutcome oo;
// Valid code
obs.setValue(new Quantity().setSystem("http://cs").setCode("code1").setValue(123));
oo = validateAndReturnOutcome(obs);
String encoded = encode(oo);
ourLog.info(encoded);
assertEquals("No issues detected during validation", oo.getIssueFirstRep().getDiagnostics(), encoded);
// Invalid code
obs.setValue(new Quantity().setSystem("http://cs").setCode("code99").setValue(123));
oo = validateAndReturnOutcome(obs);
encoded = encode(oo);
ourLog.info(encoded);
assertEquals(1, oo.getIssue().size(), encoded);
assertEquals("The code provided (http://cs#code99) is not in the value set http://vs, and a code from this value set is required: Unknown code system: http://cs", oo.getIssueFirstRep().getDiagnostics(), encoded);
assertEquals(OperationOutcome.IssueSeverity.ERROR, oo.getIssueFirstRep().getSeverity(), encoded);
}
/**
* By default an unknown code system should fail vaildation
*/
@Test
public void testValidateCodeInValueSetWithUnknownCodeSystem_Warning() {
myUnknownCodeSystemWarningValidationSupport.setAllowNonExistentCodeSystem(true);
createStructureDefWithBindingToUnknownCs();
Observation obs = new Observation();
obs.getMeta().addProfile("http://sd");
obs.getText().setDivAsString("<div>Hello</div>");
obs.getText().setStatus(Narrative.NarrativeStatus.GENERATED);
obs.getCategoryFirstRep().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/observation-category").setCode("vital-signs");
obs.getCode().setText("hello");
obs.setSubject(new Reference("Patient/123"));
obs.addPerformer(new Reference("Practitioner/123"));
obs.setEffective(DateTimeType.now());
obs.setStatus(ObservationStatus.FINAL);
OperationOutcome oo;
String encoded;
// Valid code
obs.setValue(new Quantity().setSystem("http://cs").setCode("code1").setValue(123));
oo = validateAndReturnOutcome(obs);
encoded = encode(oo);
ourLog.info(encoded);
assertEquals("No issues detected during validation", oo.getIssueFirstRep().getDiagnostics(), encoded);
// Invalid code
obs.setValue(new Quantity().setSystem("http://cs").setCode("code99").setValue(123));
oo = validateAndReturnOutcome(obs);
encoded = encode(oo);
ourLog.info(encoded);
assertEquals(1, oo.getIssue().size(), encoded);
assertEquals("Error Unknown code system: http://cs validating Coding", oo.getIssueFirstRep().getDiagnostics(), encoded);
assertEquals(OperationOutcome.IssueSeverity.WARNING, oo.getIssueFirstRep().getSeverity(), encoded);
}
public void createStructureDefWithBindingToUnknownCs() {
myValidationSupport.fetchCodeSystem("http://not-exist"); // preload DefaultProfileValidationSupport
ValueSet vs = new ValueSet();
@ -132,32 +223,6 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
.setBinding(new ElementDefinition.ElementDefinitionBindingComponent().setStrength(Enumerations.BindingStrength.REQUIRED).setValueSet("http://vs"))
.setId("Observation.value[x]");
myStructureDefinitionDao.create(sd);
Observation obs = new Observation();
obs.getMeta().addProfile("http://sd");
obs.getText().setDivAsString("<div>Hello</div>");
obs.getText().setStatus(Narrative.NarrativeStatus.GENERATED);
obs.getCategoryFirstRep().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/observation-category").setCode("vital-signs");
obs.getCode().setText("hello");
obs.setSubject(new Reference("Patient/123"));
obs.addPerformer(new Reference("Practitioner/123"));
obs.setEffective(DateTimeType.now());
obs.setStatus(ObservationStatus.FINAL);
OperationOutcome oo;
// Valid code
obs.setValue(new Quantity().setSystem("http://cs").setCode("code1").setValue(123));
oo = validateAndReturnOutcome(obs);
ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo));
assertEquals("No issues detected during validation", oo.getIssueFirstRep().getDiagnostics(), encode(oo));
// Invalid code
obs.setValue(new Quantity().setSystem("http://cs").setCode("code99").setValue(123));
oo = validateAndReturnOutcome(obs);
ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo));
assertEquals("The code provided (http://cs#code99) is not in the value set http://vs, and a code from this value set is required: Unknown code {http://cs}code99 - Unable to expand ValueSet[http://vs]", oo.getIssueFirstRep().getDiagnostics(), encode(oo));
}
@Test
@ -620,8 +685,6 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
}
@Test
public void testValidateValueSet() {
String input = "{\n" +
@ -692,7 +755,7 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
assertThat(ooString, containsString("Unknown code in fragment CodeSystem 'http://example.com/codesystem#foo'"));
assertThat(oo.getIssue().stream().map(t->t.getSeverity().toCode()).collect(Collectors.toList()), contains("warning", "warning"));
assertThat(oo.getIssue().stream().map(t -> t.getSeverity().toCode()).collect(Collectors.toList()), contains("warning", "warning"));
}
@ -1080,8 +1143,8 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
}
private String encode(IBaseResource thePatient) {
return myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(thePatient);
private String encode(IBaseResource theResource) {
return myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(theResource);
}
@ -1245,21 +1308,6 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
}
}
@AfterEach
public void after() {
FhirInstanceValidator val = AopTestUtils.getTargetObject(myValidatorModule);
val.setBestPracticeWarningLevel(IResourceValidator.BestPracticeWarningLevel.Warning);
myDaoConfig.setAllowExternalReferences(new DaoConfig().isAllowExternalReferences());
myDaoConfig.setMaximumExpansionSize(DaoConfig.DEFAULT_MAX_EXPANSION_SIZE);
myDaoConfig.setPreExpandValueSets(new DaoConfig().isPreExpandValueSets());
BaseTermReadSvcImpl.setInvokeOnNextCallForUnitTest(null);
myValidationSettings.setLocalReferenceValidationDefaultPolicy(IResourceValidator.ReferenceValidationPolicy.IGNORE);
myFhirCtx.setParserErrorHandler(new StrictErrorHandler());
}
@Test
public void testValidateCapabilityStatement() {
@ -1277,7 +1325,7 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
cs.getText().setStatus(Narrative.NarrativeStatus.GENERATED).getDiv().setValue("<div>aaaa</div>");
CapabilityStatement.CapabilityStatementRestComponent rest = cs.addRest();
CapabilityStatement.CapabilityStatementRestResourceComponent patient = rest.addResource();
patient .setType("Patient");
patient.setType("Patient");
patient.addSearchParam().setName("foo").setType(Enumerations.SearchParamType.DATE).setDefinition("http://example.com/name");
@ -1676,5 +1724,4 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
}
}

4
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationDstu3Test.java
View File

@ -34,6 +34,7 @@ import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test {
@ -245,8 +246,7 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test {
IValidationSupport.CodeValidationResult result = myValueSetDao.validateCode(new UriType("http://loinc.org/vs"), null, new StringType("10013-1-9999999999"), new StringType(ITermLoaderSvc.LOINC_URI), null, null, null, mySrd);
assertFalse(result.isOk());
assertEquals("Unknown code {http://loinc.org}10013-1-9999999999 - Unable to expand ValueSet[http://loinc.org/vs]", result.getMessage());
assertNull(result);
}
private Set<String> toExpandedCodes(ValueSet theExpanded) {

58
hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/UnknownCodeSystemWarningValidationSupport.java Normal file
View File

@ -0,0 +1,58 @@
package org.hl7.fhir.common.hapi.validation.support;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.support.ConceptValidationOptions;
import ca.uhn.fhir.context.support.ValidationSupportContext;
import org.hl7.fhir.exceptions.TerminologyServiceException;
import org.hl7.fhir.instance.model.api.IBaseResource;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
/**
* This validation support module may be placed at the end of a {@link ValidationSupportChain}
* in order to configure the validator to generate a warning if a resource being validated
* contains an unknown code system.
*
* Note that this module must also be activated by calling {@link #setAllowNonExistentCodeSystem(boolean)}
* in order to specify that unknown code systems should be allowed.
*/
public class UnknownCodeSystemWarningValidationSupport extends BaseValidationSupport {
public static final boolean ALLOW_NON_EXISTENT_CODE_SYSTEM_DEFAULT = false;
private boolean myAllowNonExistentCodeSystem = ALLOW_NON_EXISTENT_CODE_SYSTEM_DEFAULT;
/**
* Constructor
*/
public UnknownCodeSystemWarningValidationSupport(FhirContext theFhirContext) {
super(theFhirContext);
}
@Override
public boolean isValueSetSupported(ValidationSupportContext theValidationSupportContext, String theValueSetUrl) {
return true;
}
@Nullable
@Override
public CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, @Nonnull IBaseResource theValueSet) {
IBaseResource codeSystem = theValidationSupportContext.getRootValidationSupport().fetchCodeSystem(theCodeSystem);
if (codeSystem != null) {
return null;
}
String message = "Unknown code system: " + theCodeSystem;
if (!myAllowNonExistentCodeSystem) {
return new CodeValidationResult()
.setSeverity(IssueSeverity.ERROR)
.setMessage(message);
}
throw new TerminologyServiceException(message);
}
public void setAllowNonExistentCodeSystem(boolean theAllowNonExistentCodeSystem) {
myAllowNonExistentCodeSystem = theAllowNonExistentCodeSystem;
}
}