Issue 3072 using an external terminology server in the fhir server for codes in fhir resources (#3081)

* Let RemoteTerminologyServiceValidationSupport.validateCodeInValueSet work when code system is present, even when theOptions.isInferSystem() is true

* Add test for specific use case added

* Consider ValueSets with multiple component includes

* Fix typo

Co-authored-by: juan.marchionatto <juan.marchionatto@smilecdr.com>

hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupport.java

@@ -7,18 +7,22 @@ import ca.uhn.fhir.context.support.IValidationSupport;
 import ca.uhn.fhir.context.support.ValidationSupportContext;
 import ca.uhn.fhir.rest.client.api.IGenericClient;
 import ca.uhn.fhir.util.BundleUtil;
+import ca.uhn.fhir.util.JsonUtil;
 import ca.uhn.fhir.util.ParametersUtil;
 import org.apache.commons.lang3.Validate;
+import org.checkerframework.framework.qual.InvisibleQualifier;
 import org.hl7.fhir.instance.model.api.IBaseBundle;
 import org.hl7.fhir.instance.model.api.IBaseParameters;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.r4.model.CodeSystem;
 import org.hl7.fhir.r4.model.ValueSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nonnull;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
 
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
@@ -30,6 +34,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
  * operation in order to validate codes.
  */
 public class RemoteTerminologyServiceValidationSupport extends BaseValidationSupport implements IValidationSupport {
+	private static final Logger ourLog = LoggerFactory.getLogger(RemoteTerminologyServiceValidationSupport.class);
 
 	private String myBaseUrl;
 	private List<Object> myClientInterceptors = new ArrayList<>();
@@ -57,10 +62,7 @@ public class RemoteTerminologyServiceValidationSupport extends BaseValidationSup
 		// so let's try to get it from the VS if is is not present
 		String codeSystem = theCodeSystem;
 		if (isNotBlank(theCode) && isBlank(codeSystem)) {
-			ValueSet vs = (ValueSet) theValueSet;
+			codeSystem = extractCodeSystemForCode((ValueSet) theValueSet, theCode);
-			if ( vs.getCompose() != null && vs.getCompose().getInclude() != null && vs.getCompose().getInclude().size() > 0) {
-				codeSystem = vs.getCompose().getInclude().iterator().next().getSystem();
-			}
 		}
 
 	 	// Remote terminology services shouldn't be used to validate codes with an implied system
@@ -75,6 +77,54 @@ public class RemoteTerminologyServiceValidationSupport extends BaseValidationSup
 		return invokeRemoteValidateCode(codeSystem, theCode, theDisplay, valueSetUrl, valueSet);
 	}
 
+	/**
+	 * Try to obtain the codeSystem of the received code from the received ValueSet
+	 */
+	private String extractCodeSystemForCode(ValueSet theValueSet, String theCode) {
+		if (theValueSet.getCompose() == null || theValueSet.getCompose().getInclude() == null
+					|| theValueSet.getCompose().getInclude().isEmpty()) {
+			return null;
+		}
+
+		if (theValueSet.getCompose().getInclude().size() == 1) {
+			ValueSet.ConceptSetComponent include = theValueSet.getCompose().getInclude().iterator().next();
+			return getVersionedCodeSystem(include);
+		}
+
+		// when component has more than one include, their codeSystem(s) could be different, so we need to make sure
+		// that we are picking up the system for the include to which the code corresponds
+		for (ValueSet.ConceptSetComponent include: theValueSet.getCompose().getInclude()) {
+			if (include.hasSystem()) {
+				for (ValueSet.ConceptReferenceComponent concept : include.getConcept()) {
+					if (concept.hasCodeElement() && concept.getCode().equals(theCode)) {
+						return getVersionedCodeSystem(include);
+					}
+				}
+			}
+		}
+
+		// at this point codeSystem couldn't be extracted for a multi-include ValueSet. Just on case it was
+		// because the format was not well handled, let's allow to watch the VS by an easy logging change
+		try {
+			ourLog.trace("CodeSystem couldn't be extracted for code: {} for ValueSet: {}",
+				theCode, JsonUtil.serialize(theValueSet));
+		} catch (IOException theE) {
+			ourLog.error("IOException trying to serialize ValueSet to json: " + theE);
+		}
+
+		return null;
+	}
+
+
+	private String getVersionedCodeSystem(ValueSet.ConceptSetComponent theComponent) {
+			String codeSystem = theComponent.getSystem();
+			if ( ! codeSystem.contains("|") && theComponent.hasVersion()) {
+				codeSystem += "|" + theComponent.getVersion();
+			}
+			return codeSystem;
+	}
+
+
 	@Override
 	public IBaseResource fetchCodeSystem(String theSystem) {
 		IGenericClient client = provideClient();
@@ -136,37 +186,13 @@ public class RemoteTerminologyServiceValidationSupport extends BaseValidationSup
 
 		IGenericClient client = provideClient();
 
-		IBaseParameters input = ParametersUtil.newInstance(getFhirContext());
+		IBaseParameters input = buildValidateCodeInputParameters(theCodeSystem, theCode, theDisplay, theValueSetUrl, theValueSet);
 
 		String resourceType = "ValueSet";
 		if (theValueSet == null && theValueSetUrl == null) {
 			resourceType = "CodeSystem";
-
-			ParametersUtil.addParameterToParametersUri(getFhirContext(), input, "url", theCodeSystem);
-			ParametersUtil.addParameterToParametersString(getFhirContext(), input, "code", theCode);
-			if (isNotBlank(theDisplay)) {
-				ParametersUtil.addParameterToParametersString(getFhirContext(), input, "display", theDisplay);
 		}
 
-		} else {
-
-			if (isNotBlank(theValueSetUrl)) {
-				ParametersUtil.addParameterToParametersUri(getFhirContext(), input, "url", theValueSetUrl);
-			}
-			ParametersUtil.addParameterToParametersString(getFhirContext(), input, "code", theCode);
-			if (isNotBlank(theCodeSystem)) {
-				ParametersUtil.addParameterToParametersUri(getFhirContext(), input, "system", theCodeSystem);
-			}
-			if (isNotBlank(theDisplay)) {
-				ParametersUtil.addParameterToParametersString(getFhirContext(), input, "display", theDisplay);
-			}
-			if (theValueSet != null) {
-				ParametersUtil.addParameterToParameters(getFhirContext(), input, "valueSet", theValueSet);
-			}
-
-		}
-
-
 		IBaseParameters output = client
 			.operation()
 			.onType(resourceType)
@@ -203,6 +229,35 @@ public class RemoteTerminologyServiceValidationSupport extends BaseValidationSup
 		return retVal;
 	}
 
+	protected IBaseParameters buildValidateCodeInputParameters(String theCodeSystem, String theCode, String theDisplay, String theValueSetUrl, IBaseResource theValueSet) {
+		IBaseParameters params = ParametersUtil.newInstance(getFhirContext());
+
+		if (theValueSet == null && theValueSetUrl == null) {
+			ParametersUtil.addParameterToParametersUri(getFhirContext(), params, "url", theCodeSystem);
+			ParametersUtil.addParameterToParametersString(getFhirContext(), params, "code", theCode);
+			if (isNotBlank(theDisplay)) {
+				ParametersUtil.addParameterToParametersString(getFhirContext(), params, "display", theDisplay);
+			}
+			return params;
+		}
+
+		if (isNotBlank(theValueSetUrl)) {
+			ParametersUtil.addParameterToParametersUri(getFhirContext(), params, "url", theValueSetUrl);
+		}
+		ParametersUtil.addParameterToParametersString(getFhirContext(), params, "code", theCode);
+		if (isNotBlank(theCodeSystem)) {
+			ParametersUtil.addParameterToParametersUri(getFhirContext(), params, "system", theCodeSystem);
+		}
+		if (isNotBlank(theDisplay)) {
+			ParametersUtil.addParameterToParametersString(getFhirContext(), params, "display", theDisplay);
+		}
+		if (theValueSet != null) {
+			ParametersUtil.addParameterToParameters(getFhirContext(), params, "valueSet", theValueSet);
+		}
+		return params;
+	}
+
+
 	/**
 	 * Sets the FHIR Terminology Server base URL
 	 *

hapi-fhir-validation/src/test/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupportTest.java

@@ -3,15 +3,21 @@ package org.hl7.fhir.common.hapi.validation.support;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.support.ConceptValidationOptions;
 import ca.uhn.fhir.context.support.IValidationSupport;
+import ca.uhn.fhir.parser.IJsonLikeParser;
 import ca.uhn.fhir.rest.annotation.IdParam;
 import ca.uhn.fhir.rest.annotation.Operation;
 import ca.uhn.fhir.rest.annotation.OperationParam;
 import ca.uhn.fhir.rest.annotation.RequiredParam;
 import ca.uhn.fhir.rest.annotation.Search;
+import ca.uhn.fhir.rest.client.api.IClientInterceptor;
+import ca.uhn.fhir.rest.client.api.IHttpRequest;
+import ca.uhn.fhir.rest.client.api.IHttpResponse;
 import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
 import ca.uhn.fhir.rest.param.UriParam;
 import ca.uhn.fhir.rest.server.IResourceProvider;
 import ca.uhn.fhir.test.utilities.server.RestfulServerExtension;
+import ca.uhn.fhir.util.ParametersUtil;
+import com.google.common.collect.Lists;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.r4.model.BooleanType;
 import org.hl7.fhir.r4.model.CodeSystem;
@@ -23,10 +29,12 @@ import org.hl7.fhir.r4.model.UriType;
 import org.hl7.fhir.r4.model.ValueSet;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -35,6 +43,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.lessThan;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 public class RemoteTerminologyServiceValidationSupportTest {
 
@@ -64,7 +73,7 @@ public class RemoteTerminologyServiceValidationSupportTest {
 
 		mySvc = new RemoteTerminologyServiceValidationSupport(ourCtx);
 		mySvc.setBaseUrl(baseUrl);
-		mySvc.addClientInterceptor(new LoggingInterceptor(false));
+		mySvc.addClientInterceptor(new LoggingInterceptor(true));
 	}
 
 	@AfterEach
@@ -152,7 +161,7 @@ public class RemoteTerminologyServiceValidationSupportTest {
 	 * Remote terminology services shouldn't be used to validate codes with an implied system
 	 */
 	@Test
-	public void testValidateCodeInValueSet_InferSystem_codeSystemNotPresent() {
+	public void testValidateCodeInValueSet_InferSystem() {
 		createNextValueSetReturnParameters(true, DISPLAY, null);
 
 		ValueSet valueSet = new ValueSet();
@@ -166,8 +175,11 @@ public class RemoteTerminologyServiceValidationSupportTest {
 	 * Remote terminology services can be used to validate codes when code system is present,
 	 * even when inferSystem is true
 	 */
+	@Nested
+	public class ExtractCodeSystemFromValueSet {
+
 		@Test
-	public void testValidateCodeInValueSet_InferSystem_codeSystemIsPresent() {
+		public void testUniqueComposeInclude() {
 			createNextValueSetReturnParameters(true, DISPLAY, null);
 
 			ValueSet valueSet = new ValueSet();
@@ -183,6 +195,139 @@ public class RemoteTerminologyServiceValidationSupportTest {
 			assertNotNull(outcome);
 		}
 
+
+		@Nested
+		public class MultiComposeIncludeValueSet {
+
+			@Test
+			public void SystemNotPresentReturnsNull() {
+				createNextValueSetReturnParameters(true, DISPLAY, null);
+
+				ValueSet valueSet = new ValueSet();
+				valueSet.setUrl(VALUE_SET_URL);
+				valueSet.setCompose(new ValueSet.ValueSetComposeComponent().setInclude(
+					Lists.newArrayList(new ValueSet.ConceptSetComponent(), new ValueSet.ConceptSetComponent()) ));
+
+				IValidationSupport.CodeValidationResult outcome = mySvc.validateCodeInValueSet(null,
+					new ConceptValidationOptions().setInferSystem(true), null, CODE, DISPLAY, valueSet);
+
+				assertNull(outcome);
+			}
+
+
+			@Test
+			public void SystemPresentCodeNotPresentReturnsNull() {
+				createNextValueSetReturnParameters(true, DISPLAY, null);
+
+				ValueSet valueSet = new ValueSet();
+				valueSet.setUrl(VALUE_SET_URL);
+				String systemUrl = "http://hl7.org/fhir/ValueSet/administrative-gender";
+				String systemUrl2 = "http://hl7.org/fhir/ValueSet/other-valueset";
+				valueSet.setCompose(new ValueSet.ValueSetComposeComponent().setInclude(
+					Lists.newArrayList(
+						new ValueSet.ConceptSetComponent().setSystem(systemUrl),
+						new ValueSet.ConceptSetComponent().setSystem(systemUrl2)) ));
+
+				IValidationSupport.CodeValidationResult outcome = mySvc.validateCodeInValueSet(null,
+					new ConceptValidationOptions().setInferSystem(true), null, CODE, DISPLAY, valueSet);
+
+				assertNull(outcome);
+			}
+
+
+			@Test
+			public void SystemPresentCodePresentValidatesOKNoVersioned() {
+				createNextValueSetReturnParameters(true, DISPLAY, null);
+
+				ValueSet valueSet = new ValueSet();
+				valueSet.setUrl(VALUE_SET_URL);
+				String systemUrl = "http://hl7.org/fhir/ValueSet/administrative-gender";
+				String systemUrl2 = "http://hl7.org/fhir/ValueSet/other-valueset";
+				valueSet.setCompose(new ValueSet.ValueSetComposeComponent().setInclude(
+					Lists.newArrayList(
+						new ValueSet.ConceptSetComponent().setSystem(systemUrl),
+						new ValueSet.ConceptSetComponent().setSystem(systemUrl2).setConcept(
+							Lists.newArrayList(
+								new ValueSet.ConceptReferenceComponent().setCode("not-the-code"),
+								new ValueSet.ConceptReferenceComponent().setCode(CODE) )
+						)) ));
+
+				TestClientInterceptor requestInterceptor = new TestClientInterceptor();
+				mySvc.addClientInterceptor(requestInterceptor);
+
+				IValidationSupport.CodeValidationResult outcome = mySvc.validateCodeInValueSet(null,
+					new ConceptValidationOptions().setInferSystem(true), null, CODE, DISPLAY, valueSet);
+
+				assertNotNull(outcome);
+				assertEquals(systemUrl2, requestInterceptor.getCapturedSystemParameter());
+			}
+
+
+			@Test
+			public void SystemPresentCodePresentValidatesOKVersioned() {
+				createNextValueSetReturnParameters(true, DISPLAY, null);
+
+				ValueSet valueSet = new ValueSet();
+				valueSet.setUrl(VALUE_SET_URL);
+				String systemUrl = "http://hl7.org/fhir/ValueSet/administrative-gender";
+				String systemVersion = "3.0.2";
+				String systemUrl2 = "http://hl7.org/fhir/ValueSet/other-valueset";
+				String system2Version = "4.0.1";
+				valueSet.setCompose(new ValueSet.ValueSetComposeComponent().setInclude(
+					Lists.newArrayList(
+						new ValueSet.ConceptSetComponent().setSystem(systemUrl).setVersion(systemVersion),
+						new ValueSet.ConceptSetComponent().setSystem(systemUrl2).setVersion(system2Version).setConcept(
+							Lists.newArrayList(
+								new ValueSet.ConceptReferenceComponent().setCode("not-the-code"),
+								new ValueSet.ConceptReferenceComponent().setCode(CODE) )
+						)) ));
+
+				TestClientInterceptor requestInterceptor = new TestClientInterceptor();
+				mySvc.addClientInterceptor(requestInterceptor);
+
+				IValidationSupport.CodeValidationResult outcome = mySvc.validateCodeInValueSet(null,
+					new ConceptValidationOptions().setInferSystem(true), null, CODE, DISPLAY, valueSet);
+
+				assertNotNull(outcome);
+				assertEquals(systemUrl2 + "|" + system2Version, requestInterceptor.getCapturedSystemParameter());
+			}
+
+
+		}
+
+		/**
+		 * Captures the system parameter of the request
+		 */
+		private class TestClientInterceptor implements IClientInterceptor {
+
+			private String capturedSystemParameter;
+
+			@Override
+			public void interceptRequest(IHttpRequest theRequest) {
+				try {
+					String content = theRequest.getRequestBodyFromStream();
+					if (content != null) {
+						IJsonLikeParser parser = (IJsonLikeParser) ourCtx.newJsonParser();
+						Parameters params = parser.parseResource(Parameters.class, content);
+						List<String> systemValues = ParametersUtil.getNamedParameterValuesAsString(
+							ourCtx, params, "system");
+						assertEquals(1, systemValues.size());
+						capturedSystemParameter = systemValues.get(0);
+					}
+				} catch (IOException theE) {
+					theE.printStackTrace();
+				}
+			}
+
+			@Override
+			public void interceptResponse(IHttpResponse theResponse) throws IOException { }
+
+			public String getCapturedSystemParameter() { return capturedSystemParameter; }
+		}
+	}
+
+
+
 	@Test
 	public void testIsValueSetSupported_False() {
 		myValueSetProvider.myNextReturnValueSets = new ArrayList<>();