Jr 20240325 concept map unmatched codes (#5809)

* update model

* convert results

* migration

* extend test coverage in cli

* fix broken test

* change log

* code review feedback

* spotless

hapi-fhir-base/src/main/resources/ca/uhn/fhir/i18n/hapi-messages.properties

@@ -155,6 +155,7 @@ ca.uhn.fhir.jpa.search.builder.predicate.TokenPredicateBuilder.invalidCodeMissin
 
 ca.uhn.fhir.jpa.term.TermConceptMappingSvcImpl.matchesFound=Matches found
 ca.uhn.fhir.jpa.term.TermConceptMappingSvcImpl.noMatchesFound=No Matches found
+ca.uhn.fhir.jpa.term.TermConceptMappingSvcImpl.onlyNegativeMatchesFound=Only negative matches found
 
 ca.uhn.fhir.jpa.dao.JpaResourceDaoSearchParameter.invalidSearchParamExpression=The expression "{0}" can not be evaluated and may be invalid: {1}
 

hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/ExportConceptMapToCsvCommandR4Test.java

@@ -93,7 +93,8 @@ public class ExportConceptMapToCsvCommandR4Test {
 			"\"http://example.com/codesystem/2\",\"Version 2s\",\"http://example.com/codesystem/3\",\"Version 3t\",\"Code 2a\",\"Display 2a\",\"Code 3a\",\"Display 3a\",\"equal\",\"3a This is a comment.\"\n" +
 			"\"http://example.com/codesystem/2\",\"Version 2s\",\"http://example.com/codesystem/3\",\"Version 3t\",\"Code 2b\",\"Display 2b\",\"Code 3b\",\"Display 3b\",\"equal\",\"3b This is a comment.\"\n" +
 			"\"http://example.com/codesystem/2\",\"Version 2s\",\"http://example.com/codesystem/3\",\"Version 3t\",\"Code 2c\",\"Display 2c\",\"Code 3c\",\"Display 3c\",\"equal\",\"3c This is a comment.\"\n" +
-			"\"http://example.com/codesystem/2\",\"Version 2s\",\"http://example.com/codesystem/3\",\"Version 3t\",\"Code 2d\",\"Display 2d\",\"Code 3d\",\"Display 3d\",\"equal\",\"3d This is a comment.\"\n";
+			"\"http://example.com/codesystem/2\",\"Version 2s\",\"http://example.com/codesystem/3\",\"Version 3t\",\"Code 2d\",\"Display 2d\",\"Code 3d\",\"Display 3d\",\"equal\",\"3d This is a comment.\"\n" +
+			"\"http://example.com/codesystem/2\",\"Version 2s\",\"http://example.com/codesystem/3\",\"Version 3t\",\"Code 2e\",\"Display 2e\",\"\",\"\",\"unmatched\",\"3e This is a comment.\"\n";
 		String result = IOUtils.toString(new FileInputStream(FILE), Charsets.UTF_8);
 		assertEquals(expected, result);
 
@@ -272,6 +273,16 @@ public class ExportConceptMapToCsvCommandR4Test {
 			.setEquivalence(ConceptMapEquivalence.EQUAL)
 			.setComment("3d This is a comment.");
 
+		element = group.addElement();
+		element
+			.setCode("Code 2e")
+			.setDisplay("Display 2e");
+
+		target = element.addTarget();
+		target
+			.setEquivalence(ConceptMapEquivalence.UNMATCHED)
+			.setComment("3e This is a comment.");
+
 		return conceptMap;
 	}
 }

hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/ImportCsvToConceptMapCommandDstu3Test.java

@@ -273,7 +273,7 @@ public class ImportCsvToConceptMapCommandDstu3Test {
 		assertEquals(CS_URL_3, group.getTarget());
 		assertEquals("Version 3t", group.getTargetVersion());
 
-		assertEquals(4, group.getElement().size());
+		assertEquals(5, group.getElement().size());
 
 		source = group.getElement().get(0);
 		assertEquals("Code 2a", source.getCode());
@@ -323,6 +323,19 @@ public class ImportCsvToConceptMapCommandDstu3Test {
 		assertEquals(ConceptMapEquivalence.EQUAL, target.getEquivalence());
 		assertEquals("3d This is a comment.", target.getComment());
 
+		// ensure unmatched codes are handled correctly
+		source = group.getElement().get(4);
+		assertEquals("Code 2e", source.getCode());
+		assertEquals("Display 2e", source.getDisplay());
+
+		assertEquals(1, source.getTarget().size());
+
+		target = source.getTarget().get(0);
+		assertNull(target.getCode());
+		assertNull(target.getDisplay());
+		assertEquals(ConceptMapEquivalence.UNMATCHED, target.getEquivalence());
+		assertEquals("3e This is a comment.", target.getComment());
+
 		App.main(myTlsAuthenticationTestHelper.createBaseRequestGeneratingCommandArgs(
 			new String[]{
 				ImportCsvToConceptMapCommand.COMMAND,

hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/ImportCsvToConceptMapCommandR4Test.java

@@ -284,7 +284,7 @@ public class ImportCsvToConceptMapCommandR4Test {
 		assertEquals(CS_URL_3, group.getTarget());
 		assertEquals("Version 3t", group.getTargetVersion());
 
-		assertEquals(4, group.getElement().size());
+		assertEquals(5, group.getElement().size());
 
 		source = group.getElement().get(0);
 		assertEquals("Code 2a", source.getCode());
@@ -334,6 +334,19 @@ public class ImportCsvToConceptMapCommandR4Test {
 		assertEquals(ConceptMapEquivalence.EQUAL, target.getEquivalence());
 		assertEquals("3d This is a comment.", target.getComment());
 
+		// ensure unmatched codes are handled correctly
+		source = group.getElement().get(4);
+		assertEquals("Code 2e", source.getCode());
+		assertEquals("Display 2e", source.getDisplay());
+
+		assertEquals(1, source.getTarget().size());
+
+		target = source.getTarget().get(0);
+		assertNull(target.getCode());
+		assertNull(target.getDisplay());
+		assertEquals(ConceptMapEquivalence.UNMATCHED, target.getEquivalence());
+		assertEquals("3e This is a comment.", target.getComment());
+
 		App.main(myTlsAuthenticationTestHelper.createBaseRequestGeneratingCommandArgs(
 			new String[]{
 				ImportCsvToConceptMapCommand.COMMAND,

hapi-fhir-cli/hapi-fhir-cli-api/src/test/resources/import-csv-to-conceptmap-command-test-input.csv

@@ -11,3 +11,4 @@
 "http://example.com/codesystem/2","Version 2s","http://example.com/codesystem/3","Version 3t","Code 2b","Display 2b","Code 3b","Display 3b","equal","3b This is a comment."
 "http://example.com/codesystem/2","Version 2s","http://example.com/codesystem/3","Version 3t","Code 2c","Display 2c","Code 3c","Display 3c","equal","3c This is a comment."
 "http://example.com/codesystem/2","Version 2s","http://example.com/codesystem/3","Version 3t","Code 2d","Display 2d","Code 3d","Display 3d","equal","3d This is a comment."
+"http://example.com/codesystem/2","Version 2s","http://example.com/codesystem/3","Version 3t","Code 2e","Display 2e","","","unmatched","3e This is a comment."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5816-unmatched-translate-target.yaml

@@ -0,0 +1,5 @@
+---
+type: add
+issue: 5816
+title: "The ConceptMap/$translate operation will include targets with an equivalence code of `unmatched` in the response 
+regardless of whether the target has a code."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptMapGroupElementTarget.java

@@ -58,7 +58,7 @@ public class TermConceptMapGroupElementTarget implements Serializable {
 			foreignKey = @ForeignKey(name = "FK_TCMGETARGET_ELEMENT"))
 	private TermConceptMapGroupElement myConceptMapGroupElement;
 
-	@Column(name = "TARGET_CODE", nullable = false, length = TermConcept.MAX_CODE_LENGTH)
+	@Column(name = "TARGET_CODE", nullable = true, length = TermConcept.MAX_CODE_LENGTH)
 	private String myCode;
 
 	@Column(name = "TARGET_DISPLAY", nullable = true, length = TermConcept.MAX_DISP_LENGTH)

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java

@@ -119,6 +119,19 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 		init680();
 		init680_Part2();
 		init700();
+		init720();
+	}
+
+	protected void init720() {
+		// Start of migrations from 7.0 to 7.2
+
+		Builder version = forVersion(VersionEnum.V7_2_0);
+
+		// allow null codes in concept map targets
+		version.onTable("TRM_CONCEPT_MAP_GRP_ELM_TGT")
+				.modifyColumn("20240327.1", "TARGET_CODE")
+				.nullable()
+				.withType(ColumnTypeEnum.STRING, 500);
 	}
 
 	protected void init700() {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermConceptClientMappingSvcImpl.java

@@ -51,6 +51,7 @@ import org.hibernate.ScrollMode;
 import org.hibernate.ScrollableResults;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r4.model.Coding;
+import org.hl7.fhir.r4.model.Enumerations;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -416,6 +417,10 @@ public class TermConceptClientMappingSvcImpl implements ITermConceptClientMappin
 			theTranslationResult.setResult(false);
 			msg = myContext.getLocalizer().getMessage(TermConceptMappingSvcImpl.class, "noMatchesFound");
 			theTranslationResult.setMessage(msg);
+		} else if (isOnlyNegativeMatches(theTranslationResult)) {
+			theTranslationResult.setResult(false);
+			msg = myContext.getLocalizer().getMessage(TermConceptMappingSvcImpl.class, "onlyNegativeMatchesFound");
+			theTranslationResult.setMessage(msg);
 		} else {
 			theTranslationResult.setResult(true);
 			msg = myContext.getLocalizer().getMessage(TermConceptMappingSvcImpl.class, "matchesFound");
@@ -423,6 +428,21 @@ public class TermConceptClientMappingSvcImpl implements ITermConceptClientMappin
 		}
 	}
 
+	/**
+	 * Evaluates whether a translation result contains any positive matches or only negative ones. This is required
+	 * because the <a href="https://hl7.org/fhir/R4/conceptmap-operation-translate.html">FHIR specification</a> states
+	 * that the result field "can only be true if at least one returned match has an equivalence which is not unmatched
+	 * or disjoint".
+	 * @param theTranslationResult the translation result to be evaluated
+	 * @return true if all the potential matches in the result have a negative valence (i.e., "unmatched" and "disjoint")
+	 */
+	private boolean isOnlyNegativeMatches(TranslateConceptResults theTranslationResult) {
+		return theTranslationResult.getResults().stream()
+				.map(TranslateConceptResult::getEquivalence)
+				.allMatch(t -> StringUtils.equals(Enumerations.ConceptMapEquivalence.UNMATCHED.toCode(), t)
+						|| StringUtils.equals(Enumerations.ConceptMapEquivalence.DISJOINT.toCode(), t));
+	}
+
 	private boolean alreadyContainsMapping(
 			List<TranslateConceptResult> elements, TranslateConceptResult translationMatch) {
 		for (TranslateConceptResult nextExistingElement : elements) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermConceptMappingSvcImpl.java

@@ -41,6 +41,7 @@ import org.hl7.fhir.r4.model.BooleanType;
 import org.hl7.fhir.r4.model.CodeType;
 import org.hl7.fhir.r4.model.Coding;
 import org.hl7.fhir.r4.model.ConceptMap;
+import org.hl7.fhir.r4.model.Enumerations;
 import org.hl7.fhir.r4.model.IdType;
 import org.hl7.fhir.r4.model.Parameters;
 import org.hl7.fhir.r4.model.StringType;
@@ -218,13 +219,17 @@ public class TermConceptMappingSvcImpl extends TermConceptClientMappingSvcImpl i
 						if (element.hasTarget()) {
 							TermConceptMapGroupElementTarget termConceptMapGroupElementTarget;
 							for (ConceptMap.TargetElementComponent elementTarget : element.getTarget()) {
-								if (isBlank(elementTarget.getCode())) {
+								if (isBlank(elementTarget.getCode())
+										&& elementTarget.getEquivalence()
+												!= Enumerations.ConceptMapEquivalence.UNMATCHED) {
 									continue;
 								}
 								termConceptMapGroupElementTarget = new TermConceptMapGroupElementTarget();
 								termConceptMapGroupElementTarget.setConceptMapGroupElement(termConceptMapGroupElement);
-								termConceptMapGroupElementTarget.setCode(elementTarget.getCode());
+								if (isNotBlank(elementTarget.getCode())) {
-								termConceptMapGroupElementTarget.setDisplay(elementTarget.getDisplay());
+									termConceptMapGroupElementTarget.setCode(elementTarget.getCode());
+									termConceptMapGroupElementTarget.setDisplay(elementTarget.getDisplay());
+								}
 								termConceptMapGroupElementTarget.setEquivalence(elementTarget.getEquivalence());
 								termConceptMapGroupElement
 										.getConceptMapGroupElementTargets()

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ConceptMapTest.java

@@ -307,6 +307,86 @@ public class ResourceProviderR4ConceptMapTest extends BaseResourceProviderR4Test
 		assertFalse(hasParameterByName(respParams, "match"));
 	}
 
+	@Test
+	public void testTranslateByCodeSystemsAndSourceCodeMappedToCodelessTarget() {
+		// ensure that the current behaviour when a target does not have a code is preserved, and no matches returned
+		ConceptMap conceptMap = myConceptMapDao.read(myConceptMapId, mySrd);
+
+		ourLog.debug("ConceptMap:\n" + myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(conceptMap));
+
+		Parameters inParams = new Parameters();
+		inParams.addParameter().setName("system").setValue(new UriType(CS_URL_4));
+		inParams.addParameter().setName("targetsystem").setValue(new UriType(CS_URL_3));
+		inParams.addParameter().setName("code").setValue(new CodeType("89012"));
+
+		ourLog.debug("Request Parameters:\n" + myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(inParams));
+
+		Parameters respParams = myClient
+			.operation()
+			.onType(ConceptMap.class)
+			.named("translate")
+			.withParameters(inParams)
+			.execute();
+
+		ourLog.debug("Response Parameters\n" + myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(respParams));
+
+		ParametersParameterComponent param = getParameterByName(respParams, "result");
+		assertFalse(((BooleanType) param.getValue()).booleanValue());
+
+		param = getParameterByName(respParams, "message");
+		assertEquals("No Matches found", ((StringType) param.getValue()).getValueAsString());
+
+		assertFalse(hasParameterByName(respParams, "match"));
+	}
+
+	@Test
+	public void testTranslateByCodeSystemsAndSourceCodeWithEquivalenceUnmatched() {
+		// the equivalence code 'unmatched' is an exception - it does not normally have a target code,
+		// so it will be included in the collection of matches even if there is no code present
+		ConceptMap conceptMap = myConceptMapDao.read(myConceptMapId, mySrd);
+
+		ourLog.debug("ConceptMap:\n" + myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(conceptMap));
+
+		Parameters inParams = new Parameters();
+		inParams.addParameter().setName("system").setValue(new UriType(CS_URL_4));
+		inParams.addParameter().setName("targetsystem").setValue(new UriType(CS_URL_3));
+		inParams.addParameter().setName("code").setValue(new CodeType("89123"));
+
+		ourLog.debug("Request Parameters:\n" + myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(inParams));
+
+		Parameters respParams = myClient
+			.operation()
+			.onType(ConceptMap.class)
+			.named("translate")
+			.withParameters(inParams)
+			.execute();
+
+		ourLog.debug("Response Parameters\n" + myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(respParams));
+
+		ParametersParameterComponent param = getParameterByName(respParams, "result");
+		assertFalse(((BooleanType) param.getValue()).booleanValue());
+
+		param = getParameterByName(respParams, "message");
+		assertEquals("Only negative matches found", ((StringType) param.getValue()).getValueAsString());
+
+		assertEquals(1, getNumberOfParametersByName(respParams, "match"));
+
+		param = getParameterByName(respParams, "match");
+		assertEquals(3, param.getPart().size());
+		ParametersParameterComponent part = getPartByName(param, "equivalence");
+		assertEquals("unmatched", ((CodeType) part.getValue()).getCode());
+		part = getPartByName(param, "concept");
+		Coding coding = (Coding) part.getValue();
+		assertNull(coding.getCode());
+		assertNull(coding.getDisplay());
+		assertFalse(coding.getUserSelected());
+		assertEquals(CS_URL_3, coding.getSystem());
+		assertEquals("Version 1", coding.getVersion());
+		part = getPartByName(param, "source");
+		assertEquals(CM_URL, ((UriType) part.getValue()).getValueAsString());
+	}
+
+
 	@Test
 	public void testTranslateUsingPredicatesWithCodeOnly() {
 		ConceptMap conceptMap = myConceptMapDao.read(myConceptMapId);

hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java

@@ -935,6 +935,28 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil
 		target.setDisplay("Target Code 34567");
 		target.setEquivalence(ConceptMapEquivalence.NARROWER);
 
+		group = conceptMap.addGroup();
+		group.setSource(CS_URL_4);
+		group.setSourceVersion("Version 1");
+		group.setTarget(CS_URL_3);
+		group.setTargetVersion("Version 1");
+
+		// This one should not show up in the results because it doesn't have a code
+		element = group.addElement();
+		element.setCode("89012");
+		element.setDisplay("Source Code 89012");
+
+		target = element.addTarget();
+		target.setEquivalence(ConceptMapEquivalence.DISJOINT);
+
+		// This one should show up in the results because unmatched targets are allowed to be codeless
+		element = group.addElement();
+		element.setCode("89123");
+		element.setDisplay("Source Code 89123");
+
+		target = element.addTarget();
+		target.setEquivalence(ConceptMapEquivalence.UNMATCHED);
+
 		return conceptMap;
 	}
 

hapi-fhir-jpaserver-test-utilities/src/main/resources/migration/releases/V7_2_0/data/H2_EMBEDDED.sql

@@ -0,0 +1 @@
+INSERT INTO TRM_CONCEPT_MAP_GRP_ELM_TGT (PID, TARGET_CODE, CONCEPT_MAP_URL, TARGET_DISPLAY, TARGET_EQUIVALENCE, SYSTEM_URL, SYSTEM_VERSION, VALUESET_URL, CONCEPT_MAP_GRP_ELM_PID) VALUES (61, NULL, NULL, 'PYRIDOXINE', 'UNMATCHED', NULL, NULL, NULL, 60);

hapi-fhir-jpaserver-test-utilities/src/main/resources/migration/releases/V7_2_0/data/MSSQL_2012.sql

@@ -0,0 +1 @@
+INSERT INTO TRM_CONCEPT_MAP_GRP_ELM_TGT (PID, TARGET_CODE, CONCEPT_MAP_URL, TARGET_DISPLAY, TARGET_EQUIVALENCE, SYSTEM_URL, SYSTEM_VERSION, VALUESET_URL, CONCEPT_MAP_GRP_ELM_PID) VALUES (61, NULL, NULL, 'PYRIDOXINE', 'UNMATCHED', NULL, NULL, NULL, 60);

hapi-fhir-jpaserver-test-utilities/src/main/resources/migration/releases/V7_2_0/data/ORACLE_12C.sql

@@ -0,0 +1 @@
+INSERT INTO TRM_CONCEPT_MAP_GRP_ELM_TGT (PID, TARGET_CODE, CONCEPT_MAP_URL, TARGET_DISPLAY, TARGET_EQUIVALENCE, SYSTEM_URL, SYSTEM_VERSION, VALUESET_URL, CONCEPT_MAP_GRP_ELM_PID) VALUES (61, NULL, NULL, 'PYRIDOXINE', 'UNMATCHED', NULL, NULL, NULL, 60);

hapi-fhir-jpaserver-test-utilities/src/main/resources/migration/releases/V7_2_0/data/POSTGRES_9_4.sql

@@ -0,0 +1 @@
+INSERT INTO TRM_CONCEPT_MAP_GRP_ELM_TGT (PID, TARGET_CODE, CONCEPT_MAP_URL, TARGET_DISPLAY, TARGET_EQUIVALENCE, SYSTEM_URL, SYSTEM_VERSION, VALUESET_URL, CONCEPT_MAP_GRP_ELM_PID) VALUES (61, NULL, NULL, 'PYRIDOXINE', 'UNMATCHED', NULL, NULL, NULL, 60);

hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/embedded/HapiSchemaMigrationTest.java

@@ -83,7 +83,8 @@ public class HapiSchemaMigrationTest {
 			VersionEnum.V5_4_0,
 			VersionEnum.V5_5_0,
 			VersionEnum.V6_0_0,
-			VersionEnum.V6_6_0
+			VersionEnum.V6_6_0,
+			VersionEnum.V7_2_0
 		);
 
 		int fromVersion = 0;
@@ -92,6 +93,7 @@ public class HapiSchemaMigrationTest {
 
 		for (int i = 0; i < allVersions.length; i++) {
 			toVersion = allVersions[i];
+			ourLog.info("Applying migrations for {}", toVersion);
 			migrate(theDriverType, dataSource, hapiMigrationStorageSvc, toVersion);
 			if (dataVersions.contains(toVersion)) {
 				myEmbeddedServersExtension.insertPersistenceTestData(theDriverType, toVersion);