Resolve #1320: "import-csv-to-conceptmap is failing unexpectedly." (#1322)

* Reproduced issue in unit test and identified potential solution.

* Fixed using BOMInputStream.

* Added changelog.

* Added a missing word in changelog.

* Added a missing word in changelog. Take two.

hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/AbstractImportExportCsvConceptMapCommand.java

@@ -27,10 +27,19 @@ import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.commons.io.input.BOMInputStream;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
-import java.util.*;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
@@ -64,6 +73,29 @@ public abstract class AbstractImportExportCsvConceptMapCommand extends BaseComma
 		addRequiredOption(theOptions, FHIR_VERSION_PARAM, FHIR_VERSION_PARAM_LONGOPT, FHIR_VERSION_PARAM_NAME, FHIR_VERSION_PARAM_DESC + versions);
 	}
 
+	protected BufferedReader getBufferedReader() throws IOException {
+		return new BufferedReader(getInputStreamReader());
+	}
+
+	protected InputStreamReader getInputStreamReader() throws IOException {
+		return new InputStreamReader(getBOMInputStream());
+	}
+
+	protected BOMInputStream getBOMInputStream() throws IOException {
+		return new BOMInputStream(
+			getInputStream(),
+			false,
+			ByteOrderMark.UTF_8,
+			ByteOrderMark.UTF_16BE,
+			ByteOrderMark.UTF_16LE,
+			ByteOrderMark.UTF_32BE,
+			ByteOrderMark.UTF_32LE);
+	}
+
+	protected InputStream getInputStream() throws IOException {
+		return Files.newInputStream(Paths.get(file), StandardOpenOption.READ);
+	}
+
 	@Override
 	public void run(CommandLine theCommandLine) throws ParseException, ExecutionException {
 		parseFhirContext(theCommandLine);

hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/ImportCsvToConceptMapCommand.java

@@ -39,8 +39,6 @@ import org.hl7.fhir.r4.model.UriType;
 
 import java.io.IOException;
 import java.io.Reader;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.Map;
@@ -166,7 +164,7 @@ public class ImportCsvToConceptMapCommand extends AbstractImportExportCsvConcept
 		ourLog.info("Converting CSV to ConceptMap...");
 		ConceptMap retVal = new ConceptMap();
 		try (
-			Reader reader = Files.newBufferedReader(Paths.get(file));
+			Reader reader = getBufferedReader();
 			CSVParser csvParser = new CSVParser(
 				reader,
 				CSVFormat

hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/ImportCsvToConceptMapCommandR4Test.java

@@ -369,4 +369,79 @@ public class ImportCsvToConceptMapCommandR4Test {
 
 		assertEquals("http://localhost:" + ourPort + "/ConceptMap/1/_history/2", conceptMap.getId());
 	}
+
+	@Test
+	public void testImportCsvToConceptMapCommandWithByteOrderMark() throws FHIRException {
+		ClassLoader classLoader = getClass().getClassLoader();
+		File fileToImport = new File(classLoader.getResource("loinc-to-phenx.csv").getFile());
+		ImportCsvToConceptMapCommandR4Test.file = fileToImport.getAbsolutePath();
+
+		App.main(new String[] {"import-csv-to-conceptmap",
+			"-v", ourVersion,
+			"-t", ourBase,
+			"-u", "http://loinc.org/cm/loinc-to-phenx",
+			"-i", "http://loinc.org",
+			"-o", "http://phenxtoolkit.org",
+			"-f", file,
+			"-l"});
+
+		Bundle response = ourClient
+			.search()
+			.forResource(ConceptMap.class)
+			.where(ConceptMap.URL.matches().value("http://loinc.org/cm/loinc-to-phenx"))
+			.returnBundle(Bundle.class)
+			.execute();
+
+		ConceptMap conceptMap = (ConceptMap) response.getEntryFirstRep().getResource();
+
+		ourLog.info(ourCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(conceptMap));
+
+		assertEquals("http://localhost:" + ourPort + "/ConceptMap/1/_history/1", conceptMap.getId());
+
+		assertEquals("http://loinc.org/cm/loinc-to-phenx", conceptMap.getUrl());
+		assertEquals("http://loinc.org", conceptMap.getSourceUriType().getValueAsString());
+		assertEquals("http://phenxtoolkit.org", conceptMap.getTargetUriType().getValueAsString());
+
+		assertEquals(1, conceptMap.getGroup().size());
+
+		ConceptMapGroupComponent group = conceptMap.getGroup().get(0);
+		assertEquals("http://loinc.org", group.getSource());
+		assertNull(group.getSourceVersion());
+		assertEquals("http://phenxtoolkit.org", group.getTarget());
+		assertNull(group.getTargetVersion());
+
+		assertEquals(1, group.getElement().size());
+
+		SourceElementComponent source = group.getElement().get(0);
+		assertEquals("65191-9", source.getCode());
+		assertEquals("During the past 30 days, about how often did you feel restless or fidgety [Kessler 6 Distress]", source.getDisplay());
+
+		assertEquals(1, source.getTarget().size());
+
+		TargetElementComponent target = source.getTarget().get(0);
+		assertEquals("PX121301010300", target.getCode());
+		assertEquals("PX121301_Restless", target.getDisplay());
+		assertEquals(ConceptMapEquivalence.EQUIVALENT, target.getEquivalence());
+		assertNull(target.getComment());
+
+		App.main(new String[] {"import-csv-to-conceptmap",
+			"-v", ourVersion,
+			"-t", ourBase,
+			"-u", "http://loinc.org/cm/loinc-to-phenx",
+			"-i", "http://loinc.org",
+			"-o", "http://phenxtoolkit.org",
+			"-f", file,
+			"-l"});
+
+		response = ourClient
+			.search()
+			.forResource(ConceptMap.class)
+			.where(ConceptMap.URL.matches().value("http://loinc.org/cm/loinc-to-phenx"))
+			.returnBundle(Bundle.class)
+			.execute();
+
+		conceptMap = (ConceptMap) response.getEntryFirstRep().getResource();
+
+		assertEquals("http://localhost:" + ourPort + "/ConceptMap/1/_history/2", conceptMap.getId());
+	}
 }

hapi-fhir-cli/hapi-fhir-cli-api/src/test/resources/loinc-to-phenx.csv

@@ -0,0 +1,2 @@
+SOURCE_CODE_SYSTEM,SOURCE_CODE_SYSTEM_VERSION,TARGET_CODE_SYSTEM,TARGET_CODE_SYSTEM_VERSION,SOURCE_CODE,SOURCE_DISPLAY,TARGET_CODE,TARGET_DISPLAY,EQUIVALENCE,COMMENT
+http://loinc.org,,http://phenxtoolkit.org,,65191-9,"During the past 30 days, about how often did you feel restless or fidgety [Kessler 6 Distress]",PX121301010300,PX121301_Restless,equivalent,

src/changes/changes.xml

@@ -16,6 +16,10 @@
 				The JPA server did not correctly index Timing fields where the timing contained
 				a period but no individual events. This has been corrected.
 			</action>
+			<action type="fix" issue="1320">
+				The HAPI FHIR CLI import-csv-to-conceptmap command was not accounting for byte order marks in
+				CSV files (e.g. some Excel CSV files). This has been fixed.
+			</action>
 		</release>
 		<release version="3.8.0" date="2019-05-30" description="Hippo">
 			<action type="fix">