Upload Terminology with Endpoint Validation (#4035)

* add failing test

* Fix doc upload issue

* Add change log yaml file

* update due to review from Nathan

Co-authored-by: nathaniel.doef <nathaniel.doef@smilecdr.com>
Co-authored-by: weiping202209 <weiping.yang@smilecdr.com>

hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/UploadTerminologyCommand.java

@@ -38,6 +38,7 @@ import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.input.CountingInputStream;
+import org.apache.commons.lang3.StringUtils;
 import org.hl7.fhir.instance.model.api.IBaseParameters;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.ICompositeType;
@@ -242,10 +243,11 @@ public class UploadTerminologyCommand extends BaseRequestGeneratingCommand {
 
 		byte[] bytes = theBytes;
 		String fileName = theFileName;
+		String suffix = fileName.substring(fileName.lastIndexOf("."));
 
 		if (bytes.length > ourTransferSizeLimit) {
 			ourLog.info("File size is greater than {} - Going to use a local file reference instead of a direct HTTP transfer. Note that this will only work when executing this command on the same server as the FHIR server itself.", FileUtil.formatFileSize(ourTransferSizeLimit));
-			String suffix = fileName.substring(fileName.lastIndexOf("."));
+
 			try {
 				File tempFile = File.createTempFile("hapi-fhir-cli", suffix);
 				tempFile.deleteOnExit();
@@ -260,6 +262,7 @@ public class UploadTerminologyCommand extends BaseRequestGeneratingCommand {
 		}
 
 		ICompositeType attachment = AttachmentUtil.newInstance(myFhirCtx);
+		AttachmentUtil.setContentType(myFhirCtx, attachment, getContentType(suffix));
 		AttachmentUtil.setUrl(myFhirCtx, attachment, fileName);
 		if (bytes != null) {
 			AttachmentUtil.setData(myFhirCtx, attachment, bytes);
@@ -267,6 +270,25 @@ public class UploadTerminologyCommand extends BaseRequestGeneratingCommand {
 		ParametersUtil.addParameterToParameters(myFhirCtx, theInputParameters, TerminologyUploaderProvider.PARAM_FILE, attachment);
 	}
 
+	/*
+	 * Files may be included in the attachment as raw CSV/JSON/XML files, or may also be combined into a compressed ZIP file.
+	 * Content Type reference: https://smilecdr.com/docs/terminology/uploading.html#delta-add-operation
+	 */
+	private String getContentType(String theSuffix) {
+		String retVal = "";
+		if(StringUtils.isNotBlank(theSuffix)) {
+			switch (theSuffix.toLowerCase()) {
+				case "csv" : retVal = "text/csv"; break;
+				case "xml" : retVal = "application/xml"; break;
+				case "json" : retVal = "application/json"; break;
+				case "zip" : retVal = "application/zip"; break;
+				default: retVal = "text/plain";
+			}
+		}
+		ourLog.debug("File suffix given was {} and contentType is {}, defaulting to content type text/plain", theSuffix, retVal);
+		return retVal;
+	}
+
 	private enum ModeEnum {
 		SNAPSHOT, ADD, REMOVE
 	}

hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/UploadTerminologyCommandTest.java

@@ -1,19 +1,26 @@
 package ca.uhn.fhir.cli;
 
 import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.context.support.DefaultProfileValidationSupport;
 import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider;
 import ca.uhn.fhir.jpa.term.UploadStatistics;
 import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
-import ca.uhn.fhir.test.utilities.TlsAuthenticationTestHelper;
+import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor;
 import ca.uhn.fhir.test.utilities.BaseRestServerHelper;
 import ca.uhn.fhir.test.utilities.RestServerDstu3Helper;
 import ca.uhn.fhir.test.utilities.RestServerR4Helper;
+import ca.uhn.fhir.test.utilities.TlsAuthenticationTestHelper;
+import ca.uhn.fhir.validation.FhirValidator;
 import com.google.common.base.Charsets;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.hamcrest.Matchers;
+import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService;
+import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport;
+import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain;
+import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -460,6 +467,19 @@ public class UploadTerminologyCommandTest {
 	@ParameterizedTest
 	@MethodSource("paramsProvider")
 	public void testUploadICD10UsingCompressedFile(String theFhirVersion, boolean theIncludeTls) throws IOException {
+		uploadICD10UsingCompressedFile(theFhirVersion, theIncludeTls);
+	}
+
+	@ParameterizedTest
+	@MethodSource("paramsProvider")
+	public void testUploadTerminologyWithEndpointValidation(String theFhirVersion, boolean theIncludeTls) throws IOException {
+		RequestValidatingInterceptor requestValidatingInterceptor = createRequestValidatingInterceptor();
+		myBaseRestServerHelper.registerInterceptor(requestValidatingInterceptor);
+
+		uploadICD10UsingCompressedFile(theFhirVersion, theIncludeTls);
+	}
+
+	private void uploadICD10UsingCompressedFile(String theFhirVersion, boolean theIncludeTls) throws IOException {
 		if (FHIR_VERSION_DSTU3.equals(theFhirVersion)) {
 			when(myTermLoaderSvc.loadIcd10cm(anyList(), any())).thenReturn(new UploadStatistics(100, new org.hl7.fhir.dstu3.model.IdType("CodeSystem/101")));
 		} else if (FHIR_VERSION_R4.equals(theFhirVersion)) {
@@ -486,6 +506,23 @@ public class UploadTerminologyCommandTest {
 		assertThat(IOUtils.toByteArray(listOfDescriptors.get(0).getInputStream()).length, greaterThan(100));
 	}
 
+	private RequestValidatingInterceptor createRequestValidatingInterceptor(){
+		FhirInstanceValidator fhirInstanceValidator = new FhirInstanceValidator(myCtx);
+		ValidationSupportChain validationSupport = new ValidationSupportChain(
+			new DefaultProfileValidationSupport(myCtx),
+			new InMemoryTerminologyServerValidationSupport(myCtx),
+			new CommonCodeSystemsTerminologyService(myCtx)
+		);
+
+		fhirInstanceValidator.setValidationSupport(validationSupport);
+		FhirValidator fhirValidator = myCtx.newValidator();
+		fhirValidator.registerValidatorModule(fhirInstanceValidator);
+
+		RequestValidatingInterceptor requestValidatingInterceptor = new RequestValidatingInterceptor();
+		requestValidatingInterceptor.setValidatorModules(List.of(fhirInstanceValidator));
+		return requestValidatingInterceptor;
+	}
+
 	private synchronized void writeConceptAndHierarchyFiles() throws IOException {
 		if (!myConceptsFile.exists()) {
 			try (FileWriter w = new FileWriter(myConceptsFile, false)) {

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4035-upload-terminology-failure-with-endpoint-validation.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 4091
+jira: SMILE-3977
+title: "Previously, when the upload-terminology command was used to upload a terminology file with endpoint validation enabled, a validation error occurred due to a missing file content type.
+ This has been fixed by specifying the file content type of the uploaded file."