honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow uploading term deltas using CS resource (#1555) 

* Work on accepting codesystem reources for delta operations

* Ongoing work on term uploader

* Restore the ability to use CodeSystem resources for the delta

* Add tests

* Fix NPE

* Test fixes
This commit is contained in:
James Agnew 2019-10-22 17:11:39 -04:00 committed by GitHub
parent 464c6c5b45
commit 9b94e4e26d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 473 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/UploadTerminologyCommand.java
View File

@ -23,6 +23,7 @@ package ca.uhn.fhir.cli;
import ca.uhn.fhir.jpa.model.util.JpaConstants;
import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider;
import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
import ca.uhn.fhir.rest.api.EncodingEnum;
import ca.uhn.fhir.rest.client.api.IGenericClient;
import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException;
@ -39,6 +40,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CountingInputStream;
import org.hl7.fhir.instance.model.api.IBaseParameters;
import org.hl7.fhir.instance.model.api.ICompositeType;
import org.hl7.fhir.r4.model.CodeSystem;
import java.io.*;
import java.util.zip.ZipEntry;
@ -132,7 +134,7 @@ public class UploadTerminologyCommand extends BaseCommand {
for (String nextDataFile : theDatafile) {
try (FileInputStream fileInputStream = new FileInputStream(nextDataFile)) {
if (!nextDataFile.endsWith(".zip")) {
if (nextDataFile.endsWith(".csv")) {
ourLog.info("Compressing and adding file: {}", nextDataFile);
ZipEntry nextEntry = new ZipEntry(stripPath(nextDataFile));
@ -146,12 +148,29 @@ public class UploadTerminologyCommand extends BaseCommand {
zipOutputStream.flush();
ourLog.info("Finished compressing {}", nextDataFile);
} else {
} else if (nextDataFile.endsWith(".zip")) {
ourLog.info("Adding file: {}", nextDataFile);
ourLog.info("Adding ZIP file: {}", nextDataFile);
String fileName = "file:" + nextDataFile;
addFileToRequestBundle(theInputParameters, fileName, IOUtils.toByteArray(fileInputStream));
} else if (nextDataFile.endsWith(".json") || nextDataFile.endsWith(".xml")) {
ourLog.info("Adding CodeSystem resource file: {}", nextDataFile);
String contents = IOUtils.toString(fileInputStream, Charsets.UTF_8);
EncodingEnum encoding = EncodingEnum.detectEncodingNoDefault(contents);
if (encoding == null) {
throw new ParseException("Could not detect FHIR encoding for file: " + nextDataFile);
}
CodeSystem resource = encoding.newParser(myFhirCtx).parseResource(CodeSystem.class, contents);
ParametersUtil.addParameterToParameters(myFhirCtx, theInputParameters, TerminologyUploaderProvider.PARAM_CODESYSTEM, resource);
} else {
throw new ParseException("Don't know how to handle file: " + nextDataFile);
}
}

175
hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/UploadTerminologyCommandTest.java
View File

@ -14,7 +14,9 @@ import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.hamcrest.Matchers;
import org.hl7.fhir.r4.model.CodeSystem;
import org.hl7.fhir.r4.model.IdType;
import org.hl7.fhir.r4.model.Patient;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -29,10 +31,10 @@ import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.matchesPattern;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.*;
@ -53,14 +55,18 @@ public class UploadTerminologyCommandTest extends BaseTest {
private int myPort;
private String myConceptsFileName = "target/concepts.csv";
private String myHierarchyFileName = "target/hierarchy.csv";
private File myConceptsFile = new File(myConceptsFileName);
private String myHierarchyFileName = "target/hierarchy.csv";
private File myHierarchyFile = new File(myHierarchyFileName);
private String myCodeSystemFileName = "target/codesystem.json";
private File myCodeSystemFile = new File(myCodeSystemFileName);
private String myTextFileName = "target/hello.txt";
private File myTextFile = new File(myTextFileName);
private File myArchiveFile;
private String myArchiveFileName;
@Test
public void testAddDelta() throws IOException {
public void testDeltaAdd() throws IOException {
writeConceptAndHierarchyFiles();
@ -85,7 +91,82 @@ public class UploadTerminologyCommandTest extends BaseTest {
}
@Test
public void testAddDeltaUsingCompressedFile() throws IOException {
public void testDeltaAddUsingCodeSystemResource() throws IOException {
try (FileWriter w = new FileWriter(myCodeSystemFile, false)) {
CodeSystem cs = new CodeSystem();
cs.addConcept().setCode("CODE").setDisplay("Display");
myCtx.newJsonParser().encodeResourceToWriter(cs, w);
}
when(myTermLoaderSvc.loadDeltaAdd(eq("http://foo"), anyList(), any())).thenReturn(new UploadStatistics(100, new IdType("CodeSystem/101")));
App.main(new String[]{
UploadTerminologyCommand.UPLOAD_TERMINOLOGY,
"-v", "r4",
"-m", "ADD",
"-t", "http://localhost:" + myPort,
"-u", "http://foo",
"-d", myCodeSystemFileName
});
verify(myTermLoaderSvc, times(1)).loadDeltaAdd(eq("http://foo"), myDescriptorListCaptor.capture(), any());
List<ITermLoaderSvc.FileDescriptor> listOfDescriptors = myDescriptorListCaptor.getValue();
assertEquals(1, listOfDescriptors.size());
assertEquals("concepts.csv", listOfDescriptors.get(0).getFilename());
String uploadFile = IOUtils.toString(listOfDescriptors.get(0).getInputStream(), Charsets.UTF_8);
assertThat(uploadFile, containsString("CODE,Display"));
}
@Test
public void testDeltaAddInvalidResource() throws IOException {
try (FileWriter w = new FileWriter(myCodeSystemFile, false)) {
Patient patient = new Patient();
patient.setActive(true);
myCtx.newJsonParser().encodeResourceToWriter(patient, w);
}
try {
App.main(new String[]{
UploadTerminologyCommand.UPLOAD_TERMINOLOGY,
"-v", "r4",
"-m", "ADD",
"-t", "http://localhost:" + myPort,
"-u", "http://foo",
"-d", myCodeSystemFileName
});
fail();
} catch (Error e) {
assertThat(e.toString(), containsString("Incorrect resource type found, expected \"CodeSystem\" but found \"Patient\""));
}
}
@Test
public void testDeltaAddInvalidFileType() throws IOException {
try (FileWriter w = new FileWriter(myTextFileName, false)) {
w.append("Help I'm a Bug");
}
try {
App.main(new String[]{
UploadTerminologyCommand.UPLOAD_TERMINOLOGY,
"-v", "r4",
"-m", "ADD",
"-t", "http://localhost:" + myPort,
"-u", "http://foo",
"-d", myTextFileName
});
fail();
} catch (Error e) {
assertThat(e.toString(), containsString("Don't know how to handle file:"));
}
}
@Test
public void testDeltaAddUsingCompressedFile() throws IOException {
writeConceptAndHierarchyFiles();
writeArchiveFile(myConceptsFile, myHierarchyFile);
@ -109,33 +190,28 @@ public class UploadTerminologyCommandTest extends BaseTest {
assertThat(IOUtils.toByteArray(listOfDescriptors.get(0).getInputStream()).length, greaterThan(100));
}
private void writeArchiveFile(File... theFiles) throws IOException {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
ZipOutputStream zipOutputStream = new ZipOutputStream(byteArrayOutputStream, Charsets.UTF_8);
@Test
public void testDeltaAddInvalidFileName() throws IOException {
for (File next : theFiles) {
ZipEntry nextEntry = new ZipEntry(UploadTerminologyCommand.stripPath(next.getAbsolutePath()));
zipOutputStream.putNextEntry(nextEntry);
writeConceptAndHierarchyFiles();
try (FileInputStream fileInputStream = new FileInputStream(next)) {
IOUtils.copy(fileInputStream, zipOutputStream);
}
}
zipOutputStream.flush();
zipOutputStream.close();
myArchiveFile = File.createTempFile("temp", ".zip");
myArchiveFile.deleteOnExit();
myArchiveFileName = myArchiveFile.getAbsolutePath();
try (FileOutputStream fos = new FileOutputStream(myArchiveFile, false)) {
fos.write(byteArrayOutputStream.toByteArray());
try {
App.main(new String[]{
UploadTerminologyCommand.UPLOAD_TERMINOLOGY,
"-v", "r4",
"-m", "ADD",
"-t", "http://localhost:" + myPort,
"-u", "http://foo",
"-d", myConceptsFileName + "/foo.csv",
"-d", myHierarchyFileName
});
} catch (Error e) {
assertThat(e.toString(), Matchers.containsString("FileNotFoundException: target/concepts.csv/foo.csv"));
}
}
@Test
public void testRemoveDelta() throws IOException {
public void testDeltaRemove() throws IOException {
writeConceptAndHierarchyFiles();
when(myTermLoaderSvc.loadDeltaRemove(eq("http://foo"), anyList(), any())).thenReturn(new UploadStatistics(100, new IdType("CodeSystem/101")));
@ -215,6 +291,31 @@ public class UploadTerminologyCommandTest extends BaseTest {
assertThat(IOUtils.toByteArray(listOfDescriptors.get(0).getInputStream()).length, greaterThan(100));
}
private void writeArchiveFile(File... theFiles) throws IOException {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
ZipOutputStream zipOutputStream = new ZipOutputStream(byteArrayOutputStream, Charsets.UTF_8);
for (File next : theFiles) {
ZipEntry nextEntry = new ZipEntry(UploadTerminologyCommand.stripPath(next.getAbsolutePath()));
zipOutputStream.putNextEntry(nextEntry);
try (FileInputStream fileInputStream = new FileInputStream(next)) {
IOUtils.copy(fileInputStream, zipOutputStream);
}
}
zipOutputStream.flush();
zipOutputStream.close();
myArchiveFile = File.createTempFile("temp", ".zip");
myArchiveFile.deleteOnExit();
myArchiveFileName = myArchiveFile.getAbsolutePath();
try (FileOutputStream fos = new FileOutputStream(myArchiveFile, false)) {
fos.write(byteArrayOutputStream.toByteArray());
}
}
private void writeConceptAndHierarchyFiles() throws IOException {
try (FileWriter w = new FileWriter(myConceptsFile, false)) {
@ -231,26 +332,6 @@ public class UploadTerminologyCommandTest extends BaseTest {
}
}
@Test
public void testAddInvalidFileName() throws IOException {
writeConceptAndHierarchyFiles();
try {
App.main(new String[]{
UploadTerminologyCommand.UPLOAD_TERMINOLOGY,
"-v", "r4",
"-m", "ADD",
"-t", "http://localhost:" + myPort,
"-u", "http://foo",
"-d", myConceptsFileName + "/foo.csv",
"-d", myHierarchyFileName
});
} catch (Error e) {
assertThat(e.toString(), Matchers.containsString("FileNotFoundException: target/concepts.csv/foo.csv"));
}
}
@After
public void after() throws Exception {
@ -259,6 +340,8 @@ public class UploadTerminologyCommandTest extends BaseTest {
FileUtils.deleteQuietly(myConceptsFile);
FileUtils.deleteQuietly(myHierarchyFile);
FileUtils.deleteQuietly(myArchiveFile);
FileUtils.deleteQuietly(myCodeSystemFile);
FileUtils.deleteQuietly(myTextFile);
UploadTerminologyCommand.setTransferSizeLimitForUnitTest(-1);
}

1
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaProvider.java
View File

@ -47,6 +47,7 @@ public class BaseJpaProvider {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseJpaProvider.class);
@Autowired
protected DaoConfig myDaoConfig;
@Autowired
private FhirContext myContext;
public BaseJpaProvider() {

173
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/TerminologyUploaderProvider.java
View File

@ -21,9 +21,13 @@ package ca.uhn.fhir.jpa.provider;
*/
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.jpa.model.util.JpaConstants;
import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl;
import ca.uhn.fhir.jpa.term.UploadStatistics;
import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
import ca.uhn.fhir.jpa.term.custom.ConceptHandler;
import ca.uhn.fhir.jpa.term.custom.HierarchyHandler;
import ca.uhn.fhir.rest.annotation.Operation;
import ca.uhn.fhir.rest.annotation.OperationParam;
import ca.uhn.fhir.rest.api.server.RequestDetails;
@ -32,9 +36,15 @@ import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.util.AttachmentUtil;
import ca.uhn.fhir.util.ParametersUtil;
import ca.uhn.fhir.util.ValidateUtil;
import com.google.common.base.Charsets;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import org.hl7.fhir.convertors.VersionConvertor_30_40;
import org.hl7.fhir.instance.model.api.IBaseParameters;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.ICompositeType;
import org.hl7.fhir.instance.model.api.IPrimitiveType;
import org.hl7.fhir.r4.model.CodeSystem;
import org.springframework.beans.factory.annotation.Autowired;
import javax.annotation.Nonnull;
@ -43,22 +53,20 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.*;
import static org.apache.commons.lang3.StringUtils.*;
public class TerminologyUploaderProvider extends BaseJpaProvider {
public static final String PARAM_FILE = "file";
public static final String PARAM_CODESYSTEM = "codeSystem";
public static final String PARAM_SYSTEM = "system";
private static final String RESP_PARAM_CONCEPT_COUNT = "conceptCount";
private static final String RESP_PARAM_TARGET = "target";
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyUploaderProvider.class);
private static final String RESP_PARAM_SUCCESS = "success";
@Autowired
private FhirContext myCtx;
@Autowired
private ITermLoaderSvc myTerminologyLoaderSvc;
@ -73,7 +81,7 @@ public class TerminologyUploaderProvider extends BaseJpaProvider {
* Constructor
*/
public TerminologyUploaderProvider(FhirContext theContext, ITermLoaderSvc theTerminologyLoaderSvc) {
myCtx = theContext;
setContext(theContext);
myTerminologyLoaderSvc = theTerminologyLoaderSvc;
}
@ -102,8 +110,8 @@ public class TerminologyUploaderProvider extends BaseJpaProvider {
throw new InvalidRequestException("No '" + PARAM_FILE + "' parameter, or package had no data");
}
for (ICompositeType next : theFiles) {
ValidateUtil.isTrueOrThrowInvalidRequest(myCtx.getElementDefinition(next.getClass()).getName().equals("Attachment"), "Package must be of type Attachment");
}
ValidateUtil.isTrueOrThrowInvalidRequest(getContext().getElementDefinition(next.getClass()).getName().equals("Attachment"), "Package must be of type Attachment");
}
try {
List<ITermLoaderSvc.FileDescriptor> localFiles = convertAttachmentsToFileDescriptors(theFiles);
@ -127,10 +135,10 @@ public class TerminologyUploaderProvider extends BaseJpaProvider {
break;
}
IBaseParameters retVal = ParametersUtil.newInstance(myCtx);
ParametersUtil.addParameterToParametersBoolean(myCtx, retVal, RESP_PARAM_SUCCESS, true);
ParametersUtil.addParameterToParametersInteger(myCtx, retVal, RESP_PARAM_CONCEPT_COUNT, stats.getUpdatedConceptCount());
ParametersUtil.addParameterToParametersReference(myCtx, retVal, RESP_PARAM_TARGET, stats.getTarget().getValue());
IBaseParameters retVal = ParametersUtil.newInstance(getContext());
ParametersUtil.addParameterToParametersBoolean(getContext(), retVal, RESP_PARAM_SUCCESS, true);
ParametersUtil.addParameterToParametersInteger(getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, stats.getUpdatedConceptCount());
ParametersUtil.addParameterToParametersReference(getContext(), retVal, RESP_PARAM_TARGET, stats.getTarget().getValue());
return retVal;
} finally {
@ -149,15 +157,17 @@ public class TerminologyUploaderProvider extends BaseJpaProvider {
HttpServletRequest theServletRequest,
@OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem,
@OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles,
@OperationParam(name = PARAM_CODESYSTEM, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "CodeSystem") List<IBaseResource> theCodeSystems,
RequestDetails theRequestDetails
) {
startRequest(theServletRequest);
try {
validateHaveSystem(theSystem);
validateHaveFiles(theFiles);
validateHaveFiles(theFiles, theCodeSystems);
List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles);
convertCodeSystemsToFileDescriptors(files, theCodeSystems);
UploadStatistics outcome = myTerminologyLoaderSvc.loadDeltaAdd(theSystem.getValue(), files, theRequestDetails);
return toDeltaResponse(outcome);
} finally {
@ -178,15 +188,17 @@ public class TerminologyUploaderProvider extends BaseJpaProvider {
HttpServletRequest theServletRequest,
@OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem,
@OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles,
@OperationParam(name = PARAM_CODESYSTEM, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "CodeSystem") List<IBaseResource> theCodeSystems,
RequestDetails theRequestDetails
) {
startRequest(theServletRequest);
try {
validateHaveSystem(theSystem);
validateHaveFiles(theFiles);
validateHaveFiles(theFiles, theCodeSystems);
List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles);
convertCodeSystemsToFileDescriptors(files, theCodeSystems);
UploadStatistics outcome = myTerminologyLoaderSvc.loadDeltaRemove(theSystem.getValue(), files, theRequestDetails);
return toDeltaResponse(outcome);
} finally {
@ -195,13 +207,96 @@ public class TerminologyUploaderProvider extends BaseJpaProvider {
}
private void convertCodeSystemsToFileDescriptors(List<ITermLoaderSvc.FileDescriptor> theFiles, List<IBaseResource> theCodeSystems) {
Map<String, String> codes = new LinkedHashMap<>();
Multimap<String, String> codeToParentCodes = ArrayListMultimap.create();
if (theCodeSystems != null) {
for (IBaseResource nextCodeSystemUncast : theCodeSystems) {
CodeSystem nextCodeSystem = canonicalizeCodeSystem(nextCodeSystemUncast);
convertCodeSystemCodesToCsv(nextCodeSystem.getConcept(), codes, null, codeToParentCodes);
}
}
// Create concept file
if (codes.size() > 0) {
StringBuilder b = new StringBuilder();
b.append(ConceptHandler.CODE);
b.append(",");
b.append(ConceptHandler.DISPLAY);
b.append("\n");
for (Map.Entry<String, String> nextEntry : codes.entrySet()) {
b.append(nextEntry.getKey());
b.append(",");
b.append(defaultString(nextEntry.getValue()));
b.append("\n");
}
byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
String fileName = TermLoaderSvcImpl.CUSTOM_CONCEPTS_FILE;
ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor = new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
theFiles.add(fileDescriptor);
}
// Create hierarchy file
if (codeToParentCodes.size() > 0) {
StringBuilder b = new StringBuilder();
b.append(HierarchyHandler.CHILD);
b.append(",");
b.append(HierarchyHandler.PARENT);
b.append("\n");
for (Map.Entry<String, String> nextEntry : codeToParentCodes.entries()) {
b.append(nextEntry.getKey());
b.append(",");
b.append(defaultString(nextEntry.getValue()));
b.append("\n");
}
byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
String fileName = TermLoaderSvcImpl.CUSTOM_HIERARCHY_FILE;
ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor = new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
theFiles.add(fileDescriptor);
}
}
@SuppressWarnings("EnumSwitchStatementWhichMissesCases")
@Nonnull
CodeSystem canonicalizeCodeSystem(@Nonnull IBaseResource theCodeSystem) {
RuntimeResourceDefinition resourceDef = getContext().getResourceDefinition(theCodeSystem);
ValidateUtil.isTrueOrThrowInvalidRequest(resourceDef.getName().equals("CodeSystem"), "Resource '%s' is not a CodeSystem", resourceDef.getName());
CodeSystem nextCodeSystem;
switch (getContext().getVersion().getVersion()) {
case DSTU3:
nextCodeSystem = VersionConvertor_30_40.convertCodeSystem((org.hl7.fhir.dstu3.model.CodeSystem) theCodeSystem);
break;
case R5:
nextCodeSystem = org.hl7.fhir.convertors.conv40_50.CodeSystem.convertCodeSystem((org.hl7.fhir.r5.model.CodeSystem) theCodeSystem);
break;
default:
nextCodeSystem = (CodeSystem) theCodeSystem;
}
return nextCodeSystem;
}
private void convertCodeSystemCodesToCsv(List<CodeSystem.ConceptDefinitionComponent> theConcept, Map<String, String> theCodes, String theParentCode, Multimap<String, String> theCodeToParentCodes) {
for (CodeSystem.ConceptDefinitionComponent nextConcept : theConcept) {
if (isNotBlank(nextConcept.getCode())) {
theCodes.put(nextConcept.getCode(), nextConcept.getDisplay());
if (isNotBlank(theParentCode)) {
theCodeToParentCodes.put(nextConcept.getCode(), theParentCode);
}
convertCodeSystemCodesToCsv(nextConcept.getConcept(), theCodes, nextConcept.getCode(), theCodeToParentCodes);
}
}
}
private void validateHaveSystem(IPrimitiveType<String> theSystem) {
if (theSystem == null || isBlank(theSystem.getValueAsString())) {
throw new InvalidRequestException("Missing mandatory parameter: " + PARAM_SYSTEM);
}
}
private void validateHaveFiles(List<ICompositeType> theFiles) {
private void validateHaveFiles(List<ICompositeType> theFiles, List<IBaseResource> theCodeSystems) {
if (theFiles != null) {
for (ICompositeType nextFile : theFiles) {
if (!nextFile.isEmpty()) {
@ -209,45 +304,53 @@ public class TerminologyUploaderProvider extends BaseJpaProvider {
}
}
}
if (theCodeSystems != null) {
for (IBaseResource next : theCodeSystems) {
if (!next.isEmpty()) {
return;
}
}
}
throw new InvalidRequestException("Missing mandatory parameter: " + PARAM_FILE);
}
@Nonnull
private List<ITermLoaderSvc.FileDescriptor> convertAttachmentsToFileDescriptors(@OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles) {
List<ITermLoaderSvc.FileDescriptor> files = new ArrayList<>();
for (ICompositeType next : theFiles) {
if (theFiles != null) {
for (ICompositeType next : theFiles) {
String nextUrl = AttachmentUtil.getOrCreateUrl(myCtx, next).getValue();
ValidateUtil.isNotBlankOrThrowUnprocessableEntity(nextUrl, "Missing Attachment.url value");
String nextUrl = AttachmentUtil.getOrCreateUrl(getContext(), next).getValue();
ValidateUtil.isNotBlankOrThrowUnprocessableEntity(nextUrl, "Missing Attachment.url value");
byte[] nextData;
if (nextUrl.startsWith("localfile:")) {
String nextLocalFile = nextUrl.substring("localfile:".length());
byte[] nextData;
if (nextUrl.startsWith("localfile:")) {
String nextLocalFile = nextUrl.substring("localfile:".length());
if (isNotBlank(nextLocalFile)) {
ourLog.info("Reading in local file: {}", nextLocalFile);
File nextFile = new File(nextLocalFile);
if (!nextFile.exists() || !nextFile.isFile()) {
throw new InvalidRequestException("Unknown file: " + nextFile.getName());
if (isNotBlank(nextLocalFile)) {
ourLog.info("Reading in local file: {}", nextLocalFile);
File nextFile = new File(nextLocalFile);
if (!nextFile.exists() || !nextFile.isFile()) {
throw new InvalidRequestException("Unknown file: " + nextFile.getName());
}
files.add(new FileBackedFileDescriptor(nextFile));
}
files.add(new FileBackedFileDescriptor(nextFile));
} else {
nextData = AttachmentUtil.getOrCreateData(getContext(), next).getValue();
ValidateUtil.isTrueOrThrowInvalidRequest(nextData != null && nextData.length > 0, "Missing Attachment.data value");
files.add(new ITermLoaderSvc.ByteArrayFileDescriptor(nextUrl, nextData));
}
} else {
nextData = AttachmentUtil.getOrCreateData(myCtx, next).getValue();
ValidateUtil.isTrueOrThrowInvalidRequest(nextData != null && nextData.length > 0, "Missing Attachment.data value");
files.add(new ITermLoaderSvc.ByteArrayFileDescriptor(nextUrl, nextData));
}
}
return files;
}
private IBaseParameters toDeltaResponse(UploadStatistics theOutcome) {
IBaseParameters retVal = ParametersUtil.newInstance(myCtx);
ParametersUtil.addParameterToParametersInteger(myCtx, retVal, RESP_PARAM_CONCEPT_COUNT, theOutcome.getUpdatedConceptCount());
ParametersUtil.addParameterToParametersReference(myCtx, retVal, RESP_PARAM_TARGET, theOutcome.getTarget().getValue());
IBaseParameters retVal = ParametersUtil.newInstance(getContext());
ParametersUtil.addParameterToParametersInteger(getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, theOutcome.getUpdatedConceptCount());
ParametersUtil.addParameterToParametersReference(getContext(), retVal, RESP_PARAM_TARGET, theOutcome.getTarget().getValue());
return retVal;
}

19
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/custom/CustomTerminologySet.java
View File

@ -34,6 +34,7 @@ import org.apache.commons.lang3.Validate;
import javax.annotation.Nonnull;
import java.util.*;
import java.util.stream.Collectors;
public class CustomTerminologySet {
@ -163,12 +164,28 @@ public class CustomTerminologySet {
TermLoaderSvcImpl.iterateOverZipFile(theDescriptors, TermLoaderSvcImpl.CUSTOM_HIERARCHY_FILE, hierarchyHandler, ',', QuoteMode.NON_NUMERIC, false);
}
// Find root concepts
Map<String, Integer> codesInOrder = new HashMap<>();
for (String nextCode : code2concept.keySet()) {
codesInOrder.put(nextCode, codesInOrder.size());
}
List<TermConcept> rootConcepts = new ArrayList<>();
for (TermConcept nextConcept : code2concept.values()) {
// Find root concepts
if (nextConcept.getParents().isEmpty()) {
rootConcepts.add(nextConcept);
}
// Sort children so they appear in the same order as they did in the concepts.csv file
nextConcept.getChildren().sort((o1,o2)->{
String code1 = o1.getChild().getCode();
String code2 = o2.getChild().getCode();
int order1 = codesInOrder.get(code1);
int order2 = codesInOrder.get(code2);
return order1 - order2;
});
}
return new CustomTerminologySet(code2concept.size(), unanchoredChildConceptsToParentCodes, rootConcepts);

6
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/custom/HierarchyHandler.java
View File

@ -34,6 +34,8 @@ import static org.apache.commons.lang3.StringUtils.trim;
public class HierarchyHandler implements IRecordHandler {
public static final String PARENT = "PARENT";
public static final String CHILD = "CHILD";
private final Map<String, TermConcept> myCode2Concept;
private final ArrayListMultimap<TermConcept, String> myUnanchoredChildConceptsToParentCodes;
@ -44,8 +46,8 @@ public class HierarchyHandler implements IRecordHandler {
@Override
public void accept(CSVRecord theRecord) {
String parent = trim(theRecord.get("PARENT"));
String child = trim(theRecord.get("CHILD"));
String parent = trim(theRecord.get(PARENT));
String child = trim(theRecord.get(CHILD));
if (isNotBlank(parent) && isNotBlank(child)) {
TermConcept childConcept = myCode2Concept.get(child);

41
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java
View File

@ -9,6 +9,9 @@ import ca.uhn.fhir.jpa.config.TestR4Config;
import ca.uhn.fhir.jpa.dao.*;
import ca.uhn.fhir.jpa.dao.data.*;
import ca.uhn.fhir.jpa.dao.dstu2.FhirResourceDaoDstu2SearchNoFtTest;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.interceptor.PerformanceTracingLoggingInterceptor;
import ca.uhn.fhir.jpa.model.entity.ModelConfig;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString;
@ -66,9 +69,14 @@ import org.springframework.transaction.annotation.Transactional;
import javax.persistence.EntityManager;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
@ -452,6 +460,39 @@ public abstract class BaseJpaR4Test extends BaseJpaTest {
}
protected void assertHierarchyContains(String... theStrings) {
List<String> hierarchy = runInTransaction(() -> {
List<String> hierarchyHolder = new ArrayList<>();
TermCodeSystem codeSystem = myTermCodeSystemDao.findAll().iterator().next();
TermCodeSystemVersion csv = codeSystem.getCurrentVersion();
List<TermConcept> codes = myTermConceptDao.findByCodeSystemVersion(csv);
List<TermConcept> rootCodes = codes.stream().filter(t -> t.getParents().isEmpty()).collect(Collectors.toList());
flattenExpansionHierarchy(hierarchyHolder, rootCodes, "");
return hierarchyHolder;
});
if (theStrings.length == 0) {
assertThat("\n" + String.join("\n", hierarchy), hierarchy, empty());
} else {
assertThat("\n" + String.join("\n", hierarchy), hierarchy, contains(theStrings));
}
}
private static void flattenExpansionHierarchy(List<String> theFlattenedHierarchy, List<TermConcept> theCodes, String thePrefix) {
theCodes.sort((o1, o2) -> {
int s1 = o1.getSequence() != null ? o1.getSequence() : o1.getCode().hashCode();
int s2 = o2.getSequence() != null ? o2.getSequence() : o2.getCode().hashCode();
return s1 - s2;
});
for (TermConcept nextCode : theCodes) {
String hierarchyEntry = thePrefix + nextCode.getCode() + " seq=" + nextCode.getSequence();
theFlattenedHierarchy.add(hierarchyEntry);
List<TermConcept> children = nextCode.getChildCodes();
flattenExpansionHierarchy(theFlattenedHierarchy, children, thePrefix + " ");
}
}
@AfterClass
public static void afterClassClearContextBaseJpaR4Test() {
ourValueSetDao.purgeCaches();

66
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/TerminologyUploaderProviderTest.java Normal file
View File

@ -0,0 +1,66 @@
package ca.uhn.fhir.jpa.provider;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.test.BaseTest;
import org.hl7.fhir.r4.model.CodeSystem;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class TerminologyUploaderProviderTest extends BaseTest {
@Test
public void testCanonicalizeR3() {
TerminologyUploaderProvider provider = new TerminologyUploaderProvider();
provider.setContext(FhirContext.forDstu3());
org.hl7.fhir.dstu3.model.CodeSystem input = new org.hl7.fhir.dstu3.model.CodeSystem();
input.addConcept().setCode("FOO").setDisplay("Foo");
CodeSystem canonical = provider.canonicalizeCodeSystem(input);
assertEquals("FOO", canonical.getConcept().get(0).getCode());
}
@Test
public void testCanonicalizeR4() {
TerminologyUploaderProvider provider = new TerminologyUploaderProvider();
provider.setContext(FhirContext.forR4());
org.hl7.fhir.r4.model.CodeSystem input = new org.hl7.fhir.r4.model.CodeSystem();
input.addConcept().setCode("FOO").setDisplay("Foo");
CodeSystem canonical = provider.canonicalizeCodeSystem(input);
assertEquals("FOO", canonical.getConcept().get(0).getCode());
}
@Test
public void testCanonicalizeR5() {
TerminologyUploaderProvider provider = new TerminologyUploaderProvider();
provider.setContext(FhirContext.forR5());
org.hl7.fhir.r5.model.CodeSystem input = new org.hl7.fhir.r5.model.CodeSystem();
input.addConcept().setCode("FOO").setDisplay("Foo");
CodeSystem canonical = provider.canonicalizeCodeSystem(input);
assertEquals("FOO", canonical.getConcept().get(0).getCode());
}
@Test
public void testCanonicalizeR5_WrongType() {
TerminologyUploaderProvider provider = new TerminologyUploaderProvider();
provider.setContext(FhirContext.forR5());
org.hl7.fhir.r5.model.Patient input = new org.hl7.fhir.r5.model.Patient();
try {
provider.canonicalizeCodeSystem(input);
} catch (InvalidRequestException e) {
assertEquals("Resource 'Patient' is not a CodeSystem", e.getMessage());
}
}
}

48
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/TerminologyUploaderProviderR4Test.java
View File

@ -11,7 +11,10 @@ import org.apache.commons.io.IOUtils;
import org.hl7.fhir.r4.model.*;
import org.junit.AfterClass;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import javax.lang.model.util.Types;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
@ -25,12 +28,13 @@ import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.*;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
public class TerminologyUploaderProviderR4Test extends BaseResourceProviderR4Test {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyUploaderProviderR4Test.class);
private byte[] createSctZip() throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ZipOutputStream zos = new ZipOutputStream(bos);
@ -173,7 +177,7 @@ public class TerminologyUploaderProviderR4Test extends BaseResourceProviderR4Tes
}
@Test
public void testApplyDeltaAdd() throws IOException {
public void testApplyDeltaAdd_UsingCsv() throws IOException {
String conceptsCsv = loadResource("/custom_term/concepts.csv");
Attachment conceptsAttachment = new Attachment()
.setData(conceptsCsv.getBytes(Charsets.UTF_8))
@ -208,6 +212,46 @@ public class TerminologyUploaderProviderR4Test extends BaseResourceProviderR4Tes
));
}
@Test
public void testApplyDeltaAdd_UsingCodeSystem() {
CodeSystem codeSystem = new CodeSystem();
codeSystem.setUrl("http://foo/cs");
CodeSystem.ConceptDefinitionComponent chem = codeSystem.addConcept().setCode("CHEM").setDisplay("Chemistry");
chem.addConcept().setCode("HB").setDisplay("Hemoglobin");
chem.addConcept().setCode("NEUT").setDisplay("Neutrophils");
CodeSystem.ConceptDefinitionComponent micro = codeSystem.addConcept().setCode("MICRO").setDisplay("Microbiology");
micro.addConcept().setCode("C&S").setDisplay("Culture And Sensitivity");
LoggingInterceptor interceptor = new LoggingInterceptor(true);
ourClient.registerInterceptor(interceptor);
Parameters outcome = ourClient
.operation()
.onType(CodeSystem.class)
.named(JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_ADD)
.withParameter(Parameters.class, TerminologyUploaderProvider.PARAM_SYSTEM, new UriType("http://foo/cs"))
.andParameter(TerminologyUploaderProvider.PARAM_CODESYSTEM, codeSystem)
.prettyPrint()
.execute();
ourClient.unregisterInterceptor(interceptor);
String encoded = myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome);
ourLog.info(encoded);
assertThat(encoded, stringContainsInOrder(
"\"name\": \"conceptCount\"",
"\"valueInteger\": 5",
"\"name\": \"target\"",
"\"reference\": \"CodeSystem/"
));
assertHierarchyContains(
"CHEM seq=1",
" HB seq=1",
" NEUT seq=2",
"MICRO seq=2",
" C&S seq=1"
);
}
@Test
public void testApplyDeltaAdd_MissingSystem() throws IOException {
String conceptsCsv = loadResource("/custom_term/concepts.csv");

31
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcDeltaR4Test.java
View File

@ -434,38 +434,7 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
}
private void assertHierarchyContains(String... theStrings) {
List<String> hierarchy = runInTransaction(() -> {
List<String> hierarchyHolder = new ArrayList<>();
TermCodeSystem codeSystem = myTermCodeSystemDao.findAll().iterator().next();
TermCodeSystemVersion csv = codeSystem.getCurrentVersion();
List<TermConcept> codes = myTermConceptDao.findByCodeSystemVersion(csv);
List<TermConcept> rootCodes = codes.stream().filter(t -> t.getParents().isEmpty()).collect(Collectors.toList());
flattenExpansionHierarchy(hierarchyHolder, rootCodes, "");
return hierarchyHolder;
});
if (theStrings.length == 0) {
assertThat("\n" + String.join("\n", hierarchy), hierarchy, empty());
} else {
assertThat("\n" + String.join("\n", hierarchy), hierarchy, contains(theStrings));
}
}
private void flattenExpansionHierarchy(List<String> theFlattenedHierarchy, List<TermConcept> theCodes, String thePrefix) {
theCodes.sort((o1, o2) -> {
int s1 = o1.getSequence() != null ? o1.getSequence() : o1.getCode().hashCode();
int s2 = o2.getSequence() != null ? o2.getSequence() : o2.getCode().hashCode();
return s1 - s2;
});
for (TermConcept nextCode : theCodes) {
String hierarchyEntry = thePrefix + nextCode.getCode() + " seq=" + nextCode.getSequence();
theFlattenedHierarchy.add(hierarchyEntry);
List<TermConcept> children = nextCode.getChildCodes();
flattenExpansionHierarchy(theFlattenedHierarchy, children, thePrefix + " ");
}
}
private ValueSet expandNotPresentCodeSystem() {
ValueSet vs = new ValueSet();

4
hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/method/MethodUtil.java
View File

@ -227,7 +227,11 @@ public class MethodUtil {
param = new OperationParameter(theContext, op.name(), operationParam);
if (isNotBlank(operationParam.typeName())) {
BaseRuntimeElementDefinition<?> elementDefinition = theContext.getElementDefinition(operationParam.typeName());
if (elementDefinition == null) {
elementDefinition = theContext.getResourceDefinition(operationParam.typeName());
}
org.apache.commons.lang3.Validate.notNull(elementDefinition, "Unknown type name in @OperationParam: typeName=\"%s\"", operationParam.typeName());
Class<?> newParameterType = elementDefinition.getImplementingClass();
if (!declaredParameterType.isAssignableFrom(newParameterType)) {
throw new ConfigurationException("Non assignable parameter typeName=\"" + operationParam.typeName() + "\" specified on method " + theMethod);

5
src/changes/changes.xml
View File

@ -60,7 +60,10 @@
A new set of operations have been added to the JPA server that allow CodeSystem deltas to be
uploaded. A CodeSystem Delta consists of a set of codes and relationships that are added or
removed incrementally to the live CodeSystem without requiring a downtime or a complete
upload of the contents. In addition, the HAPI FHIR CLI
upload of the contents. Deltas may be specified using either a custom CSV format or a partial
CodeSystem resource.
<br/>
In addition, the HAPI FHIR CLI
<code>upload-terminology</code> command has been modified to support this new functionality.
]]>
</action>