Add current-version parameter to upload-terminology command.

This commit is contained in:
juan.marchionatto 2021-07-21 16:18:35 -04:00
parent 0a27b08aad
commit e0e52c666c
10 changed files with 1151 additions and 756 deletions

View File

@ -22,6 +22,7 @@ package ca.uhn.fhir.cli;
import ca.uhn.fhir.jpa.model.util.JpaConstants;
import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider;
import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl;
import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
import ca.uhn.fhir.rest.api.EncodingEnum;
import ca.uhn.fhir.rest.client.api.IGenericClient;
@ -38,6 +39,7 @@ import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CountingInputStream;
import org.apache.commons.lang3.StringUtils;
import org.hl7.fhir.instance.model.api.IBaseParameters;
import org.hl7.fhir.instance.model.api.ICompositeType;
import org.hl7.fhir.r4.model.CodeSystem;
@ -58,6 +60,7 @@ public class UploadTerminologyCommand extends BaseRequestGeneratingCommand {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(UploadTerminologyCommand.class);
private static final long DEFAULT_TRANSFER_SIZE_LIMIT = 10 * FileUtils.ONE_MB;
private static long ourTransferSizeLimit = DEFAULT_TRANSFER_SIZE_LIMIT;
private static final String CURRENT_VERSION = "current-version";
@Override
public String getCommandDescription() {
@ -76,6 +79,8 @@ public class UploadTerminologyCommand extends BaseRequestGeneratingCommand {
addRequiredOption(options, "u", "url", true, "The code system URL associated with this upload (e.g. " + ITermLoaderSvc.SCT_URI + ")");
addOptionalOption(options, "d", "data", true, "Local file to use to upload (can be a raw file or a ZIP containing the raw file)");
addOptionalOption(options, "m", "mode", true, "The upload mode: SNAPSHOT (default), ADD, REMOVE");
addOptionalOption(options, null, CURRENT_VERSION, true, "For LOINC upload. If specified as 'false' the uploaded version will not become current. " +
"Allowed values: true (default) or false");
return options;
}
@ -102,6 +107,8 @@ public class UploadTerminologyCommand extends BaseRequestGeneratingCommand {
throw new ParseException("No data file provided");
}
String makeCurrentVersionOption = theCommandLine.getOptionValue(CURRENT_VERSION);
IGenericClient client = newClient(theCommandLine);
if (theCommandLine.hasOption(VERBOSE_LOGGING_PARAM)) {
@ -120,10 +127,11 @@ public class UploadTerminologyCommand extends BaseRequestGeneratingCommand {
requestName = JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_REMOVE;
break;
}
invokeOperation(termUrl, datafile, client, requestName);
invokeOperation(termUrl, datafile, client, requestName, makeCurrentVersionOption);
}
private void invokeOperation(String theTermUrl, String[] theDatafile, IGenericClient theClient, String theOperationName) throws ParseException {
private void invokeOperation(String theTermUrl, String[] theDatafile,
IGenericClient theClient, String theOperationName, String makeCurrentVersionOption) throws ParseException {
IBaseParameters inputParameters = ParametersUtil.newInstance(myFhirCtx);
boolean isDeltaOperation =
@ -132,6 +140,16 @@ public class UploadTerminologyCommand extends BaseRequestGeneratingCommand {
ParametersUtil.addParameterToParametersUri(myFhirCtx, inputParameters, TerminologyUploaderProvider.PARAM_SYSTEM, theTermUrl);
if (StringUtils.isNotBlank(makeCurrentVersionOption)) {
boolean isMakeCurrentVersion = Boolean.parseBoolean(makeCurrentVersionOption);
ParametersUtil.addParameterToParametersString(
myFhirCtx, inputParameters, TermLoaderSvcImpl.MAKE_CURRENT_VERSION, String.valueOf(isMakeCurrentVersion));
}
//todo: is this the case?. Check with client
// if (mode != ModeEnum.SNAPSHOT && ! isMakeCurrentVersion) {
// throw new ParseException("Delta operations must use (or default to) " + CURRENT_VERSION + "=true parameter");
// }
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
ZipOutputStream zipOutputStream = new ZipOutputStream(byteArrayOutputStream, Charsets.UTF_8);
int compressedSourceBytesCount = 0;

View File

@ -2,14 +2,17 @@ package ca.uhn.fhir.cli;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider;
import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl;
import ca.uhn.fhir.jpa.term.UploadStatistics;
import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.client.api.IGenericClient;
import ca.uhn.fhir.rest.client.interceptor.CapturingInterceptor;
import ca.uhn.fhir.rest.server.RestfulServer;
import ca.uhn.fhir.test.BaseTest;
import ca.uhn.fhir.test.utilities.JettyUtil;
import ca.uhn.fhir.test.utilities.server.RestfulServerExtension;
import ca.uhn.fhir.util.ParametersUtil;
import com.google.common.base.Charsets;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
@ -22,6 +25,7 @@ import org.eclipse.jetty.servlet.ServletHolder;
import org.hamcrest.Matchers;
import org.hl7.fhir.r4.model.CodeSystem;
import org.hl7.fhir.r4.model.IdType;
import org.hl7.fhir.r4.model.Parameters;
import org.hl7.fhir.r4.model.Patient;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@ -58,6 +62,8 @@ import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.anyList;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ -346,139 +352,248 @@ public class UploadTerminologyCommandTest extends BaseTest {
}
@Nested
public class HeaderPassthroughOptionTests {
public class RestulServerExtensionTests {
@RegisterExtension
public final RestfulServerExtension myRestfulServerExtension = new RestfulServerExtension(myCtx);
private final String headerKey1 = "test-header-key-1";
private final String headerValue1 = "test header value-1";
@Nested
public class HeaderPassthroughOptionTests {
private final CapturingInterceptor myCapturingInterceptor = new CapturingInterceptor();
private final UploadTerminologyCommand testedCommand =
new RequestCapturingUploadTerminologyCommand(myCapturingInterceptor);
private final String headerKey1 = "test-header-key-1";
private final String headerValue1 = "test header value-1";
@BeforeEach
public void before() {
when(myTermLoaderSvc.loadCustom(eq("http://foo"), anyList(), any()))
.thenReturn(new UploadStatistics(100, new IdType("CodeSystem/101")));
private final CapturingInterceptor myCapturingInterceptor = new CapturingInterceptor();
private final UploadTerminologyCommand testedCommand =
new RequestCapturingUploadTerminologyCommand(myCapturingInterceptor);
TerminologyUploaderProvider provider = new TerminologyUploaderProvider(myCtx, myTermLoaderSvc);
myRestfulServerExtension.registerProvider(provider);
}
@BeforeEach
public void before() {
when(myTermLoaderSvc.loadCustom(eq("http://foo"), anyList(), any()))
.thenReturn(new UploadStatistics(100, new IdType("CodeSystem/101")));
@Test
public void oneHeader() throws Exception {
String[] args = new String[] {
"-v", "r4",
"-m", "SNAPSHOT",
"-t", "http://localhost:" + myRestfulServerExtension.getPort(),
"-u", "http://foo",
"-d", myConceptsFileName,
"-d", myHierarchyFileName,
"-hp", "\"" + headerKey1 + ":" + headerValue1 + "\""
};
writeConceptAndHierarchyFiles();
final CommandLine commandLine = new DefaultParser().parse(testedCommand.getOptions(), args, true);
testedCommand.run(commandLine);
assertNotNull(myCapturingInterceptor.getLastRequest());
Map<String, List<String>> allHeaders = myCapturingInterceptor.getLastRequest().getAllHeaders();
assertFalse(allHeaders.isEmpty());
assertTrue(allHeaders.containsKey(headerKey1));
assertEquals(1, allHeaders.get(headerKey1).size());
assertThat(allHeaders.get(headerKey1), hasItems(headerValue1));
}
@Test
public void twoHeadersSameKey() throws Exception {
final String headerValue2 = "test header value-2";
String[] args = new String[] {
"-v", "r4",
"-m", "SNAPSHOT",
"-t", "http://localhost:" + myRestfulServerExtension.getPort(),
"-u", "http://foo",
"-d", myConceptsFileName,
"-d", myHierarchyFileName,
"-hp", "\"" + headerKey1 + ":" + headerValue1 + "\"",
"-hp", "\"" + headerKey1 + ":" + headerValue2 + "\""
};
writeConceptAndHierarchyFiles();
final CommandLine commandLine = new DefaultParser().parse(testedCommand.getOptions(), args, true);
testedCommand.run(commandLine);
assertNotNull(myCapturingInterceptor.getLastRequest());
Map<String, List<String>> allHeaders = myCapturingInterceptor.getLastRequest().getAllHeaders();
assertFalse(allHeaders.isEmpty());
assertEquals(2, allHeaders.get(headerKey1).size());
assertTrue(allHeaders.containsKey(headerKey1));
assertEquals(2, allHeaders.get(headerKey1).size());
assertEquals(headerValue1, allHeaders.get(headerKey1).get(0));
assertEquals(headerValue2, allHeaders.get(headerKey1).get(1));
}
@Test
public void twoHeadersDifferentKeys() throws Exception {
final String headerKey2 = "test-header-key-2";
final String headerValue2 = "test header value-2";
String[] args = new String[] {
"-v", "r4",
"-m", "SNAPSHOT",
"-t", "http://localhost:" + myRestfulServerExtension.getPort(),
"-u", "http://foo",
"-d", myConceptsFileName,
"-d", myHierarchyFileName,
"-hp", "\"" + headerKey1 + ":" + headerValue1 + "\"",
"-hp", "\"" + headerKey2 + ":" + headerValue2 + "\""
};
writeConceptAndHierarchyFiles();
final CommandLine commandLine = new DefaultParser().parse(testedCommand.getOptions(), args, true);
testedCommand.run(commandLine);
assertNotNull(myCapturingInterceptor.getLastRequest());
Map<String, List<String>> allHeaders = myCapturingInterceptor.getLastRequest().getAllHeaders();
assertFalse(allHeaders.isEmpty());
assertTrue(allHeaders.containsKey(headerKey1));
assertEquals(1, allHeaders.get(headerKey1).size());
assertThat(allHeaders.get(headerKey1), hasItems(headerValue1));
assertTrue(allHeaders.containsKey(headerKey2));
assertEquals(1, allHeaders.get(headerKey2).size());
assertThat(allHeaders.get(headerKey2), hasItems(headerValue2));
}
private class RequestCapturingUploadTerminologyCommand extends UploadTerminologyCommand {
private CapturingInterceptor myCapturingInterceptor;
public RequestCapturingUploadTerminologyCommand(CapturingInterceptor theCapturingInterceptor) {
myCapturingInterceptor = theCapturingInterceptor;
TerminologyUploaderProvider provider = new TerminologyUploaderProvider(myCtx, myTermLoaderSvc);
myRestfulServerExtension.registerProvider(provider);
}
@Override
protected IGenericClient newClient(CommandLine theCommandLine) throws ParseException {
IGenericClient client = super.newClient(theCommandLine);
client.getInterceptorService().registerInterceptor(myCapturingInterceptor);
return client;
@Test
public void oneHeader() throws Exception {
String[] args = new String[] {
"-v", "r4",
"-m", "SNAPSHOT",
"-t", "http://localhost:" + myRestfulServerExtension.getPort(),
"-u", "http://foo",
"-d", myConceptsFileName,
"-d", myHierarchyFileName,
"-hp", "\"" + headerKey1 + ":" + headerValue1 + "\""
};
writeConceptAndHierarchyFiles();
final CommandLine commandLine = new DefaultParser().parse(testedCommand.getOptions(), args, true);
testedCommand.run(commandLine);
assertNotNull(myCapturingInterceptor.getLastRequest());
Map<String, List<String>> allHeaders = myCapturingInterceptor.getLastRequest().getAllHeaders();
assertFalse(allHeaders.isEmpty());
assertTrue(allHeaders.containsKey(headerKey1));
assertEquals(1, allHeaders.get(headerKey1).size());
assertThat(allHeaders.get(headerKey1), hasItems(headerValue1));
}
@Test
public void twoHeadersSameKey() throws Exception {
final String headerValue2 = "test header value-2";
String[] args = new String[] {
"-v", "r4",
"-m", "SNAPSHOT",
"-t", "http://localhost:" + myRestfulServerExtension.getPort(),
"-u", "http://foo",
"-d", myConceptsFileName,
"-d", myHierarchyFileName,
"-hp", "\"" + headerKey1 + ":" + headerValue1 + "\"",
"-hp", "\"" + headerKey1 + ":" + headerValue2 + "\""
};
writeConceptAndHierarchyFiles();
final CommandLine commandLine = new DefaultParser().parse(testedCommand.getOptions(), args, true);
testedCommand.run(commandLine);
assertNotNull(myCapturingInterceptor.getLastRequest());
Map<String, List<String>> allHeaders = myCapturingInterceptor.getLastRequest().getAllHeaders();
assertFalse(allHeaders.isEmpty());
assertEquals(2, allHeaders.get(headerKey1).size());
assertTrue(allHeaders.containsKey(headerKey1));
assertEquals(2, allHeaders.get(headerKey1).size());
assertEquals(headerValue1, allHeaders.get(headerKey1).get(0));
assertEquals(headerValue2, allHeaders.get(headerKey1).get(1));
}
@Test
public void twoHeadersDifferentKeys() throws Exception {
final String headerKey2 = "test-header-key-2";
final String headerValue2 = "test header value-2";
String[] args = new String[] {
"-v", "r4",
"-m", "SNAPSHOT",
"-t", "http://localhost:" + myRestfulServerExtension.getPort(),
"-u", "http://foo",
"-d", myConceptsFileName,
"-d", myHierarchyFileName,
"-hp", "\"" + headerKey1 + ":" + headerValue1 + "\"",
"-hp", "\"" + headerKey2 + ":" + headerValue2 + "\""
};
writeConceptAndHierarchyFiles();
final CommandLine commandLine = new DefaultParser().parse(testedCommand.getOptions(), args, true);
testedCommand.run(commandLine);
assertNotNull(myCapturingInterceptor.getLastRequest());
Map<String, List<String>> allHeaders = myCapturingInterceptor.getLastRequest().getAllHeaders();
assertFalse(allHeaders.isEmpty());
assertTrue(allHeaders.containsKey(headerKey1));
assertEquals(1, allHeaders.get(headerKey1).size());
assertThat(allHeaders.get(headerKey1), hasItems(headerValue1));
assertTrue(allHeaders.containsKey(headerKey2));
assertEquals(1, allHeaders.get(headerKey2).size());
assertThat(allHeaders.get(headerKey2), hasItems(headerValue2));
}
private class RequestCapturingUploadTerminologyCommand extends UploadTerminologyCommand {
private CapturingInterceptor myCapturingInterceptor;
public RequestCapturingUploadTerminologyCommand(CapturingInterceptor theCapturingInterceptor) {
myCapturingInterceptor = theCapturingInterceptor;
}
@Override
protected IGenericClient newClient(CommandLine theCommandLine) throws ParseException {
IGenericClient client = super.newClient(theCommandLine);
client.getInterceptorService().registerInterceptor(myCapturingInterceptor);
return client;
}
}
}
@Nested
public class CurrentVersionParameterTests {
private TerminologyUploaderProvider terminologyUploaderProvider;
private final UploadTerminologyCommand testedCommand = new UploadTerminologyCommand();
@Captor private ArgumentCaptor<RequestDetails> myRequestDetailsCaptor;
@BeforeEach
public void before() throws IOException {
writeConceptAndHierarchyFiles();
terminologyUploaderProvider = spy(new TerminologyUploaderProvider(myCtx, myTermLoaderSvc));
myRestfulServerExtension.registerProvider(terminologyUploaderProvider);
}
@Test
public void noMakeCurrentVersionParam() throws ParseException {
String[] args = new String[]{
"-v", "r4",
"-m", "SNAPSHOT",
"-t", "http://localhost:" + myRestfulServerExtension.getPort(),
"-u", "http://loinc.org",
"-d", myConceptsFileName,
"-d", myHierarchyFileName
};
final CommandLine commandLine = new DefaultParser()
.parse(testedCommand.getOptions(), args, true);
doReturn(ParametersUtil.newInstance(myCtx)).when(terminologyUploaderProvider)
.uploadSnapshot(any(), any(), any(), any(), myRequestDetailsCaptor.capture());
testedCommand.run(commandLine);
RequestDetails requestDetails = myRequestDetailsCaptor.getValue();
Parameters params = (Parameters) requestDetails.getResource();
assertFalse(params.hasParameter(TermLoaderSvcImpl.MAKE_CURRENT_VERSION));
}
@Test
public void explicitMakeCurrentVersionTrueParam() throws ParseException {
String[] args = new String[]{
"-v", "r4",
"-m", "SNAPSHOT",
"-t", "http://localhost:" + myRestfulServerExtension.getPort(),
"-u", "http://loinc.org",
"-d", myConceptsFileName,
"-d", myHierarchyFileName,
"--current-version", "true"
};
final CommandLine commandLine = new DefaultParser()
.parse(testedCommand.getOptions(), args, true);
doReturn(ParametersUtil.newInstance(myCtx)).when(terminologyUploaderProvider)
.uploadSnapshot(any(), any(), any(), any(), myRequestDetailsCaptor.capture());
testedCommand.run(commandLine);
RequestDetails requestDetails = myRequestDetailsCaptor.getValue();
Parameters params = (Parameters) requestDetails.getResource();
assertTrue(params.hasParameter(TermLoaderSvcImpl.MAKE_CURRENT_VERSION));
assertEquals("true", params.getParameter(TermLoaderSvcImpl.MAKE_CURRENT_VERSION).toString());
}
@Test
public void MakeCurrentVersionFalseParamReachesProvider() throws ParseException {
String[] args = new String[]{
"-v", "r4",
"-m", "SNAPSHOT",
"-t", "http://localhost:" + myRestfulServerExtension.getPort(),
"-u", "http://loinc.org",
"-d", myConceptsFileName,
"-d", myHierarchyFileName,
"--current-version", "false"
};
final CommandLine commandLine = new DefaultParser()
.parse(testedCommand.getOptions(), args, true);
doReturn(ParametersUtil.newInstance(myCtx)).when(terminologyUploaderProvider)
.uploadSnapshot(any(), any(), any(), any(), myRequestDetailsCaptor.capture());
testedCommand.run(commandLine);
RequestDetails requestDetails = myRequestDetailsCaptor.getValue();
Parameters params = (Parameters) requestDetails.getResource();
List<String> values = ParametersUtil.getNamedParameterValuesAsString(
myCtx, params, TermLoaderSvcImpl.MAKE_CURRENT_VERSION);
assertEquals(1, values.size());
assertEquals("false", values.get(0));
}
}
}
private void writeArchiveFile(File... theFiles) throws IOException {
private void writeArchiveFile(File... theFiles) throws IOException {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
ZipOutputStream zipOutputStream = new ZipOutputStream(byteArrayOutputStream, Charsets.UTF_8);

View File

@ -0,0 +1,5 @@
---
type: add
issue: 2786
title: "Add 'current-version' parameter to upload-terminology command to allow loading version
without becoming current. Available for LOINC uploads".

View File

@ -102,6 +102,7 @@ public class TerminologyUploaderProvider extends BaseJpaProvider {
public IBaseParameters uploadSnapshot(
HttpServletRequest theServletRequest,
@OperationParam(name = PARAM_SYSTEM, min = 1, typeName = "uri") IPrimitiveType<String> theCodeSystemUrl,
@OperationParam(name = TermLoaderSvcImpl.MAKE_CURRENT_VERSION, min = 0, max = 1) IPrimitiveType<String> isMakeCurrentVersion,
@OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles,
RequestDetails theRequestDetails
) {

View File

@ -86,9 +86,6 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static org.apache.commons.lang3.StringUtils.defaultIfBlank;
import static org.apache.commons.lang3.StringUtils.defaultString;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
@ -381,7 +378,9 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
@Override
@Transactional
public IIdType storeNewCodeSystemVersion(CodeSystem theCodeSystemResource, TermCodeSystemVersion theCodeSystemVersion, RequestDetails theRequest, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) {
public IIdType storeNewCodeSystemVersion(CodeSystem theCodeSystemResource, TermCodeSystemVersion theCodeSystemVersion,
RequestDetails theRequest, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps, boolean theIsMakeItCurrentVersion) {
assert TransactionSynchronizationManager.isActualTransactionActive();
Validate.notBlank(theCodeSystemResource.getUrl(), "theCodeSystemResource must have a URL");
@ -396,7 +395,8 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
populateCodeSystemVersionProperties(theCodeSystemVersion, theCodeSystemResource, resource);
storeNewCodeSystemVersion(codeSystemResourcePid, theCodeSystemResource.getUrl(), theCodeSystemResource.getName(), theCodeSystemResource.getVersion(), theCodeSystemVersion, resource);
storeNewCodeSystemVersion(codeSystemResourcePid, theCodeSystemResource.getUrl(), theCodeSystemResource.getName(),
theCodeSystemResource.getVersion(), theCodeSystemVersion, resource, theIsMakeItCurrentVersion);
myDeferredStorageSvc.addConceptMapsToStorageQueue(theConceptMaps);
myDeferredStorageSvc.addValueSetsToStorageQueue(theValueSets);
@ -406,7 +406,10 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
@Override
@Transactional
public void storeNewCodeSystemVersion(ResourcePersistentId theCodeSystemResourcePid, String theSystemUri, String theSystemName, String theCodeSystemVersionId, TermCodeSystemVersion theCodeSystemVersion, ResourceTable theCodeSystemResourceTable) {
public void storeNewCodeSystemVersion(ResourcePersistentId theCodeSystemResourcePid, String theSystemUri,
String theSystemName, String theCodeSystemVersionId, TermCodeSystemVersion theCodeSystemVersion,
ResourceTable theCodeSystemResourceTable, boolean theIsMakeItCurrentVersion) {
assert TransactionSynchronizationManager.isActualTransactionActive();
ourLog.debug("Storing code system");
@ -470,32 +473,39 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
ourLog.debug("Saving code system");
codeSystem.setCurrentVersion(codeSystemToStore);
if (codeSystem.getPid() == null) {
codeSystem = myCodeSystemDao.saveAndFlush(codeSystem);
}
ourLog.debug("Setting CodeSystemVersion[{}] on {} concepts...", codeSystem.getPid(), totalCodeCount);
for (TermConcept next : conceptsToSave) {
populateVersion(next, codeSystemToStore);
}
ourLog.debug("Saving {} concepts...", totalCodeCount);
IdentityHashMap<TermConcept, Object> conceptsStack2 = new IdentityHashMap<>();
for (TermConcept next : conceptsToSave) {
persistChildren(next, codeSystemToStore, conceptsStack2, totalCodeCount);
}
ourLog.debug("Done saving concepts, flushing to database");
if (myDeferredStorageSvc.isStorageQueueEmpty() == false) {
ourLog.info("Note that some concept saving has been deferred");
if (theIsMakeItCurrentVersion) {
makeCodeSystemCurrentVersion(codeSystem, codeSystemToStore, conceptsToSave, totalCodeCount);
}
}
private void makeCodeSystemCurrentVersion(TermCodeSystem theCodeSystem, TermCodeSystemVersion theCodeSystemToStore,
Collection<TermConcept> theConceptsToSave, int theTotalCodeCount) {
theCodeSystem.setCurrentVersion(theCodeSystemToStore);
if (theCodeSystem.getPid() == null) {
theCodeSystem = myCodeSystemDao.saveAndFlush(theCodeSystem);
}
ourLog.debug("Setting CodeSystemVersion[{}] on {} concepts...", theCodeSystem.getPid(), theTotalCodeCount);
for (TermConcept next : theConceptsToSave) {
populateVersion(next, theCodeSystemToStore);
}
ourLog.debug("Saving {} concepts...", theTotalCodeCount);
IdentityHashMap<TermConcept, Object> conceptsStack2 = new IdentityHashMap<>();
for (TermConcept next : theConceptsToSave) {
persistChildren(next, theCodeSystemToStore, conceptsStack2, theTotalCodeCount);
}
ourLog.debug("Done saving concepts, flushing to database");
if (! myDeferredStorageSvc.isStorageQueueEmpty()) {
ourLog.info("Note that some concept saving has been deferred");
}
}
private TermCodeSystemVersion getExistingTermCodeSystemVersion(Long theCodeSystemVersionPid, String theCodeSystemVersion) {
TermCodeSystemVersion existing;
if (theCodeSystemVersion == null) {

View File

@ -38,9 +38,11 @@ import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.util.ClasspathUtil;
import ca.uhn.fhir.util.ParametersUtil;
import ca.uhn.fhir.util.ValidateUtil;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
@ -49,10 +51,12 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.CodeSystem;
import org.hl7.fhir.r4.model.ConceptMap;
import org.hl7.fhir.r4.model.Enumerations;
import org.hl7.fhir.r4.model.Parameters;
import org.hl7.fhir.r4.model.ValueSet;
import org.springframework.aop.support.AopUtils;
import org.springframework.beans.factory.annotation.Autowired;
@ -148,6 +152,7 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
public static final String CUSTOM_CONCEPTS_FILE = "concepts.csv";
public static final String CUSTOM_HIERARCHY_FILE = "hierarchy.csv";
public static final String CUSTOM_PROPERTIES_FILE = "properties.csv";
public static final String MAKE_CURRENT_VERSION = "isMakeCurrentVersion";
static final String IMGTHLA_HLA_NOM_TXT = "hla_nom.txt";
static final String IMGTHLA_HLA_XML = "hla.xml";
static final String CUSTOM_CODESYSTEM_JSON = "codesystem.json";
@ -188,7 +193,7 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
@Override
public UploadStatistics loadImgthla(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
try (LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles)) {
try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
List<String> mandatoryFilenameFragments = Arrays.asList(
IMGTHLA_HLA_NOM_TXT,
IMGTHLA_HLA_XML
@ -201,11 +206,23 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
}
}
@VisibleForTesting
LoadedFileDescriptors getLoadedFileDescriptors(List<FileDescriptor> theFiles) {
return new LoadedFileDescriptors(theFiles);
}
@Override
public UploadStatistics loadLoinc(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
try (LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles)) {
try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
Properties uploadProperties = getProperties(descriptors, LOINC_UPLOAD_PROPERTIES_FILE.getCode());
String codeSystemVersionId = uploadProperties.getProperty(LOINC_CODESYSTEM_VERSION.getCode());
boolean isMakeCurrentVersion = getIsMakeCurrentVersion(theRequestDetails);
if (StringUtils.isBlank(codeSystemVersionId) && ! isMakeCurrentVersion) {
throw new InvalidRequestException("'" + LOINC_CODESYSTEM_VERSION.getCode() +
"' property is required when 'current-version' parameter is 'false'");
}
List<String> mandatoryFilenameFragments = Arrays.asList(
uploadProperties.getProperty(LOINC_ANSWERLIST_FILE.getCode(), LOINC_ANSWERLIST_FILE_DEFAULT.getCode()),
uploadProperties.getProperty(LOINC_ANSWERLIST_LINK_FILE.getCode(), LOINC_ANSWERLIST_LINK_FILE_DEFAULT.getCode()),
@ -238,21 +255,47 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
ourLog.info("Beginning LOINC processing");
if (isMakeCurrentVersion) {
if (codeSystemVersionId != null) {
processLoincFiles(descriptors, theRequestDetails, uploadProperties, false, isMakeCurrentVersion);
uploadProperties.remove(LOINC_CODESYSTEM_VERSION.getCode());
}
ourLog.info("Uploading CodeSystem and making it current version");
String codeSystemVersionId = uploadProperties.getProperty(LOINC_CODESYSTEM_VERSION.getCode());
if (codeSystemVersionId != null) {
// Load the code system with version and then remove the version property.
processLoincFiles(descriptors, theRequestDetails, uploadProperties, false);
uploadProperties.remove(LOINC_CODESYSTEM_VERSION.getCode());
} else {
ourLog.info("Uploading CodeSystem without updating current version");
}
// Load the same code system with null version. This will become the default version.
return processLoincFiles(descriptors, theRequestDetails, uploadProperties, true);
return processLoincFiles(descriptors, theRequestDetails, uploadProperties, true, isMakeCurrentVersion);
}
}
private boolean getIsMakeCurrentVersion(RequestDetails theRequestDetails) throws InvalidRequestException{
boolean defaultResponse = true;
IBaseResource requestResource = theRequestDetails.getResource();
if (requestResource == null) return defaultResponse;
if (! (requestResource instanceof Parameters)) return defaultResponse;
Parameters parameters = (Parameters) theRequestDetails.getResource();
List<String> values = ParametersUtil.getNamedParameterValuesAsString(FhirContext.forR4(), parameters, MAKE_CURRENT_VERSION);
if (CollectionUtils.isEmpty(values)) return true;
if (values.size() > 1) {
throw new InvalidRequestException("Parameter '" + MAKE_CURRENT_VERSION + "' was specified more than once.");
}
if (! values.get(0).equalsIgnoreCase("false") && ! values.get(0).equalsIgnoreCase("true")) {
throw new InvalidRequestException("Parameter " + MAKE_CURRENT_VERSION + " only accepts values: ['true' or 'false']");
}
return Boolean.parseBoolean(values.get(0));
}
@Override
public UploadStatistics loadSnomedCt(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
try (LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles)) {
try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
List<String> expectedFilenameFragments = Arrays.asList(
SCT_FILE_DESCRIPTION,
@ -279,7 +322,7 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
int count = 0;
try (LoadedFileDescriptors compressedDescriptors = new LoadedFileDescriptors(theFiles)) {
try (LoadedFileDescriptors compressedDescriptors = getLoadedFileDescriptors(theFiles)) {
for (FileDescriptor nextDescriptor : compressedDescriptors.getUncompressedFileDescriptors()) {
if (nextDescriptor.getFilename().toLowerCase(Locale.US).endsWith(".xml")) {
try (InputStream inputStream = nextDescriptor.getInputStream();
@ -296,13 +339,13 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
cs.setVersion(codeSystemVersion.getCodeSystemVersionId());
IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, cs, null, null);
IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, cs, null, null, true);
return new UploadStatistics(count, target);
}
@Override
public UploadStatistics loadCustom(String theSystem, List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
try (LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles)) {
try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
Optional<String> codeSystemContent = loadFile(descriptors, CUSTOM_CODESYSTEM_JSON, CUSTOM_CODESYSTEM_XML);
CodeSystem codeSystem;
if (codeSystemContent.isPresent()) {
@ -321,7 +364,7 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
CustomTerminologySet terminologySet = CustomTerminologySet.load(descriptors, false);
TermCodeSystemVersion csv = terminologySet.toCodeSystemVersion();
IIdType target = storeCodeSystem(theRequestDetails, csv, codeSystem, null, null);
IIdType target = storeCodeSystem(theRequestDetails, csv, codeSystem, null, null, true);
return new UploadStatistics(terminologySet.getSize(), target);
}
}
@ -330,7 +373,7 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
@Override
public UploadStatistics loadDeltaAdd(String theSystem, List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
ourLog.info("Processing terminology delta ADD for system[{}] with files: {}", theSystem, theFiles.stream().map(t -> t.getFilename()).collect(Collectors.toList()));
try (LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles)) {
try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
CustomTerminologySet terminologySet = CustomTerminologySet.load(descriptors, false);
return myCodeSystemStorageSvc.applyDeltaCodeSystemsAdd(theSystem, terminologySet);
}
@ -339,7 +382,7 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
@Override
public UploadStatistics loadDeltaRemove(String theSystem, List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
ourLog.info("Processing terminology delta REMOVE for system[{}] with files: {}", theSystem, theFiles.stream().map(t -> t.getFilename()).collect(Collectors.toList()));
try (LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles)) {
try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
CustomTerminologySet terminologySet = CustomTerminologySet.load(descriptors, true);
return myCodeSystemStorageSvc.applyDeltaCodeSystemsRemove(theSystem, terminologySet);
}
@ -378,8 +421,9 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
}
@VisibleForTesting
@NotNull
private Properties getProperties(LoadedFileDescriptors theDescriptors, String thePropertiesFile) {
Properties getProperties(LoadedFileDescriptors theDescriptors, String thePropertiesFile) {
Properties retVal = new Properties();
try (InputStream propertyStream = TermLoaderSvcImpl.class.getResourceAsStream("/ca/uhn/fhir/jpa/term/loinc/loincupload.properties")) {
@ -515,7 +559,9 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
// return new UploadStatistics(conceptCount, target);
}
UploadStatistics processLoincFiles(LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails, Properties theUploadProperties, Boolean theCloseFiles) {
UploadStatistics processLoincFiles(LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails,
Properties theUploadProperties, Boolean theCloseFiles, boolean isMakeCurrentVersion) {
final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
final Map<String, TermConcept> code2concept = new HashMap<>();
final List<ValueSet> valueSets = new ArrayList<>();
@ -646,7 +692,7 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
int conceptCount = code2concept.size();
ourLog.info("Have {} total concepts, {} root concepts, {} ValueSets", conceptCount, rootConceptCount, valueSetCount);
IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, loincCs, valueSets, conceptMaps);
IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, loincCs, valueSets, conceptMaps, isMakeCurrentVersion);
return new UploadStatistics(conceptCount, target);
}
@ -720,12 +766,14 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
cs.setName("SNOMED CT");
cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
cs.setStatus(Enumerations.PublicationStatus.ACTIVE);
IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, cs, null, null);
IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, cs, null, null, true);
return new UploadStatistics(code2concept.size(), target);
}
private IIdType storeCodeSystem(RequestDetails theRequestDetails, final TermCodeSystemVersion theCodeSystemVersion, CodeSystem theCodeSystem, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) {
private IIdType storeCodeSystem(RequestDetails theRequestDetails, final TermCodeSystemVersion theCodeSystemVersion,
CodeSystem theCodeSystem, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps, boolean isMakeCurrentVersion) {
Validate.isTrue(theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT);
List<ValueSet> valueSets = ObjectUtils.defaultIfNull(theValueSets, Collections.emptyList());
@ -733,7 +781,8 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc {
IIdType retVal;
myDeferredStorageSvc.setProcessDeferred(false);
retVal = myCodeSystemStorageSvc.storeNewCodeSystemVersion(theCodeSystem, theCodeSystemVersion, theRequestDetails, valueSets, conceptMaps);
retVal = myCodeSystemStorageSvc.storeNewCodeSystemVersion(theCodeSystem, theCodeSystemVersion,
theRequestDetails, valueSets, conceptMaps, isMakeCurrentVersion);
myDeferredStorageSvc.setProcessDeferred(true);
return retVal;

View File

@ -20,7 +20,6 @@ package ca.uhn.fhir.jpa.term.api;
* #L%
*/
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
@ -28,10 +27,12 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.term.UploadStatistics;
import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.CodeSystem;
import org.hl7.fhir.r4.model.ValueSet;
import javax.transaction.Transactional;
import java.util.List;
/**
@ -43,12 +44,44 @@ public interface ITermCodeSystemStorageSvc {
void deleteCodeSystemVersion(TermCodeSystemVersion theCodeSystemVersion);
void storeNewCodeSystemVersion(ResourcePersistentId theCodeSystemResourcePid, String theSystemUri, String theSystemName, String theSystemVersionId, TermCodeSystemVersion theCodeSystemVersion, ResourceTable theCodeSystemResourceTable);
/**
* Default implementation supports previous signature of method which was added theIsMakeItCurrentVersion parameter
*/
@Transactional
default void storeNewCodeSystemVersion(ResourcePersistentId theCodeSystemResourcePid, String theSystemUri, String theSystemName,
String theSystemVersionId, TermCodeSystemVersion theCodeSystemVersion, ResourceTable theCodeSystemResourceTable) {
storeNewCodeSystemVersion(theCodeSystemResourcePid, theSystemUri, theSystemName, theSystemVersionId,
theCodeSystemVersion, theCodeSystemResourceTable, true);
}
void storeNewCodeSystemVersion(ResourcePersistentId theCodeSystemResourcePid, String theSystemUri, String theSystemName,
String theSystemVersionId, TermCodeSystemVersion theCodeSystemVersion, ResourceTable theCodeSystemResourceTable,
boolean theIsMakeItCurrentVersion);
/**
* Default implementation supports previous signature of method which was added theIsMakeItCurrentVersion parameter
* @return Returns the ID of the created/updated code system
*/
@Transactional
default IIdType storeNewCodeSystemVersion(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource,
TermCodeSystemVersion theCodeSystemVersion, RequestDetails theRequestDetails, List<ValueSet> theValueSets,
List<org.hl7.fhir.r4.model.ConceptMap> theConceptMaps) {
return storeNewCodeSystemVersion(theCodeSystemResource, theCodeSystemVersion,
theRequestDetails, theValueSets, theConceptMaps, true);
}
/**
* @return Returns the ID of the created/updated code system
*/
IIdType storeNewCodeSystemVersion(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource, TermCodeSystemVersion theCodeSystemVersion, RequestDetails theRequestDetails, List<ValueSet> theValueSets, List<org.hl7.fhir.r4.model.ConceptMap> theConceptMaps);
IIdType storeNewCodeSystemVersion(org.hl7.fhir.r4.model.CodeSystem theCodeSystemResource,
TermCodeSystemVersion theCodeSystemVersion, RequestDetails theRequestDetails, List<ValueSet> theValueSets,
List<org.hl7.fhir.r4.model.ConceptMap> theConceptMaps, boolean theIsMakeItCurrentVersion);
void storeNewCodeSystemVersionIfNeeded(CodeSystem theCodeSystem, ResourceTable theResourceEntity);

View File

@ -52,7 +52,9 @@ public class TerminologyLoaderSvcCustomTest extends BaseLoaderTest {
// Actually do the load
mySvc.loadCustom("http://example.com/labCodes", myFiles.getFiles(), mySrd);
verify(myTermCodeSystemStorageSvc, times(1)).storeNewCodeSystemVersion(mySystemCaptor.capture(), myCsvCaptor.capture(), any(RequestDetails.class), myValueSetsCaptor.capture(), myConceptMapCaptor.capture());
verify(myTermCodeSystemStorageSvc, times(1)).storeNewCodeSystemVersion(
mySystemCaptor.capture(), myCsvCaptor.capture(), any(RequestDetails.class), myValueSetsCaptor.capture(),
myConceptMapCaptor.capture(), eq(true));
Map<String, TermConcept> concepts = extractConcepts();
// Verify codesystem
@ -82,7 +84,9 @@ public class TerminologyLoaderSvcCustomTest extends BaseLoaderTest {
// Actually do the load
mySvc.loadCustom("http://example.com/labCodes", myFiles.getFiles(), mySrd);
verify(myTermCodeSystemStorageSvc, times(1)).storeNewCodeSystemVersion(mySystemCaptor.capture(), myCsvCaptor.capture(), any(RequestDetails.class), myValueSetsCaptor.capture(), myConceptMapCaptor.capture());
verify(myTermCodeSystemStorageSvc, times(1)).storeNewCodeSystemVersion(
mySystemCaptor.capture(), myCsvCaptor.capture(), any(RequestDetails.class),
myValueSetsCaptor.capture(), myConceptMapCaptor.capture(), eq(true));
Map<String, TermConcept> concepts = extractConcepts();
// Verify codesystem
@ -101,7 +105,9 @@ public class TerminologyLoaderSvcCustomTest extends BaseLoaderTest {
// Actually do the load
mySvc.loadCustom("http://example.com/labCodes", myFiles.getFiles(), mySrd);
verify(myTermCodeSystemStorageSvc, times(1)).storeNewCodeSystemVersion(mySystemCaptor.capture(), myCsvCaptor.capture(), any(RequestDetails.class), myValueSetsCaptor.capture(), myConceptMapCaptor.capture());
verify(myTermCodeSystemStorageSvc, times(1)).storeNewCodeSystemVersion(
mySystemCaptor.capture(), myCsvCaptor.capture(), any(RequestDetails.class),
myValueSetsCaptor.capture(), myConceptMapCaptor.capture(), eq(true));
Map<String, TermConcept> concepts = extractConcepts();
TermConcept code;

View File

@ -82,7 +82,8 @@ public class TerminologyLoaderSvcSnomedCtTest extends BaseLoaderTest {
mySvc.loadSnomedCt(myFiles.getFiles(), mySrd);
verify(myTermCodeSystemStorageSvc).storeNewCodeSystemVersion(any(CodeSystem.class), myCsvCaptor.capture(), any(RequestDetails.class), anyList(), anyListOf(ConceptMap.class));
verify(myTermCodeSystemStorageSvc).storeNewCodeSystemVersion(any(CodeSystem.class), myCsvCaptor.capture(),
any(RequestDetails.class), anyList(), anyListOf(ConceptMap.class), eq(true));
TermCodeSystemVersion csv = myCsvCaptor.getValue();
TreeSet<String> allCodes = toCodes(csv, true);