Search Narrowing by Code Tweaks (#3453)

* Search narrowing tweaks

* Search narrowing tweaks

* Test

* Test fixes

* Try to fix LGTM

* Add license headers

* Test fixes

* Test fixes
This commit is contained in:
James Agnew 2022-03-07 20:27:21 -05:00 committed by GitHub
parent 0e9372775b
commit ac80c7ffaa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 393 additions and 78 deletions

View File

@ -25,7 +25,7 @@ public final class Msg {
/**
* IMPORTANT: Please update the following comment after you add a new code
* Last code value: 2070
* Last code value: 2071
*/
private Msg() {}

View File

@ -0,0 +1,5 @@
---
type: add
issue: 3452
title: "The JPA server terminology service can now process IS-A filters in ValueSet expansion on servers with
Hibernate Search disabled."

View File

@ -1,5 +1,25 @@
package ca.uhn.fhir.jpa.bulk.export.svc;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2022 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;

View File

@ -250,15 +250,15 @@ public class TermValueSet implements Serializable {
@Override
public String toString() {
return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
.append("myId", myId)
.append("myUrl", myUrl)
.append(myResource != null ? ("myResource=" + myResource.toString()) : ("myResource=(null)"))
.append("myResourcePid", myResourcePid)
.append("myName", myName)
.append(myConcepts != null ? ("myConcepts - size=" + myConcepts.size()) : ("myConcepts=(null)"))
.append("myTotalConcepts", myTotalConcepts)
.append("myTotalConceptDesignations", myTotalConceptDesignations)
.append("myExpansionStatus", myExpansionStatus)
.append("id", myId)
.append("url", myUrl)
.append(myResource != null ? ("resource=" + myResource.toString()) : ("resource=(null)"))
.append("resourcePid", myResourcePid)
.append("name", myName)
.append(myConcepts != null ? ("concepts - size=" + myConcepts.size()) : ("concepts=(null)"))
.append("totalConcepts", myTotalConcepts)
.append("totalConceptDesignations", myTotalConceptDesignations)
.append("expansionStatus", myExpansionStatus)
.toString();
}
}

View File

@ -1,5 +1,25 @@
package ca.uhn.fhir.jpa.search.autocomplete;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2022 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.google.gson.JsonObject;
import org.apache.commons.lang3.Validate;

View File

@ -132,7 +132,6 @@ import org.hl7.fhir.r4.model.IntegerType;
import org.hl7.fhir.r4.model.StringType;
import org.hl7.fhir.r4.model.ValueSet;
import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome;
import org.jetbrains.annotations.NotNull;
import org.quartz.JobExecutionContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
@ -305,26 +304,32 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
}
private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, Collection<TermConceptDesignation> theDesignations) {
private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, Collection<TermConceptDesignation> theDesignations) {
if (StringUtils.isNotEmpty(theCodeSystemVersion)) {
if (isNoneBlank(theCodeSystem, theCode)) {
if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
return true;
}
if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.excludeConcept(theCodeSystem + "|" + theCodeSystemVersion, theCode);
return true;
}
}
} else {
if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
return true;
}
if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
return true;
}
}
return false;
}
private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, String theCodeSystem, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, String theSystemVersion) {
@ -1302,18 +1307,27 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
private void handleFilterConceptAndCode(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
TermConcept code = findCode(theSystem, theFilter.getValue())
.orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue()));
TermConcept code = findCodeForFilterCriteria(theSystem, theFilter);
if (theFilter.getOp() == ValueSet.FilterOperator.ISA) {
ourLog.debug(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
b.must(f.match().field("myParentPids").matching("" + code.getId()));
} else {
throw new InvalidRequestException(Msg.code(894) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
throwInvalidFilter(theFilter, "");
}
}
@Nonnull
private TermConcept findCodeForFilterCriteria(String theSystem, ValueSet.ConceptSetFilterComponent theFilter) {
return findCode(theSystem, theFilter.getValue())
.orElseThrow(() -> new InvalidRequestException(Msg.code(2071) + "Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue()));
}
private void throwInvalidFilter(ValueSet.ConceptSetFilterComponent theFilter, String theErrorSuffix) {
throw new InvalidRequestException(Msg.code(894) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty() + theErrorSuffix);
}
private void isCodeSystemLoincOrThrowInvalidRequestException(String theSystemIdentifier, String theProperty) {
String systemUrl = getUrlFromIdentifier(theSystemIdentifier);
if (!isCodeSystemLoinc(systemUrl)) {
@ -1499,9 +1513,27 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
Validate.isTrue(((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator).getParameter().isEmpty(), "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server.");
}
Validate.isTrue(theInclude.getFilter().isEmpty(), "Can not expand ValueSet with filters - Hibernate Search is not enabled on this server.");
Validate.isTrue(isNotBlank(theSystem), "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server.");
for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) {
boolean handled = false;
switch (nextFilter.getProperty()) {
case "concept":
case "code":
if (nextFilter.getOp() == ValueSet.FilterOperator.ISA) {
theValueSetCodeAccumulator.addMessage("Processing IS-A filter in database - Note that Hibernate Search is not enabled on this server, so this operation can be inefficient.");
TermConcept code = findCodeForFilterCriteria(theSystem, nextFilter);
addConceptAndChildren(theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, code);
handled = true;
}
break;
}
if (!handled) {
throwInvalidFilter(nextFilter, " - Note that Hibernate Search is disabled on this server so not all ValueSet expansion funtionality is available.");
}
}
if (theInclude.getConcept().isEmpty()) {
Collection<TermConcept> concepts = myConceptDao.fetchConceptsAndDesignationsByVersionPid(theVersion.getPid());
@ -1531,6 +1563,15 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
private void addConceptAndChildren(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, TermConcept theConcept) {
for (TermConcept nextChild : theConcept.getChildCodes()) {
boolean added = addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), nextChild.getCode(), nextChild.getDisplay(), nextChild.getId(), nextChild.getParentPidsAsString(), nextChild.getDesignations());
if (added) {
addConceptAndChildren(theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, nextChild);
}
}
}
@Override
@Transactional
public String invalidatePreCalculatedExpansion(IIdType theValueSetId, RequestDetails theRequestDetails) {
@ -1925,21 +1966,21 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
expandValueSet(options, valueSet, accumulator);
// We are done with this ValueSet.
txTemplate.execute(t -> {
txTemplate.executeWithoutResult(t -> {
valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANDED);
valueSetToExpand.setExpansionTimestamp(new Date());
myTermValueSetDao.saveAndFlush(valueSetToExpand);
return null;
});
ourLog.info("Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", valueSet.getId(), valueSet.getUrl(), accumulator.getConceptsSaved(), sw);
} catch (Exception e) {
ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e);
txTemplate.execute(t -> {
txTemplate.executeWithoutResult(t -> {
valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.FAILED_TO_EXPAND);
myTermValueSetDao.saveAndFlush(valueSetToExpand);
return null;
});
}
}
@ -2181,7 +2222,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
return result;
} else {
return null;
return new LookupCodeResult()
.setFound(false);
}
});
}

View File

@ -189,17 +189,25 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
Integer capacityRemaining = getCapacityRemaining();
if (capacityRemaining == 0) {
String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myMaxCapacity);
msg = appendAccumulatorMessages(msg);
throw new ExpansionTooCostlyException(Msg.code(831) + msg);
}
if (myHardExpansionMaximumSize > 0 && myAddedConcepts > myHardExpansionMaximumSize) {
String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myHardExpansionMaximumSize);
msg = appendAccumulatorMessages(msg);
throw new ExpansionTooCostlyException(Msg.code(832) + msg);
}
myAddedConcepts++;
}
@Nonnull
private String appendAccumulatorMessages(String msg) {
msg += getMessages().stream().map(t->" - " + t).collect(Collectors.joining());
return msg;
}
public Integer getTotalConcepts() {
return myTotalConcepts;
}

View File

@ -62,8 +62,8 @@ public class CustomTerminologySet {
myRootConcepts = theRootConcepts;
}
public void addRootConcept(String theCode) {
addRootConcept(theCode, null);
public TermConcept addRootConcept(String theCode) {
return addRootConcept(theCode, null);
}
public TermConcept addRootConcept(String theCode, String theDisplay) {

View File

@ -25,6 +25,7 @@ import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao;
import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
import ca.uhn.fhir.jpa.entity.TermConceptProperty;
@ -187,6 +188,8 @@ public abstract class BaseJpaTest extends BaseTest {
@Autowired
protected ITermValueSetConceptDao myTermValueSetConceptDao;
@Autowired
protected ITermValueSetDao myTermValueSetDao;
@Autowired
protected ITermConceptDesignationDao myTermConceptDesignationDao;
@Autowired
protected ITermConceptPropertyDao myTermConceptPropertyDao;
@ -355,6 +358,14 @@ public abstract class BaseJpaTest extends BaseTest {
});
}
protected int logAllValueSets() {
return runInTransaction(() -> {
List<TermValueSet> valueSets = myTermValueSetDao.findAll();
ourLog.info("ValueSets:\n * {}", valueSets.stream().map(t -> t.toString()).collect(Collectors.joining("\n * ")));
return valueSets.size();
});
}
protected int logAllForcedIds() {
return runInTransaction(() -> {
List<ForcedId> forcedIds = myForcedIdDao.findAll();

View File

@ -49,6 +49,7 @@ import java.util.Set;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.containsStringIgnoringCase;
import static org.hamcrest.Matchers.empty;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -729,7 +730,8 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test {
myObservationDao.search(params).size();
fail();
} catch (InternalErrorException e) {
assertEquals(Msg.code(885) + "Expansion of ValueSet produced too many codes (maximum 1) - Operation aborted!", e.getMessage());
assertThat(e.getMessage(), containsString(Msg.code(885) + "Expansion of ValueSet produced too many codes (maximum 1) - Operation aborted!"));
}
}

View File

@ -49,6 +49,7 @@ import java.util.stream.Collectors;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.containsStringIgnoringCase;
import static org.hamcrest.Matchers.empty;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -873,7 +874,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
myObservationDao.search(params).size();
fail();
} catch (InternalErrorException e) {
assertEquals(Msg.code(885) + "Expansion of ValueSet produced too many codes (maximum 1) - Operation aborted!", e.getMessage());
assertThat(e.getMessage(), containsString(Msg.code(885) + "Expansion of ValueSet produced too many codes (maximum 1) - Operation aborted!"));
}
}

View File

@ -6,6 +6,9 @@ import ca.uhn.fhir.context.support.ValidationSupportContext;
import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
import ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl;
import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
@ -29,9 +32,11 @@ import java.io.IOException;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.startsWith;
import static org.hamcrest.Matchers.stringContainsInOrder;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@ -43,6 +48,7 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
@AfterEach
public void after() {
BaseTermReadSvcImpl.setForceDisableHibernateSearchForUnitTest(false);
myDaoConfig.setPreExpandValueSets(new DaoConfig().isPreExpandValueSets());
myDaoConfig.setMaximumExpansionSize(new DaoConfig().getMaximumExpansionSize());
}
@ -91,6 +97,210 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
}
@Test
public void testValidateCodeInValueSet_HierarchicalAndEnumeratedValueset() {
myValueSetDao.delete(myExtensionalVsId);
ourLog.info("Creating CodeSystem");
CodeSystem cs = new CodeSystem();
cs.setId("CodeSystem/cs");
cs.setUrl("http://cs");
cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
cs.setStatus(Enumerations.PublicationStatus.ACTIVE);
myCodeSystemDao.update(cs);
ourLog.info("Adding codes to codesystem");
CustomTerminologySet delta = new CustomTerminologySet();
TermConcept parent = delta.addRootConcept("parent");
for (int j = 0; j < 1200; j++) {
parent
.addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA)
.setCode("child" + j);
}
myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://cs", delta);
ourLog.info("Creating ValueSet");
ValueSet vs = new ValueSet();
vs.setId("ValueSet/vs");
vs.setUrl("http://vs");
vs.getCompose()
.addInclude()
.setSystem("http://cs")
.addFilter()
.setProperty("concept")
.setOp(ValueSet.FilterOperator.ISA)
.setValue("parent");
vs.getCompose()
.addInclude()
.setSystem("http://cs-np")
.addConcept(new ValueSet.ConceptReferenceComponent(new CodeType("code0")))
.addConcept(new ValueSet.ConceptReferenceComponent(new CodeType("code1")));
myValueSetDao.update(vs);
IValidationSupport.CodeValidationResult outcome;
ValidationSupportContext ctx = new ValidationSupportContext(myValidationSupport);
ConceptValidationOptions options = new ConceptValidationOptions();
// In memory - Hierarchy in existing CS
outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "child10", null, "http://vs");
assertNotNull(outcome);
assertTrue(outcome.isOk());
assertEquals("Code was validated against in-memory expansion of ValueSet: http://vs", outcome.getMessage());
outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "childX", null, "http://vs");
assertNotNull(outcome);
assertFalse(outcome.isOk());
assertEquals("Unknown code 'http://cs#childX' for in-memory expansion of ValueSet 'http://vs'", outcome.getMessage());
// In memory - Enumerated in non-present CS
outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "code1", null, "http://vs");
assertNotNull(outcome);
assertTrue(outcome.isOk());
assertEquals("Code was validated against in-memory expansion of ValueSet: http://vs", outcome.getMessage());
outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "codeX", null, "http://vs");
assertNotNull(outcome);
assertFalse(outcome.isOk());
assertEquals("Unknown code 'http://cs-np#codeX' for in-memory expansion of ValueSet 'http://vs'", outcome.getMessage());
// Precalculated
myTerminologyDeferredStorageSvc.saveAllDeferred();
myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
logAllValueSets();
myCachingValidationSupport.invalidateCaches();
outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "child10", null, "http://vs");
assertNotNull(outcome);
assertTrue(outcome.isOk());
assertThat(outcome.getMessage(), startsWith("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "childX", null, "http://vs");
assertNotNull(outcome);
assertFalse(outcome.isOk());
assertThat(outcome.getMessage(), containsString("Unknown code http://cs#childX"));
assertThat(outcome.getMessage(), containsString("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
// Precalculated - Enumerated in non-present CS
outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "code1", null, "http://vs");
assertNotNull(outcome);
assertTrue(outcome.isOk());
assertThat(outcome.getMessage(), startsWith("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "codeX", null, "http://vs");
assertNotNull(outcome);
assertFalse(outcome.isOk());
assertThat(outcome.getMessage(), containsString("Unknown code http://cs-np#codeX"));
assertThat(outcome.getMessage(), containsString("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
}
@Test
public void testValidateCodeInValueSet_HierarchicalAndEnumeratedValueset_HibernateSearchDisabled() {
BaseTermReadSvcImpl.setForceDisableHibernateSearchForUnitTest(true);
myValueSetDao.delete(myExtensionalVsId);
ourLog.info("Creating CodeSystem");
CodeSystem cs = new CodeSystem();
cs.setId("CodeSystem/cs");
cs.setUrl("http://cs");
cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
cs.setStatus(Enumerations.PublicationStatus.ACTIVE);
myCodeSystemDao.update(cs);
ourLog.info("Adding codes to codesystem");
CustomTerminologySet delta = new CustomTerminologySet();
TermConcept parent = delta.addRootConcept("parent");
for (int j = 0; j < 1200; j++) {
parent
.addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA)
.setCode("child" + j);
}
myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://cs", delta);
ourLog.info("Creating ValueSet");
ValueSet vs = new ValueSet();
vs.setId("ValueSet/vs");
vs.setUrl("http://vs");
vs.getCompose()
.addInclude()
.setSystem("http://cs")
.addFilter()
.setProperty("concept")
.setOp(ValueSet.FilterOperator.ISA)
.setValue("parent");
vs.getCompose()
.addInclude()
.setSystem("http://cs-np")
.addConcept(new ValueSet.ConceptReferenceComponent(new CodeType("code0")))
.addConcept(new ValueSet.ConceptReferenceComponent(new CodeType("code1")));
myValueSetDao.update(vs);
IValidationSupport.CodeValidationResult outcome;
ValidationSupportContext ctx = new ValidationSupportContext(myValidationSupport);
ConceptValidationOptions options = new ConceptValidationOptions();
// In memory - Hierarchy in existing CS
outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "child10", null, "http://vs");
assertNotNull(outcome);
assertTrue(outcome.isOk());
assertEquals("Code was validated against in-memory expansion of ValueSet: http://vs", outcome.getMessage());
outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "childX", null, "http://vs");
assertNotNull(outcome);
assertFalse(outcome.isOk());
assertEquals("Unknown code 'http://cs#childX' for in-memory expansion of ValueSet 'http://vs'", outcome.getMessage());
// In memory - Enumerated in non-present CS
outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "code1", null, "http://vs");
assertNotNull(outcome);
assertTrue(outcome.isOk());
assertEquals("Code was validated against in-memory expansion of ValueSet: http://vs", outcome.getMessage());
outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "codeX", null, "http://vs");
assertNotNull(outcome);
assertFalse(outcome.isOk());
assertEquals("Unknown code 'http://cs-np#codeX' for in-memory expansion of ValueSet 'http://vs'", outcome.getMessage());
// Precalculated
myTerminologyDeferredStorageSvc.saveAllDeferred();
myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
logAllValueSets();
myCachingValidationSupport.invalidateCaches();
outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "child10", null, "http://vs");
assertNotNull(outcome);
assertTrue(outcome.isOk());
assertThat(outcome.getMessage(), startsWith("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "childX", null, "http://vs");
assertNotNull(outcome);
assertFalse(outcome.isOk());
assertThat(outcome.getMessage(), containsString("Unknown code http://cs#childX"));
assertThat(outcome.getMessage(), containsString("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
// Precalculated - Enumerated in non-present CS
outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "code1", null, "http://vs");
assertNotNull(outcome);
assertTrue(outcome.isOk());
assertThat(outcome.getMessage(), startsWith("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "codeX", null, "http://vs");
assertNotNull(outcome);
assertFalse(outcome.isOk());
assertThat(outcome.getMessage(), containsString("Unknown code http://cs-np#codeX"));
assertThat(outcome.getMessage(), containsString("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
}
@Test
public void testValidateCodeOperationNoValueSet() {
UriType valueSetIdentifier = null;
@ -288,7 +498,8 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
myValueSetDao.expand(vs, null);
fail();
} catch (InternalErrorException e) {
assertEquals(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
assertThat(e.getMessage(), containsString(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!"));
assertThat(e.getMessage(), containsString("Performing in-memory expansion"));
}
}
@ -340,7 +551,7 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
"28571000087109",
"MODERNA COVID-19 mRNA-1273",
vs
);
);
assertTrue(outcome.isOk());
ValueSet expansion = myValueSetDao.expand(new IdType("ValueSet/vaccinecode"), new ValueSetExpansionOptions(), mySrd);
@ -349,7 +560,6 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
}
}

View File

@ -258,7 +258,7 @@ public class FhirResourceDaoR5ValueSetTest extends BaseJpaR5Test {
myValueSetDao.expand(vs, null);
fail();
} catch (InternalErrorException e) {
assertEquals(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
assertThat(e.getMessage(), containsString(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!"));
}
}

View File

@ -686,23 +686,6 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
}
@Test
public void testExpandLocalVsWithUnknownCode() {
createExternalCsAndLocalVsWithUnknownCode();
assertNotNull(myLocalValueSetId);
try {
ourClient
.operation()
.onInstance(myLocalValueSetId)
.named("expand")
.withNoParameters(Parameters.class)
.execute();
} catch (InvalidRequestException e) {
assertEquals("HTTP 400 Bad Request: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
}
}
/**
* #516
*/

View File

@ -749,7 +749,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
.withNoParameters(Parameters.class)
.execute();
} catch (InvalidRequestException e) {
assertEquals("HTTP 400 Bad Request: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
assertEquals("HTTP 400 Bad Request: HAPI-2071: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
}
}

View File

@ -661,7 +661,7 @@ public class ResourceProviderR4ValueSetVerCSNoVerTest extends BaseResourceProvid
.withNoParameters(Parameters.class)
.execute();
} catch (InvalidRequestException e) {
assertEquals("HTTP 400 Bad Request: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
assertEquals("HTTP 400 Bad Request: HAPI-2071: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
}
}

View File

@ -946,23 +946,6 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test {
}
@Test
public void testExpandLocalVsWithUnknownCode() {
createExternalCsAndLocalVsWithUnknownCode();
assertNotNull(myLocalValueSetId);
try {
myClient
.operation()
.onInstance(myLocalValueSetId)
.named("expand")
.withNoParameters(Parameters.class)
.execute();
} catch (InvalidRequestException e) {
assertEquals("HTTP 400 Bad Request: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
}
}
@Test
public void testExpandValueSetBasedOnCodeSystemWithChangedUrl() {

View File

@ -68,6 +68,7 @@ import static ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc.MAKE_LOADING_VE
import static ca.uhn.fhir.jpa.term.api.ITermLoaderSvc.LOINC_URI;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -1432,7 +1433,7 @@ public abstract class AbstractValueSetFreeTextExpansionR4Test extends BaseJpaTes
myTermSvc.expandValueSet(null, vs);
fail();
} catch (InternalErrorException e) {
assertEquals(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
assertThat(e.getMessage(), containsString(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!"));
}
// Increase the max so it won't exceed

View File

@ -246,7 +246,7 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test {
IValidationSupport.CodeValidationResult result = myValueSetDao.validateCode(new UriType("http://loinc.org/vs"), null, new StringType("10013-1-9999999999"), new StringType(ITermLoaderSvc.LOINC_URI), null, null, null, mySrd);
assertFalse(result.isOk());
assertEquals("Failed to expand ValueSet 'http://loinc.org/vs' (in-memory). Could not validate code http://loinc.org#10013-1-9999999999. Error was: " + Msg.code(702) + "null", result.getMessage());
assertEquals("Unknown code 'http://loinc.org#10013-1-9999999999' for in-memory expansion of ValueSet 'http://loinc.org/vs'", result.getMessage());
}
private Set<String> toExpandedCodes(ValueSet theExpanded) {

View File

@ -45,6 +45,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.transaction.PlatformTransactionManager;
import static ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc.MAKE_LOADING_VERSION_CURRENT;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.anyCollection;
@ -185,7 +187,7 @@ public class ValueSetExpansionR4ElasticsearchIT extends BaseJpaTest {
myTermSvc.expandValueSet(null, vs);
fail();
} catch (InternalErrorException e) {
assertEquals(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
assertThat(e.getMessage(), containsString(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!"));
}
// Increase the max so it won't exceed

View File

@ -174,15 +174,16 @@ class SearchParameterAndValueSetRuleImpl extends RuleImplOp {
if (validateCodeResult != null) {
if (validateCodeResult.isOk()) {
codeMatchCount.addMatchingCode();
theTroubleshootingLog.debug("Code {}:{} was found in VS", system, code);
theTroubleshootingLog.debug("Code {}#{} was found in ValueSet[{}] - {}", system, code, theValueSetUrl, validateCodeResult.getMessage());
if (theReturnOnFirstMatch) {
return codeMatchCount;
}
} else {
codeMatchCount.addNonMatchingCode();
theTroubleshootingLog.debug("Code {}:{} was not found in VS: {}", system, code, validateCodeResult.getMessage());
theTroubleshootingLog.debug("Code {}#{} was not found in ValueSet[{}]: {}", system, code, theValueSetUrl, validateCodeResult.getMessage());
}
} else {
theTroubleshootingLog.debug("Terminology service was unable to validate code {}#{} in ValueSet[{}] - No service was able to handle this request", system, code, theValueSetUrl);
codeMatchCount.addUnableToValidate();
}
}

View File

@ -1,5 +1,25 @@
package ca.uhn.fhir.jpa.bulk.export.api;
/*-
* #%L
* HAPI FHIR Storage api
* %%
* Copyright (C) 2014 - 2022 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import javax.transaction.Transactional;
public interface IBulkDataExportJobSchedulingHelper {

View File

@ -2,7 +2,9 @@
extraction:
java:
index:
java_version: 17
java_version: 11
maven:
version: 3.8.4
build_command:
- mvn -Dmaven.test.skip install

18
pom.xml
View File

@ -2058,8 +2058,6 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<forceJavacCompilerUse>true</forceJavacCompilerUse>
<encoding>UTF-8</encoding>
<fork>true</fork>
@ -2989,6 +2987,12 @@
</plugins>
</build>
</profile>
<!--
This profile is basically here to work around an IJ bug where the
<testSource> tag is ignored in IJ's compiler. See:
https://youtrack.jetbrains.com/issue/IDEA-85478
-->
<profile>
<id>ide</id>
<activation>
@ -3002,16 +3006,16 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>16</source>
<target>16</target>
<testSource>16</testSource>
<testTarget>16</testTarget>
<source>17</source>
<target>17</target>
<testSource>17</testSource>
<testTarget>17</testTarget>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>