Search Narrowing by Code Tweaks (#3453)

* Search narrowing tweaks

* Search narrowing tweaks

* Test

* Test fixes

* Try to fix LGTM

* Add license headers

* Test fixes

* Test fixes

hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java

@@ -25,7 +25,7 @@ public final class Msg {
 
 	/**
 	 * IMPORTANT: Please update the following comment after you add a new code
-	 * Last code value: 2070
+	 * Last code value: 2071
 	 */
 
 	private Msg() {}

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_0_0/3453-allow-isa-expansion-without-hibernate-search.yaml

@@ -0,0 +1,5 @@
+---
+type: add
+issue: 3452
+title: "The JPA server terminology service can now process IS-A filters in ValueSet expansion on servers with
+   Hibernate Search disabled."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/svc/BulkDataExportJobSchedulingHelperImpl.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.bulk.export.svc;
 
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java

@@ -250,15 +250,15 @@ public class TermValueSet implements Serializable {
 	@Override
 	public String toString() {
 		return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
-			.append("myId", myId)
+			.append("id", myId)
-			.append("myUrl", myUrl)
+			.append("url", myUrl)
-			.append(myResource != null ? ("myResource=" + myResource.toString()) : ("myResource=(null)"))
+			.append(myResource != null ? ("resource=" + myResource.toString()) : ("resource=(null)"))
-			.append("myResourcePid", myResourcePid)
+			.append("resourcePid", myResourcePid)
-			.append("myName", myName)
+			.append("name", myName)
-			.append(myConcepts != null ? ("myConcepts - size=" + myConcepts.size()) : ("myConcepts=(null)"))
+			.append(myConcepts != null ? ("concepts - size=" + myConcepts.size()) : ("concepts=(null)"))
-			.append("myTotalConcepts", myTotalConcepts)
+			.append("totalConcepts", myTotalConcepts)
-			.append("myTotalConceptDesignations", myTotalConceptDesignations)
+			.append("totalConceptDesignations", myTotalConceptDesignations)
-			.append("myExpansionStatus", myExpansionStatus)
+			.append("expansionStatus", myExpansionStatus)
 			.toString();
 	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/RawElasticJsonBuilder.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.search.autocomplete;
 
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import com.google.gson.JsonObject;
 import org.apache.commons.lang3.Validate;
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java

@@ -132,7 +132,6 @@ import org.hl7.fhir.r4.model.IntegerType;
 import org.hl7.fhir.r4.model.StringType;
 import org.hl7.fhir.r4.model.ValueSet;
 import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome;
-import org.jetbrains.annotations.NotNull;
 import org.quartz.JobExecutionContext;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
@@ -305,26 +304,32 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		}
 	}
 
-	private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, Collection<TermConceptDesignation> theDesignations) {
+	private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, Collection<TermConceptDesignation> theDesignations) {
 		if (StringUtils.isNotEmpty(theCodeSystemVersion)) {
 			if (isNoneBlank(theCodeSystem, theCode)) {
 				if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
 					theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
+					return true;
 				}
 
 				if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
 					theValueSetCodeAccumulator.excludeConcept(theCodeSystem + "|" + theCodeSystemVersion, theCode);
+					return true;
 				}
 			}
 		} else {
 			if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
 				theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
+				return true;
 			}
 
 			if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
 				theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
+				return true;
 			}
 		}
+
+		return false;
 	}
 
 	private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, String theCodeSystem, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, String theSystemVersion) {
@@ -1302,18 +1307,27 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 
 
 	private void handleFilterConceptAndCode(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
-		TermConcept code = findCode(theSystem, theFilter.getValue())
+		TermConcept code = findCodeForFilterCriteria(theSystem, theFilter);
-			.orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue()));
 
 		if (theFilter.getOp() == ValueSet.FilterOperator.ISA) {
 			ourLog.debug(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
 
 			b.must(f.match().field("myParentPids").matching("" + code.getId()));
 		} else {
-			throw new InvalidRequestException(Msg.code(894) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
+			throwInvalidFilter(theFilter, "");
 		}
 	}
 
+	@Nonnull
+	private TermConcept findCodeForFilterCriteria(String theSystem, ValueSet.ConceptSetFilterComponent theFilter) {
+		return findCode(theSystem, theFilter.getValue())
+			.orElseThrow(() -> new InvalidRequestException(Msg.code(2071) + "Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue()));
+	}
+
+	private void throwInvalidFilter(ValueSet.ConceptSetFilterComponent theFilter, String theErrorSuffix) {
+		throw new InvalidRequestException(Msg.code(894) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty() + theErrorSuffix);
+	}
+
 	private void isCodeSystemLoincOrThrowInvalidRequestException(String theSystemIdentifier, String theProperty) {
 		String systemUrl = getUrlFromIdentifier(theSystemIdentifier);
 		if (!isCodeSystemLoinc(systemUrl)) {
@@ -1499,9 +1513,27 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			Validate.isTrue(((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator).getParameter().isEmpty(), "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server.");
 		}
 
-		Validate.isTrue(theInclude.getFilter().isEmpty(), "Can not expand ValueSet with filters - Hibernate Search is not enabled on this server.");
 		Validate.isTrue(isNotBlank(theSystem), "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server.");
 
+		for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) {
+			boolean handled = false;
+			switch (nextFilter.getProperty()) {
+				case "concept":
+				case "code":
+					if (nextFilter.getOp() == ValueSet.FilterOperator.ISA) {
+						theValueSetCodeAccumulator.addMessage("Processing IS-A filter in database - Note that Hibernate Search is not enabled on this server, so this operation can be inefficient.");
+						TermConcept code = findCodeForFilterCriteria(theSystem, nextFilter);
+						addConceptAndChildren(theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, code);
+						handled = true;
+					}
+					break;
+			}
+
+			if (!handled) {
+				throwInvalidFilter(nextFilter, " - Note that Hibernate Search is disabled on this server so not all ValueSet expansion funtionality is available.");
+			}
+		}
+
 		if (theInclude.getConcept().isEmpty()) {
 
 			Collection<TermConcept> concepts = myConceptDao.fetchConceptsAndDesignationsByVersionPid(theVersion.getPid());
@@ -1531,6 +1563,15 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 
 	}
 
+	private void addConceptAndChildren(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, TermConcept theConcept) {
+		for (TermConcept nextChild : theConcept.getChildCodes()) {
+			boolean added = addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), nextChild.getCode(), nextChild.getDisplay(), nextChild.getId(), nextChild.getParentPidsAsString(), nextChild.getDesignations());
+			if (added) {
+				addConceptAndChildren(theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, nextChild);
+			}
+		}
+	}
+
 	@Override
 	@Transactional
 	public String invalidatePreCalculatedExpansion(IIdType theValueSetId, RequestDetails theRequestDetails) {
@@ -1925,21 +1966,21 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 				expandValueSet(options, valueSet, accumulator);
 
 				// We are done with this ValueSet.
-				txTemplate.execute(t -> {
+				txTemplate.executeWithoutResult(t -> {
 					valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANDED);
 					valueSetToExpand.setExpansionTimestamp(new Date());
 					myTermValueSetDao.saveAndFlush(valueSetToExpand);
-					return null;
+
 				});
 
 				ourLog.info("Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", valueSet.getId(), valueSet.getUrl(), accumulator.getConceptsSaved(), sw);
 
 			} catch (Exception e) {
 				ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e);
-				txTemplate.execute(t -> {
+				txTemplate.executeWithoutResult(t -> {
 					valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.FAILED_TO_EXPAND);
 					myTermValueSetDao.saveAndFlush(valueSetToExpand);
-					return null;
+
 				});
 			}
 		}
@@ -2181,7 +2222,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 				return result;
 
 			} else {
-				return null;
+				return new LookupCodeResult()
+					.setFound(false);
 			}
 		});
 	}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithConceptAccumulator.java

@@ -189,17 +189,25 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
 		Integer capacityRemaining = getCapacityRemaining();
 		if (capacityRemaining == 0) {
 			String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myMaxCapacity);
+			msg = appendAccumulatorMessages(msg);
 			throw new ExpansionTooCostlyException(Msg.code(831) + msg);
 		}
 
 		if (myHardExpansionMaximumSize > 0 && myAddedConcepts > myHardExpansionMaximumSize) {
 			String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myHardExpansionMaximumSize);
+			msg = appendAccumulatorMessages(msg);
 			throw new ExpansionTooCostlyException(Msg.code(832) + msg);
 		}
 
 		myAddedConcepts++;
 	}
 
+	@Nonnull
+	private String appendAccumulatorMessages(String msg) {
+		msg += getMessages().stream().map(t->" - " + t).collect(Collectors.joining());
+		return msg;
+	}
+
 	public Integer getTotalConcepts() {
 		return myTotalConcepts;
 	}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/custom/CustomTerminologySet.java

@@ -62,8 +62,8 @@ public class CustomTerminologySet {
 		myRootConcepts = theRootConcepts;
 	}
 
-	public void addRootConcept(String theCode) {
+	public TermConcept addRootConcept(String theCode) {
-		addRootConcept(theCode, null);
+		return addRootConcept(theCode, null);
 	}
 
 	public TermConcept addRootConcept(String theCode, String theDisplay) {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java

@@ -25,6 +25,7 @@ import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
 import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
 import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
 import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao;
+import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao;
 import ca.uhn.fhir.jpa.entity.TermConcept;
 import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
 import ca.uhn.fhir.jpa.entity.TermConceptProperty;
@@ -187,6 +188,8 @@ public abstract class BaseJpaTest extends BaseTest {
 	@Autowired
 	protected ITermValueSetConceptDao myTermValueSetConceptDao;
 	@Autowired
+	protected ITermValueSetDao myTermValueSetDao;
+	@Autowired
 	protected ITermConceptDesignationDao myTermConceptDesignationDao;
 	@Autowired
 	protected ITermConceptPropertyDao myTermConceptPropertyDao;
@@ -355,6 +358,14 @@ public abstract class BaseJpaTest extends BaseTest {
 		});
 	}
 
+	protected int logAllValueSets() {
+		return runInTransaction(() -> {
+			List<TermValueSet> valueSets = myTermValueSetDao.findAll();
+			ourLog.info("ValueSets:\n * {}", valueSets.stream().map(t -> t.toString()).collect(Collectors.joining("\n * ")));
+			return valueSets.size();
+		});
+	}
+
 	protected int logAllForcedIds() {
 		return runInTransaction(() -> {
 			List<ForcedId> forcedIds = myForcedIdDao.findAll();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java

@@ -49,6 +49,7 @@ import java.util.Set;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.containsStringIgnoringCase;
 import static org.hamcrest.Matchers.empty;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -729,7 +730,8 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test {
 			myObservationDao.search(params).size();
 			fail();
 		} catch (InternalErrorException e) {
-			assertEquals(Msg.code(885) + "Expansion of ValueSet produced too many codes (maximum 1) - Operation aborted!", e.getMessage());
+			assertThat(e.getMessage(), containsString(Msg.code(885) + "Expansion of ValueSet produced too many codes (maximum 1) - Operation aborted!"));
+
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java

@@ -49,6 +49,7 @@ import java.util.stream.Collectors;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.containsStringIgnoringCase;
 import static org.hamcrest.Matchers.empty;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -873,7 +874,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
 			myObservationDao.search(params).size();
 			fail();
 		} catch (InternalErrorException e) {
-			assertEquals(Msg.code(885) + "Expansion of ValueSet produced too many codes (maximum 1) - Operation aborted!", e.getMessage());
+			assertThat(e.getMessage(), containsString(Msg.code(885) + "Expansion of ValueSet produced too many codes (maximum 1) - Operation aborted!"));
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValueSetTest.java

@@ -6,6 +6,9 @@ import ca.uhn.fhir.context.support.ValidationSupportContext;
 import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
 import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
+import ca.uhn.fhir.jpa.entity.TermConcept;
+import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
+import ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl;
 import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
@@ -29,9 +32,11 @@ import java.io.IOException;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.startsWith;
 import static org.hamcrest.Matchers.stringContainsInOrder;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -43,6 +48,7 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
 
 	@AfterEach
 	public void after() {
+		BaseTermReadSvcImpl.setForceDisableHibernateSearchForUnitTest(false);
 		myDaoConfig.setPreExpandValueSets(new DaoConfig().isPreExpandValueSets());
 		myDaoConfig.setMaximumExpansionSize(new DaoConfig().getMaximumExpansionSize());
 	}
@@ -91,6 +97,210 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
 	}
 
 
+	@Test
+	public void testValidateCodeInValueSet_HierarchicalAndEnumeratedValueset() {
+		myValueSetDao.delete(myExtensionalVsId);
+
+		ourLog.info("Creating CodeSystem");
+		CodeSystem cs = new CodeSystem();
+		cs.setId("CodeSystem/cs");
+		cs.setUrl("http://cs");
+		cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
+		cs.setStatus(Enumerations.PublicationStatus.ACTIVE);
+		myCodeSystemDao.update(cs);
+
+		ourLog.info("Adding codes to codesystem");
+		CustomTerminologySet delta = new CustomTerminologySet();
+		TermConcept parent = delta.addRootConcept("parent");
+		for (int j = 0; j < 1200; j++) {
+			parent
+				.addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA)
+				.setCode("child" + j);
+		}
+		myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://cs", delta);
+
+		ourLog.info("Creating ValueSet");
+		ValueSet vs = new ValueSet();
+		vs.setId("ValueSet/vs");
+		vs.setUrl("http://vs");
+		vs.getCompose()
+			.addInclude()
+			.setSystem("http://cs")
+			.addFilter()
+			.setProperty("concept")
+			.setOp(ValueSet.FilterOperator.ISA)
+			.setValue("parent");
+		vs.getCompose()
+			.addInclude()
+			.setSystem("http://cs-np")
+			.addConcept(new ValueSet.ConceptReferenceComponent(new CodeType("code0")))
+			.addConcept(new ValueSet.ConceptReferenceComponent(new CodeType("code1")));
+		myValueSetDao.update(vs);
+
+		IValidationSupport.CodeValidationResult outcome;
+		ValidationSupportContext ctx = new ValidationSupportContext(myValidationSupport);
+		ConceptValidationOptions options = new ConceptValidationOptions();
+
+		// In memory - Hierarchy in existing CS
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "child10", null, "http://vs");
+		assertNotNull(outcome);
+		assertTrue(outcome.isOk());
+		assertEquals("Code was validated against in-memory expansion of ValueSet: http://vs", outcome.getMessage());
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "childX", null, "http://vs");
+		assertNotNull(outcome);
+		assertFalse(outcome.isOk());
+		assertEquals("Unknown code 'http://cs#childX' for in-memory expansion of ValueSet 'http://vs'", outcome.getMessage());
+
+		// In memory - Enumerated in non-present CS
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "code1", null, "http://vs");
+		assertNotNull(outcome);
+		assertTrue(outcome.isOk());
+		assertEquals("Code was validated against in-memory expansion of ValueSet: http://vs", outcome.getMessage());
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "codeX", null, "http://vs");
+		assertNotNull(outcome);
+		assertFalse(outcome.isOk());
+		assertEquals("Unknown code 'http://cs-np#codeX' for in-memory expansion of ValueSet 'http://vs'", outcome.getMessage());
+
+		// Precalculated
+
+		myTerminologyDeferredStorageSvc.saveAllDeferred();
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+		logAllValueSets();
+		myCachingValidationSupport.invalidateCaches();
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "child10", null, "http://vs");
+		assertNotNull(outcome);
+		assertTrue(outcome.isOk());
+		assertThat(outcome.getMessage(), startsWith("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "childX", null, "http://vs");
+		assertNotNull(outcome);
+		assertFalse(outcome.isOk());
+		assertThat(outcome.getMessage(), containsString("Unknown code http://cs#childX"));
+		assertThat(outcome.getMessage(), containsString("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
+
+		// Precalculated - Enumerated in non-present CS
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "code1", null, "http://vs");
+		assertNotNull(outcome);
+		assertTrue(outcome.isOk());
+		assertThat(outcome.getMessage(), startsWith("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "codeX", null, "http://vs");
+		assertNotNull(outcome);
+		assertFalse(outcome.isOk());
+		assertThat(outcome.getMessage(), containsString("Unknown code http://cs-np#codeX"));
+		assertThat(outcome.getMessage(), containsString("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
+
+	}
+
+	@Test
+	public void testValidateCodeInValueSet_HierarchicalAndEnumeratedValueset_HibernateSearchDisabled() {
+		BaseTermReadSvcImpl.setForceDisableHibernateSearchForUnitTest(true);
+
+		myValueSetDao.delete(myExtensionalVsId);
+
+		ourLog.info("Creating CodeSystem");
+		CodeSystem cs = new CodeSystem();
+		cs.setId("CodeSystem/cs");
+		cs.setUrl("http://cs");
+		cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
+		cs.setStatus(Enumerations.PublicationStatus.ACTIVE);
+		myCodeSystemDao.update(cs);
+
+		ourLog.info("Adding codes to codesystem");
+		CustomTerminologySet delta = new CustomTerminologySet();
+		TermConcept parent = delta.addRootConcept("parent");
+		for (int j = 0; j < 1200; j++) {
+			parent
+				.addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA)
+				.setCode("child" + j);
+		}
+		myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://cs", delta);
+
+		ourLog.info("Creating ValueSet");
+		ValueSet vs = new ValueSet();
+		vs.setId("ValueSet/vs");
+		vs.setUrl("http://vs");
+		vs.getCompose()
+			.addInclude()
+			.setSystem("http://cs")
+			.addFilter()
+			.setProperty("concept")
+			.setOp(ValueSet.FilterOperator.ISA)
+			.setValue("parent");
+		vs.getCompose()
+			.addInclude()
+			.setSystem("http://cs-np")
+			.addConcept(new ValueSet.ConceptReferenceComponent(new CodeType("code0")))
+			.addConcept(new ValueSet.ConceptReferenceComponent(new CodeType("code1")));
+		myValueSetDao.update(vs);
+
+		IValidationSupport.CodeValidationResult outcome;
+		ValidationSupportContext ctx = new ValidationSupportContext(myValidationSupport);
+		ConceptValidationOptions options = new ConceptValidationOptions();
+
+		// In memory - Hierarchy in existing CS
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "child10", null, "http://vs");
+		assertNotNull(outcome);
+		assertTrue(outcome.isOk());
+		assertEquals("Code was validated against in-memory expansion of ValueSet: http://vs", outcome.getMessage());
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "childX", null, "http://vs");
+		assertNotNull(outcome);
+		assertFalse(outcome.isOk());
+		assertEquals("Unknown code 'http://cs#childX' for in-memory expansion of ValueSet 'http://vs'", outcome.getMessage());
+
+		// In memory - Enumerated in non-present CS
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "code1", null, "http://vs");
+		assertNotNull(outcome);
+		assertTrue(outcome.isOk());
+		assertEquals("Code was validated against in-memory expansion of ValueSet: http://vs", outcome.getMessage());
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "codeX", null, "http://vs");
+		assertNotNull(outcome);
+		assertFalse(outcome.isOk());
+		assertEquals("Unknown code 'http://cs-np#codeX' for in-memory expansion of ValueSet 'http://vs'", outcome.getMessage());
+
+		// Precalculated
+
+		myTerminologyDeferredStorageSvc.saveAllDeferred();
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+		logAllValueSets();
+		myCachingValidationSupport.invalidateCaches();
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "child10", null, "http://vs");
+		assertNotNull(outcome);
+		assertTrue(outcome.isOk());
+		assertThat(outcome.getMessage(), startsWith("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs", "childX", null, "http://vs");
+		assertNotNull(outcome);
+		assertFalse(outcome.isOk());
+		assertThat(outcome.getMessage(), containsString("Unknown code http://cs#childX"));
+		assertThat(outcome.getMessage(), containsString("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
+
+		// Precalculated - Enumerated in non-present CS
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "code1", null, "http://vs");
+		assertNotNull(outcome);
+		assertTrue(outcome.isOk());
+		assertThat(outcome.getMessage(), startsWith("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
+
+		outcome = myValidationSupport.validateCode(ctx, options, "http://cs-np", "codeX", null, "http://vs");
+		assertNotNull(outcome);
+		assertFalse(outcome.isOk());
+		assertThat(outcome.getMessage(), containsString("Unknown code http://cs-np#codeX"));
+		assertThat(outcome.getMessage(), containsString("Code validation occurred using a ValueSet expansion that was pre-calculated at "));
+
+	}
+
 	@Test
 	public void testValidateCodeOperationNoValueSet() {
 		UriType valueSetIdentifier = null;
@@ -288,7 +498,8 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
 			myValueSetDao.expand(vs, null);
 			fail();
 		} catch (InternalErrorException e) {
-			assertEquals(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
+			assertThat(e.getMessage(), containsString(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!"));
+			assertThat(e.getMessage(), containsString("Performing in-memory expansion"));
 		}
 	}
 
@@ -340,7 +551,7 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
 			"28571000087109",
 			"MODERNA COVID-19 mRNA-1273",
 			vs
-			);
+		);
 		assertTrue(outcome.isOk());
 
 		ValueSet expansion = myValueSetDao.expand(new IdType("ValueSet/vaccinecode"), new ValueSetExpansionOptions(), mySrd);
@@ -349,7 +560,6 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
 	}
 
 
-
 }
 
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoR5ValueSetTest.java

@@ -258,7 +258,7 @@ public class FhirResourceDaoR5ValueSetTest extends BaseJpaR5Test {
 			myValueSetDao.expand(vs, null);
 			fail();
 		} catch (InternalErrorException e) {
-			assertEquals(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
+			assertThat(e.getMessage(), containsString(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!"));
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java

@@ -686,23 +686,6 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 
 	}
 
-	@Test
-	public void testExpandLocalVsWithUnknownCode() {
-		createExternalCsAndLocalVsWithUnknownCode();
-		assertNotNull(myLocalValueSetId);
-
-		try {
-			ourClient
-				.operation()
-				.onInstance(myLocalValueSetId)
-				.named("expand")
-				.withNoParameters(Parameters.class)
-				.execute();
-		} catch (InvalidRequestException e) {
-			assertEquals("HTTP 400 Bad Request: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
-		}
-	}
-
 	/**
 	 * #516
 	 */

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ValueSetNoVerCSNoVerTest.java

@@ -749,7 +749,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 				.withNoParameters(Parameters.class)
 				.execute();
 		} catch (InvalidRequestException e) {
-			assertEquals("HTTP 400 Bad Request: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
+			assertEquals("HTTP 400 Bad Request: HAPI-2071: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ValueSetVerCSNoVerTest.java

@@ -661,7 +661,7 @@ public class ResourceProviderR4ValueSetVerCSNoVerTest extends BaseResourceProvid
 				.withNoParameters(Parameters.class)
 				.execute();
 		} catch (InvalidRequestException e) {
-			assertEquals("HTTP 400 Bad Request: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
+			assertEquals("HTTP 400 Bad Request: HAPI-2071: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java

@@ -946,23 +946,6 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test {
 
 	}
 
-	@Test
-	public void testExpandLocalVsWithUnknownCode() {
-		createExternalCsAndLocalVsWithUnknownCode();
-		assertNotNull(myLocalValueSetId);
-
-		try {
-			myClient
-				.operation()
-				.onInstance(myLocalValueSetId)
-				.named("expand")
-				.withNoParameters(Parameters.class)
-				.execute();
-		} catch (InvalidRequestException e) {
-			assertEquals("HTTP 400 Bad Request: Invalid filter criteria - code does not exist: {http://example.com/my_code_system}childFOOOOOOO", e.getMessage());
-		}
-	}
-
 	@Test
 	public void testExpandValueSetBasedOnCodeSystemWithChangedUrl() {
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/AbstractValueSetFreeTextExpansionR4Test.java

@@ -68,6 +68,7 @@ import static ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc.MAKE_LOADING_VE
 import static ca.uhn.fhir.jpa.term.api.ITermLoaderSvc.LOINC_URI;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -1432,7 +1433,7 @@ public abstract class AbstractValueSetFreeTextExpansionR4Test extends BaseJpaTes
 				myTermSvc.expandValueSet(null, vs);
 				fail();
 			} catch (InternalErrorException e) {
-				assertEquals(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
+				assertThat(e.getMessage(), containsString(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!"));
 			}
 
 			// Increase the max so it won't exceed

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationDstu3Test.java

@@ -246,7 +246,7 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test {
 
 		IValidationSupport.CodeValidationResult result = myValueSetDao.validateCode(new UriType("http://loinc.org/vs"), null, new StringType("10013-1-9999999999"), new StringType(ITermLoaderSvc.LOINC_URI), null, null, null, mySrd);
 		assertFalse(result.isOk());
-		assertEquals("Failed to expand ValueSet 'http://loinc.org/vs' (in-memory). Could not validate code http://loinc.org#10013-1-9999999999. Error was: " + Msg.code(702) + "null", result.getMessage());
+		assertEquals("Unknown code 'http://loinc.org#10013-1-9999999999' for in-memory expansion of ValueSet 'http://loinc.org/vs'", result.getMessage());
 	}
 
 	private Set<String> toExpandedCodes(ValueSet theExpanded) {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4ElasticsearchIT.java

@@ -45,6 +45,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import org.springframework.transaction.PlatformTransactionManager;
 
 import static ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc.MAKE_LOADING_VERSION_CURRENT;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.ArgumentMatchers.anyCollection;
@@ -185,7 +187,7 @@ public class ValueSetExpansionR4ElasticsearchIT extends BaseJpaTest {
 			myTermSvc.expandValueSet(null, vs);
 			fail();
 		} catch (InternalErrorException e) {
-			assertEquals(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
+			assertThat(e.getMessage(), containsString(Msg.code(832) + "Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!"));
 		}
 
 		// Increase the max so it won't exceed

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/SearchParameterAndValueSetRuleImpl.java

@@ -174,15 +174,16 @@ class SearchParameterAndValueSetRuleImpl extends RuleImplOp {
 						if (validateCodeResult != null) {
 							if (validateCodeResult.isOk()) {
 								codeMatchCount.addMatchingCode();
-								theTroubleshootingLog.debug("Code {}:{} was found in VS", system, code);
+								theTroubleshootingLog.debug("Code {}#{} was found in ValueSet[{}] - {}", system, code, theValueSetUrl, validateCodeResult.getMessage());
 								if (theReturnOnFirstMatch) {
 									return codeMatchCount;
 								}
 							} else {
 								codeMatchCount.addNonMatchingCode();
-								theTroubleshootingLog.debug("Code {}:{} was not found in VS: {}", system, code, validateCodeResult.getMessage());
+								theTroubleshootingLog.debug("Code {}#{} was not found in ValueSet[{}]: {}", system, code, theValueSetUrl, validateCodeResult.getMessage());
 							}
 						} else {
+							theTroubleshootingLog.debug("Terminology service was unable to validate code {}#{} in ValueSet[{}] - No service was able to handle this request", system, code, theValueSetUrl);
 							codeMatchCount.addUnableToValidate();
 						}
 					}

hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/api/IBulkDataExportJobSchedulingHelper.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.bulk.export.api;
 
+/*-
+ * #%L
+ * HAPI FHIR Storage api
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import javax.transaction.Transactional;
 
 public interface IBulkDataExportJobSchedulingHelper {

lgtm.yml

@@ -2,7 +2,9 @@
 extraction:
   java:
     index:
-      java_version: 17
+      java_version: 11
       maven: 
         version: 3.8.4
+      build_command:
+        - mvn -Dmaven.test.skip install
 

pom.xml

@@ -2058,8 +2058,6 @@
 					<artifactId>maven-compiler-plugin</artifactId>
 					<version>3.8.1</version>
 					<configuration>
-						<source>1.8</source>
-						<target>1.8</target>
 						<forceJavacCompilerUse>true</forceJavacCompilerUse>
 						<encoding>UTF-8</encoding>
 						<fork>true</fork>
@@ -2989,6 +2987,12 @@
 				</plugins>
 			</build>
 		</profile>
+
+		<!--
+		This profile is basically here to work around an IJ bug where the
+		<testSource> tag is ignored in IJ's compiler. See:
+		https://youtrack.jetbrains.com/issue/IDEA-85478
+		-->
 		<profile>
 			<id>ide</id>
 			<activation>
@@ -3002,16 +3006,16 @@
 					<plugin>
 						<groupId>org.apache.maven.plugins</groupId>
 						<artifactId>maven-compiler-plugin</artifactId>
-						<version>3.8.1</version>
 						<configuration>
-							<source>16</source>
+							<source>17</source>
-							<target>16</target>
+							<target>17</target>
-							<testSource>16</testSource>
+							<testSource>17</testSource>
-							<testTarget>16</testTarget>
+							<testTarget>17</testTarget>
 						</configuration>
 					</plugin>
 				</plugins>
 			</build>
 		</profile>
+
 	</profiles>
 </project>