Allow search narrowing and auth interceptor by `token:in` (#3360)

* Optmize valueset expansion

* Working

* Version bump

* Add documentation

* Add test

* Checkstyle message cleanup

* Add reverse rule

* Test ficx

* Test fix

* Test fix

* Test fixes

* Test fixes

* Test fix

* Update hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/security/search_narrowing_interceptor.md

Co-authored-by: Ken Stevens <khstevens@gmail.com>

* Update hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/security/search_narrowing_interceptor.md

Co-authored-by: Ken Stevens <khstevens@gmail.com>

* Test fixes

* Fix conflict

* Add setter

* Test fixes

Co-authored-by: Ken Stevens <khstevens@gmail.com>

hapi-deployable-pom/pom.xml

@@ -4,7 +4,7 @@
     <parent>
 		 <groupId>ca.uhn.hapi.fhir</groupId>
 		 <artifactId>hapi-fhir</artifactId>
-		 <version>6.0.0-PRE1-SNAPSHOT</version>
+		 <version>6.0.0-PRE2-SNAPSHOT</version>
 		 <relativePath>../pom.xml</relativePath>
 	 </parent>
 

hapi-fhir-android/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-base/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-base/src/main/java/ca/uhn/fhir/context/support/IValidationSupport.java

@@ -246,7 +246,7 @@ public interface IValidationSupport {
 	 * @return Returns a validation result object
 	 */
 	@Nullable
-	default CodeValidationResult validateCode(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, String theValueSetUrl) {
+	default CodeValidationResult validateCode(@Nonnull ValidationSupportContext theValidationSupportContext, @Nonnull ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, String theValueSetUrl) {
 		return null;
 	}
 

hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java

@@ -25,7 +25,7 @@ public final class Msg {
 
 	/**
 	 * IMPORTANT: Please update the following comment after you add a new code
-	 * Last code value: 2024
+	 * Last code value: 2031
 	 */
 
 	private Msg() {}

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/Constants.java

@@ -218,6 +218,7 @@ public class Constants {
 	public static final String PARAMQUALIFIER_TOKEN_TEXT = ":text";
 	public static final String PARAMQUALIFIER_MDM = ":mdm";
 	public static final String PARAMQUALIFIER_TOKEN_OF_TYPE = ":of-type";
+	public static final String PARAMQUALIFIER_TOKEN_NOT = ":not";
 	public static final int STATUS_HTTP_200_OK = 200;
 	public static final int STATUS_HTTP_201_CREATED = 201;
 	public static final int STATUS_HTTP_204_NO_CONTENT = 204;
@@ -291,6 +292,10 @@ public class Constants {
 	public static final String SUBSCRIPTION_MULTITYPE_STAR = "*";
 	public static final String SUBSCRIPTION_STAR_CRITERIA = SUBSCRIPTION_MULTITYPE_PREFIX + SUBSCRIPTION_MULTITYPE_STAR + SUBSCRIPTION_MULTITYPE_SUFFIX;
 	public static final String INCLUDE_STAR = "*";
+	public static final String PARAMQUALIFIER_TOKEN_IN = ":in";
+	public static final String PARAMQUALIFIER_TOKEN_NOT_IN = ":not-in";
+	public static final String PARAMQUALIFIER_TOKEN_ABOVE = ":above";
+	public static final String PARAMQUALIFIER_TOKEN_BELOW = ":below";
 
 	static {
 		CHARSET_UTF8 = StandardCharsets.UTF_8;

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/UrlUtil.java

@@ -346,10 +346,6 @@ public class UrlUtil {
 		String url = theUrl;
 		UrlParts retVal = new UrlParts();
 		if (url.startsWith("http")) {
-			if (url.startsWith("/")) {
-				url = url.substring(1);
-			}
-
 			int qmIdx = url.indexOf('?');
 			if (qmIdx != -1) {
 				retVal.setParams(defaultIfBlank(url.substring(qmIdx + 1), null));
@@ -372,10 +368,7 @@ public class UrlUtil {
 			}
 		}
 
-		if (url.length() > 1 && url.charAt(0) == '/' && Character.isLetter(url.charAt(1)) && url.contains("?")) {
+		int nextStart = parsingStart;
-			url = url.substring(1);
-		}
-		int nextStart = 0;
 		boolean nextIsHistory = false;
 
 		for (int idx = parsingStart; idx < url.length(); idx++) {

hapi-fhir-batch/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-bom/pom.xml

@@ -3,14 +3,14 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>ca.uhn.hapi.fhir</groupId>
     <artifactId>hapi-fhir-bom</artifactId>
-    <version>6.0.0-PRE1-SNAPSHOT</version>
+    <version>6.0.0-PRE2-SNAPSHOT</version>
     <packaging>pom</packaging>
 	 <name>HAPI FHIR BOM</name>
 
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
     

hapi-fhir-checkstyle/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-checkstyle/src/main/java/ca/uhn/fhir/checks/HapiErrorCodeCheck.java

@@ -7,8 +7,8 @@ import com.puppycrawl.tools.checkstyle.api.TokenTypes;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.HashSet;
+import java.util.HashMap;
-import java.util.Set;
+import java.util.Map;
 
 /**
  * mvn -P CI,ALLMODULES checkstyle:check
@@ -17,7 +17,7 @@ import java.util.Set;
 public final class HapiErrorCodeCheck extends AbstractCheck {
 	private static final Logger ourLog = LoggerFactory.getLogger(HapiErrorCodeCheck.class);
 
-	private static final Set<Integer> ourCodesUsed = new HashSet<>();
+	private static final Map<Integer, String> ourCodesUsed = new HashMap<>();
 
 	@Override
 	public int[] getDefaultTokens() {
@@ -68,10 +68,14 @@ public final class HapiErrorCodeCheck extends AbstractCheck {
 			DetailAST numberNode = msgNode.getParent().getNextSibling().getFirstChild().getFirstChild();
 			if (TokenTypes.NUM_INT == numberNode.getType()) {
 				Integer code = Integer.valueOf(numberNode.getText());
-				if (!ourCodesUsed.add(code)) {
+				if (ourCodesUsed.containsKey(code)) {
 					log(theAst.getLineNo(), "Two different exception messages call Msg.code(" +
 						code +
-						").  Each thrown exception throw call Msg.code() with a different code.");
+						").  Each thrown exception throw call Msg.code() with a different code. " +
+						"Previously found at: " + ourCodesUsed.get(code));
+				} else {
+					String location = getFileContents().getFileName() + ":" + instantiation.getLineNo() + ":" + instantiation.getColumnNo() + "(" + code + ")";
+					ourCodesUsed.put(code, location);
 				}
 			} else {
 				log(theAst.getLineNo(), "Called Msg.code() with a non-integer argument");

hapi-fhir-checkstyle/src/test/java/ca/uhn/fhir/checks/HapiErrorCodeCheckTest.java

@@ -18,6 +18,7 @@ import java.util.Arrays;
 import java.util.List;
 
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.startsWith;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -47,7 +48,8 @@ class HapiErrorCodeCheckTest {
 		assertThat(errorLines[0], startsWith("[ERROR] "));
 		assertThat(errorLines[0], endsWith("BadClass.java:7: Exception thrown that does not call Msg.code() [HapiErrorCode]"));
 		assertThat(errorLines[1], startsWith("[ERROR] "));
-		assertThat(errorLines[1], endsWith("BadClass.java:11: Two different exception messages call Msg.code(2).  Each thrown exception throw call Msg.code() with a different code. [HapiErrorCode]"));
+		assertThat(errorLines[1], containsString("BadClass.java:11: Two different exception messages call Msg.code(2).  Each thrown exception throw call Msg.code() with a different code."));
+		assertThat(errorLines[1], containsString("BadClass.java:9:9"));
 	}
 
 	private Checker buildChecker() throws CheckstyleException {

hapi-fhir-cli/hapi-fhir-cli-api/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-cli/hapi-fhir-cli-app/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-cli</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../../hapi-deployable-pom</relativePath>
 	</parent>
 

hapi-fhir-cli/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-client-okhttp/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-client/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-converter/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-dist/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-docs/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-docs/src/main/java/ca/uhn/hapi/fhir/docs/AuthorizationInterceptors.java

@@ -255,7 +255,7 @@ public class AuthorizationInterceptors {
 
 			} else {
 
-				throw new AuthenticationException(Msg.code(645) + "Unknown bearer token");
+				throw new AuthenticationException("Unknown bearer token");
 
 			}
 
@@ -265,4 +265,38 @@ public class AuthorizationInterceptors {
 	//END SNIPPET: narrowing
 
 
+	//START SNIPPET: narrowingByCode
+	public class MyCodeSearchNarrowingInterceptor extends SearchNarrowingInterceptor {
+
+		/**
+		 * This method must be overridden to provide the list of compartments
+		 * and/or resources that the current user should have access to
+		 */
+		@Override
+		protected AuthorizedList buildAuthorizedList(RequestDetails theRequestDetails) {
+			// Process authorization header - The following is a fake
+			// implementation. Obviously we'd want something more real
+			// for a production scenario.
+			String authHeader = theRequestDetails.getHeader("Authorization");
+			if ("Bearer dfw98h38r".equals(authHeader)) {
+
+				return new AuthorizedList()
+					// When searching for Observations, narrow the search to only include Observations
+					// with a code indicating that it is a Vital Signs Observation
+					.addCodeInValueSet("Observation", "code", "http://hl7.org/fhir/ValueSet/observation-vitalsignresult")
+					// When searching for Encounters, narrow the search to exclude Encounters where
+					// the Encounter class is in a ValueSet containing forbidden class codes
+					.addCodeNotInValueSet("Encounter", "class", "http://my-forbidden-encounter-classes");
+
+			} else {
+
+				throw new AuthenticationException("Unknown bearer token");
+
+			}
+
+		}
+
+	}
+	//END SNIPPET: narrowingByCode
+
 }

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_0_0/3360-add-support-for-not-in-search.yaml

@@ -0,0 +1,6 @@
+---
+type: add
+issue: 3360
+title: "Support has been added to the JPA server for token `:not-in` queries. Similar to `:not` queries, resources will
+  currently be considered to match if any codes in the relevant resource field are not found in the given ValueSet (as
+  opposed to matching if *all* codes are not in the given ValueSet)."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_0_0/3360-add-support-for-vs-membership-in-authorization-interceptor.yaml

@@ -0,0 +1,5 @@
+---
+type: add
+issue: 3360
+title: "SearchNarrowingInterceptor can now be used to automatically narrow searches to include a `code:in` or `code:not-in`
+  expression, for mandating that results must be in a specified list of codes."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_0_0/3360-add-support-for-vs-membership-in-search-narrowing-interceptor.yaml

@@ -0,0 +1,4 @@
+---
+type: add
+issue: 3360
+title: "The SearchNarrowingInterceptor can now narrow searches to require a `token:in` or `token:not-in` parameter."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_0_0/3360-improve-vs-expand-performance.yaml

@@ -0,0 +1,5 @@
+---
+type: add
+issue: 3360
+title: "Performance for JPA Server ValueSet expansion has been significantly optimized
+   in order to minimize database lookups, especially with large expansions."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/security/search_narrowing_interceptor.md

@@ -1,11 +1,11 @@
 # Search Narrowing Interceptor
 
-HAPI FHIR 3.7.0 introduced a new interceptor, the [SearchNarrowingInterceptor](/hapi-fhir/apidocs/hapi-fhir-server/ca/uhn/fhir/rest/server/interceptor/auth/SearchNarrowingInterceptor.html).
+The [SearchNarrowingInterceptor](/hapi-fhir/apidocs/hapi-fhir-server/ca/uhn/fhir/rest/server/interceptor/auth/SearchNarrowingInterceptor.html) can be used to automatically narrow or constrain the scope of FHIR searches.
 
 * [SearchNarrowingInterceptor JavaDoc](/apidocs/hapi-fhir-server/ca/uhn/fhir/rest/server/interceptor/auth/SearchNarrowingInterceptor.html)
 * [SearchNarrowingInterceptor Source](https://github.com/hapifhir/hapi-fhir/blob/master/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/SearchNarrowingInterceptor.java)
 
-This interceptor is designed to be used in conjunction with AuthorizationInterceptor. It uses a similar strategy where a dynamic list is built up for each request, but the purpose of this interceptor is to modify client searches that are received (after HAPI FHIR received the HTTP request, but before the search is actually performed) to restrict the search to only search for specific resources or compartments that the user has access to.
+This interceptor is designed to be used in conjunction with the [Authorization Interceptor](./authorization_interceptor.html). It uses a similar strategy where a dynamic list is built up for each request, but the purpose of this interceptor is to modify client searches that are received (after HAPI FHIR receives the HTTP request, but before the search is actually performed) to restrict the search to only search for specific resources or compartments that the user has access to.
 
 This could be used, for example, to allow the user to perform a search for:
 
@@ -25,3 +25,15 @@ An example of this interceptor follows:
 {{snippet:classpath:/ca/uhn/hapi/fhir/docs/AuthorizationInterceptors.java|narrowing}}
 ``` 
 
+# Constraining by ValueSet Membership
+
+SearchNarrowingInterceptor can also be used to narrow searches by automatically appending `token:in` and `token:not-in` parameters.
+
+In the example below, searches are narrowed as shown below:
+
+* Searches for http://localhost:8000/Observation become http://localhost:8000/Observation?code:in=http://hl7.org/fhir/ValueSet/observation-vitalsignresult
+* Searches for http://localhost:8000/Encounter become http://localhost:8000/Encounter?class:not-in=http://my-forbidden-encounter-classes
+
+```java
+{{snippet:classpath:/ca/uhn/hapi/fhir/docs/AuthorizationInterceptors.java|narrowingByCode}}
+``` 

hapi-fhir-jacoco/pom.xml

@@ -11,7 +11,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jaxrsserver-base/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpa/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>

hapi-fhir-jpaserver-base/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java

@@ -16,7 +16,6 @@ import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter;
 import ca.uhn.fhir.jpa.batch.config.BatchConstants;
 import ca.uhn.fhir.jpa.batch.config.NonPersistedBatchConfigurer;
 import ca.uhn.fhir.jpa.batch.job.PartitionedUrlValidator;
-import ca.uhn.fhir.jpa.batch.mdm.MdmBatchJobSubmitterFactoryImpl;
 import ca.uhn.fhir.jpa.batch.mdm.MdmClearJobSubmitterImpl;
 import ca.uhn.fhir.jpa.batch.reader.BatchResourceSearcher;
 import ca.uhn.fhir.jpa.batch.svc.BatchJobSubmitterImpl;
@@ -68,7 +67,6 @@ import ca.uhn.fhir.jpa.entity.Search;
 import ca.uhn.fhir.jpa.graphql.DaoRegistryGraphQLStorageServices;
 import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor;
 import ca.uhn.fhir.jpa.interceptor.JpaConsentContextServices;
-import ca.uhn.fhir.jpa.interceptor.MdmSearchExpandingInterceptor;
 import ca.uhn.fhir.jpa.interceptor.OverridePathBasedReferentialIntegrityForDeletesInterceptor;
 import ca.uhn.fhir.jpa.interceptor.validation.RepositoryValidatingRuleBuilder;
 import ca.uhn.fhir.jpa.model.sched.ISchedulerService;
@@ -140,7 +138,6 @@ import ca.uhn.fhir.jpa.term.api.ITermConceptMappingSvc;
 import ca.uhn.fhir.jpa.util.MemoryCacheService;
 import ca.uhn.fhir.jpa.validation.JpaResourceLoader;
 import ca.uhn.fhir.jpa.validation.ValidationSettings;
-import ca.uhn.fhir.mdm.api.IMdmBatchJobSubmitterFactory;
 import ca.uhn.fhir.mdm.api.IMdmClearJobSubmitter;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter;
@@ -155,7 +152,6 @@ import org.hibernate.jpa.HibernatePersistenceProvider;
 import org.hl7.fhir.common.hapi.validation.support.UnknownCodeSystemWarningValidationSupport;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.utilities.graphql.IGraphQLStorageServices;
-import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.npm.PackageClient;
 import org.springframework.batch.core.configuration.JobRegistry;
 import org.springframework.batch.core.configuration.annotation.BatchConfigurer;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java

@@ -84,7 +84,6 @@ public abstract class BaseConfigDstu3Plus extends BaseConfig {
 	@Primary
 	@Bean
 	public IValidationSupport validationSupportChain() {
-
 		// Short timeout for code translation because TermConceptMappingSvcImpl has its own caching
 		CachingValidationSupport.CacheTimeouts cacheTimeouts = CachingValidationSupport.CacheTimeouts.defaultValues()
 			.setTranslateCodeMillis(1000);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/r4/BaseR4Config.java

@@ -52,8 +52,6 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
 @EnableTransactionManagement
 public class BaseR4Config extends BaseConfigDstu3Plus {
 
-	public static FhirContext ourFhirContext = FhirContext.forR4();
-
 	@Override
 	public FhirContext fhirContext() {
 		return fhirContextR4();
@@ -68,7 +66,7 @@ public class BaseR4Config extends BaseConfigDstu3Plus {
 	@Bean
 	@Primary
 	public FhirContext fhirContextR4() {
-		FhirContext retVal = ourFhirContext;
+		FhirContext retVal = FhirContext.forR4();
 
 		// Don't strip versions in some places
 		ParserOptions parserOptions = retVal.getParserOptions();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptDao.java

@@ -9,6 +9,7 @@ import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 
@@ -34,6 +35,16 @@ import java.util.Optional;
 
 public interface ITermConceptDao extends JpaRepository<TermConcept, Long>, IHapiFhirJpaRepository {
 
+	@Query("SELECT t FROM TermConcept t " +
+		"LEFT JOIN FETCH t.myDesignations d " +
+		"WHERE t.myId IN :pids")
+	List<TermConcept> fetchConceptsAndDesignationsByPid(@Param("pids") List<Long> thePids);
+
+	@Query("SELECT t FROM TermConcept t " +
+		"LEFT JOIN FETCH t.myDesignations d " +
+		"WHERE t.myCodeSystemVersionPid = :pid")
+	List<TermConcept> fetchConceptsAndDesignationsByVersionPid(@Param("pid") Long theCodeSystemVersionPid);
+
 	@Query("SELECT COUNT(t) FROM TermConcept t WHERE t.myCodeSystem.myId = :cs_pid")
 	Integer countByCodeSystemVersion(@Param("cs_pid") Long thePid);
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptDesignation.java

@@ -21,6 +21,8 @@ package ca.uhn.fhir.jpa.entity;
  */
 
 import ca.uhn.fhir.util.ValidateUtil;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 
 import javax.annotation.Nonnull;
 import javax.persistence.Column;
@@ -146,4 +148,17 @@ public class TermConceptDesignation implements Serializable {
 	public Long getPid() {
 		return myId;
 	}
+
+	@Override
+	public String toString() {
+		return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
+			.append("conceptPid", myConcept.getId())
+			.append("pid", myId)
+			.append("language", myLanguage)
+			.append("useSystem", myUseSystem)
+			.append("useCode", myUseCode)
+			.append("useDisplay", myUseDisplay)
+			.append("value", myValue)
+			.toString();
+	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptProperty.java

@@ -218,6 +218,7 @@ public class TermConceptProperty implements Serializable {
 	@Override
 	public String toString() {
 		return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
+			.append("conceptPid", myConcept.getId())
 			.append("key", myKey)
 			.append("value", getValue())
 			.toString();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java

@@ -238,16 +238,17 @@ public class TermValueSetConcept implements Serializable {
 	@Override
 	public String toString() {
 		return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
-			.append("myId", myId)
+			.append("id", myId)
-			.append(myValueSet != null ? ("myValueSet - id=" + myValueSet.getId()) : ("myValueSet=(null)"))
+			.append("order", myOrder)
-			.append("myValueSetPid", myValueSetPid)
+			.append("system", mySystem)
-			.append("myOrder", myOrder)
+			.append("code", myCode)
-			.append("myValueSetUrl", this.getValueSetUrl())
+			.append("valueSet", myValueSet != null ? myValueSet.getId() : "(null)")
-			.append("myValueSetName", this.getValueSetName())
+			.append("valueSetPid", myValueSetPid)
-			.append("mySystem", mySystem)
+			.append("valueSetUrl", this.getValueSetUrl())
-			.append("myCode", myCode)
+			.append("valueSetName", this.getValueSetName())
-			.append("myDisplay", myDisplay)
+			.append("display", myDisplay)
-			.append(myDesignations != null ? ("myDesignations - size=" + myDesignations.size()) : ("myDesignations=(null)"))
+			.append("designationCount", myDesignations != null ? myDesignations.size() : "(null)")
+			.append("parentPids", mySourceConceptDirectParentPids)
 			.toString();
 	}
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java

@@ -483,7 +483,7 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 				byte[] bytes = Files.readAllBytes(Paths.get(new URI(thePackageUrl)));
 				return bytes;
 			} catch (IOException | URISyntaxException e) {
-				throw new InternalErrorException(Msg.code(2024) + "Error loading \"" + thePackageUrl + "\": " + e.getMessage());
+				throw new InternalErrorException(Msg.code(2031) + "Error loading \"" + thePackageUrl + "\": " + e.getMessage());
 			}
 		} else {
 			HttpClientConnectionManager connManager = new BasicHttpClientConnectionManager();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/ValueSetOperationProvider.java

@@ -20,11 +20,11 @@ package ca.uhn.fhir.jpa.provider;
  * #L%
  */
 
-import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.context.support.ConceptValidationOptions;
 import ca.uhn.fhir.context.support.IValidationSupport;
 import ca.uhn.fhir.context.support.ValidationSupportContext;
 import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
+import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet;
@@ -37,9 +37,13 @@ import ca.uhn.fhir.rest.annotation.IdParam;
 import ca.uhn.fhir.rest.annotation.Operation;
 import ca.uhn.fhir.rest.annotation.OperationParam;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
+import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
+import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.rest.server.provider.ProviderConstants;
 import ca.uhn.fhir.util.ParametersUtil;
+import ca.uhn.fhir.util.UrlUtil;
 import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain;
 import org.hl7.fhir.instance.model.api.IBaseParameters;
 import org.hl7.fhir.instance.model.api.IBaseResource;
@@ -68,8 +72,14 @@ public class ValueSetOperationProvider extends BaseJpaProvider {
 	@Qualifier(BaseConfig.JPA_VALIDATION_SUPPORT_CHAIN)
 	private ValidationSupportChain myValidationSupportChain;
 	@Autowired
+	private IValidationSupport myValidationSupport;
+	@Autowired
 	private IFulltextSearchSvc myFulltextSearch;
 
+	public void setValidationSupport(IValidationSupport theValidationSupport) {
+		myValidationSupport = theValidationSupport;
+	}
+
 	public void setDaoConfig(DaoConfig theDaoConfig) {
 		myDaoConfig = theDaoConfig;
 	}
@@ -136,17 +146,34 @@ public class ValueSetOperationProvider extends BaseJpaProvider {
 		try {
 
 			IFhirResourceDaoValueSet<IBaseResource, ICompositeType, ICompositeType> dao = getDao();
+
+			IBaseResource valueSet = theValueSet;
 			if (haveId) {
-				return dao.expand(theId, options, theRequestDetails);
+				valueSet = dao.read(theId, theRequestDetails);
 			} else if (haveIdentifier) {
+				String url;
 				if (haveValueSetVersion) {
-					return dao.expandByIdentifier(theUrl.getValue() + "|" + theValueSetVersion.getValue(), options);
+					url = theUrl.getValue() + "|" + theValueSetVersion.getValue();
+					valueSet = myValidationSupport.fetchValueSet(url);
 				} else {
-					return dao.expandByIdentifier(theUrl.getValue(), options);
+					url = theUrl.getValue();
+					valueSet = myValidationSupport.fetchValueSet(url);
+				}
+				if (valueSet == null) {
+					throw new ResourceNotFoundException(Msg.code(2030) + "Can not find ValueSet with URL: " + UrlUtil.escapeUrlParam(url));
 				}
-			} else {
-				return dao.expand(theValueSet, options);
 			}
+
+			IValidationSupport.ValueSetExpansionOutcome outcome = myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), options, valueSet);
+			if (outcome == null) {
+				throw new InternalErrorException(Msg.code(2028) + outcome.getError());
+			}
+			if (outcome.getError() != null) {
+				throw new PreconditionFailedException(Msg.code(2029) + outcome.getError());
+			}
+
+			return outcome.getValueSet();
+
 		} finally {
 			endRequest(theServletRequest);
 		}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/TokenAutocompleteAggregation.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.search.autocomplete;
 
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneClauseBuilder;
 import com.google.gson.Gson;
 import com.google.gson.JsonArray;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/TokenAutocompleteHit.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.search.autocomplete;
 
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.util.TerserUtil;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/TokenAutocompleteSearch.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.search.autocomplete;
 
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneClauseBuilder;
@@ -80,7 +100,7 @@ class TokenAutocompleteSearch {
 							break;
 						case "":
 						default:
-							throw new IllegalArgumentException(Msg.code(2023) + "Autocomplete only accepts text search for now.");
+							throw new IllegalArgumentException(Msg.code(2027) + "Autocomplete only accepts text search for now.");
 
 					}
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/ValueSetAutocompleteOptions.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.search.autocomplete;
 
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/ValueSetAutocompleteSearch.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.search.autocomplete;
 
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.param.TokenParam;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/package-info.java

@@ -15,3 +15,23 @@
  *
  */
 package ca.uhn.fhir.jpa.search.autocomplete;
+
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/TokenPredicateBuilder.java

@@ -20,11 +20,17 @@ package ca.uhn.fhir.jpa.search.builder.predicate;
  * #L%
  */
 
-import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
 import ca.uhn.fhir.context.BaseRuntimeDeclaredChildDefinition;
+import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.context.FhirVersionEnum;
+import ca.uhn.fhir.context.RuntimeResourceDefinition;
 import ca.uhn.fhir.context.RuntimeSearchParam;
+import ca.uhn.fhir.context.support.IValidationSupport;
+import ca.uhn.fhir.context.support.ValidationSupportContext;
 import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
+import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.interceptor.model.RequestPartitionId;
 import ca.uhn.fhir.jpa.dao.LegacySearchBuilder;
 import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser;
@@ -43,20 +49,23 @@ import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.param.TokenParamModifier;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException;
+import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.util.FhirVersionIndependentConcept;
+import ca.uhn.fhir.util.UrlUtil;
 import com.google.common.collect.Sets;
 import com.healthmarketscience.sqlbuilder.BinaryCondition;
 import com.healthmarketscience.sqlbuilder.Condition;
-import com.healthmarketscience.sqlbuilder.InCondition;
 import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn;
-import org.apache.commons.lang3.StringUtils;
+import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.instance.model.api.IPrimitiveType;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -76,10 +85,14 @@ public class TokenPredicateBuilder extends BaseSearchParamPredicateBuilder {
 	private final DbColumn myColumnSystem;
 	private final DbColumn myColumnValue;
 
+	@Autowired
+	private IValidationSupport myValidationSupport;
 	@Autowired
 	private ITermReadSvc myTerminologySvc;
 	@Autowired
 	private ModelConfig myModelConfig;
+	@Autowired
+	private FhirContext myContext;
 
 	/**
 	 * Constructor
@@ -165,8 +178,20 @@ public class TokenPredicateBuilder extends BaseSearchParamPredicateBuilder {
 			 * Process token modifiers (:in, :below, :above)
 			 */
 
-			if (modifier == TokenParamModifier.IN) {
+			if (modifier == TokenParamModifier.IN || modifier == TokenParamModifier.NOT_IN) {
-				codes.addAll(myTerminologySvc.expandValueSetIntoConceptList(null, code));
+				if (myContext.getVersion().getVersion().isNewerThan(FhirVersionEnum.DSTU2)) {
+					IBaseResource valueSet = myValidationSupport.fetchValueSet(code);
+					if (valueSet == null) {
+						throw new ResourceNotFoundException(Msg.code(2024) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(code));
+					}
+					IValidationSupport.ValueSetExpansionOutcome expanded = myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), new ValueSetExpansionOptions(), valueSet);
+					codes.addAll(extractValueSetCodes(expanded.getValueSet()));
+				} else {
+					codes.addAll(myTerminologySvc.expandValueSetIntoConceptList(null, code));
+				}
+				if (modifier == TokenParamModifier.NOT_IN) {
+					operation = SearchFilterParser.CompareOperation.ne;
+				}
 			} else if (modifier == TokenParamModifier.ABOVE) {
 				system = determineSystemIfMissing(theSearchParam, code, system);
 				validateHaveSystemAndCodeForToken(paramName, code, system);
@@ -235,6 +260,46 @@ public class TokenPredicateBuilder extends BaseSearchParamPredicateBuilder {
 		return predicate;
 	}
 
+	private List<FhirVersionIndependentConcept> extractValueSetCodes(IBaseResource theValueSet) {
+		List<FhirVersionIndependentConcept> retVal = new ArrayList<>();
+
+		RuntimeResourceDefinition vsDef = myContext.getResourceDefinition("ValueSet");
+		BaseRuntimeChildDefinition expansionChild = vsDef.getChildByName("expansion");
+		Optional<IBase> expansionOpt = expansionChild.getAccessor().getFirstValueOrNull(theValueSet);
+		if (expansionOpt.isPresent()) {
+			IBase expansion = expansionOpt.get();
+			BaseRuntimeElementCompositeDefinition<?> expansionDef = (BaseRuntimeElementCompositeDefinition<?>) myContext.getElementDefinition(expansion.getClass());
+			BaseRuntimeChildDefinition containsChild = expansionDef.getChildByName("contains");
+			List<IBase> contains = containsChild.getAccessor().getValues(expansion);
+				
+			BaseRuntimeChildDefinition.IAccessor systemAccessor = null;
+			BaseRuntimeChildDefinition.IAccessor codeAccessor = null;
+			for (IBase nextContains : contains) {
+				if (systemAccessor == null) {
+					systemAccessor = myContext.getElementDefinition(nextContains.getClass()).getChildByName("system").getAccessor();
+				}
+				if (codeAccessor == null) {
+					codeAccessor = myContext.getElementDefinition(nextContains.getClass()).getChildByName("code").getAccessor();
+				}
+				String system = systemAccessor
+					.getFirstValueOrNull(nextContains)
+					.map(t->(IPrimitiveType<?>)t)
+					.map(t->t.getValueAsString())
+					.orElse(null);
+				String code = codeAccessor
+					.getFirstValueOrNull(nextContains)
+					.map(t->(IPrimitiveType<?>)t)
+					.map(t->t.getValueAsString())
+					.orElse(null);
+				if (isNotBlank(system) && isNotBlank(code)) {
+					retVal.add(new FhirVersionIndependentConcept(system, code));
+				}
+			}
+		}
+
+		return retVal;
+	}
+
 	private String determineSystemIfMissing(RuntimeSearchParam theSearchParam, String code, String theSystem) {
 		String retVal = theSystem;
 		if (retVal == null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java

@@ -20,12 +20,12 @@ package ca.uhn.fhir.jpa.term;
  * #L%
  */
 
-import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.support.ConceptValidationOptions;
 import ca.uhn.fhir.context.support.IValidationSupport;
 import ca.uhn.fhir.context.support.ValidationSupportContext;
 import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
+import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.dao.IDao;
@@ -100,6 +100,7 @@ import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep;
 import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory;
 import org.hibernate.search.engine.search.query.SearchQuery;
 import org.hibernate.search.mapper.orm.Search;
+import org.hibernate.search.mapper.orm.common.EntityReference;
 import org.hibernate.search.mapper.orm.session.SearchSession;
 import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService;
 import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport;
@@ -157,7 +158,6 @@ import javax.persistence.criteria.Join;
 import javax.persistence.criteria.JoinType;
 import javax.persistence.criteria.Predicate;
 import javax.persistence.criteria.Root;
-import javax.validation.constraints.NotNull;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -197,6 +197,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 	private static final ValueSetExpansionOptions DEFAULT_EXPANSION_OPTIONS = new ValueSetExpansionOptions();
 	private static final TermCodeSystemVersion NO_CURRENT_VERSION = new TermCodeSystemVersion().setId(-1L);
 	private static Runnable myInvokeOnNextCallForUnitTest;
+	private static boolean ourForceDisableHibernateSearchForUnitTest;
+
 	private final Cache<String, TermCodeSystemVersion> myCodeSystemCurrentVersionCache = Caffeine.newBuilder().expireAfterWrite(1, TimeUnit.MINUTES).build();
 	@Autowired
 	protected DaoRegistry myDaoRegistry;
@@ -230,6 +232,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 	@Autowired
 	private PlatformTransactionManager myTxManager;
 	@Autowired
+	private ITermConceptDao myTermConceptDao;
+	@Autowired
 	private ITermValueSetConceptViewDao myTermValueSetConceptViewDao;
 	@Autowired
 	private ITermValueSetConceptViewOracleDao myTermValueSetConceptViewOracleDao;
@@ -261,18 +265,21 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		return cs != null;
 	}
 
-	private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, TermConcept theConcept, boolean theAdd, String theValueSetIncludeVersion) {
+	private boolean addCodeIfNotAlreadyAdded(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, TermConcept theConcept, boolean theAdd, String theValueSetIncludeVersion) {
 		String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri();
 		String codeSystemVersion = theConcept.getCodeSystemVersion().getCodeSystemVersionId();
 		String code = theConcept.getCode();
 		String display = theConcept.getDisplay();
 		Long sourceConceptPid = theConcept.getId();
+		String directParentPids = "";
 
-		String directParentPids = theConcept
+		if (theExpansionOptions != null && theExpansionOptions.isIncludeHierarchy()) {
-			.getParents()
+			directParentPids = theConcept
-			.stream()
+				.getParents()
-			.map(t -> t.getParent().getId().toString())
+				.stream()
-			.collect(Collectors.joining(" "));
+				.map(t -> t.getParent().getId().toString())
+				.collect(Collectors.joining(" "));
+		}
 
 		Collection<TermConceptDesignation> designations = theConcept.getDesignations();
 		if (StringUtils.isNotEmpty(theValueSetIncludeVersion)) {
@@ -282,11 +289,11 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		}
 	}
 
-	private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids) {
+	private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, Collection<TermConceptDesignation> theDesignations) {
 		if (StringUtils.isNotEmpty(theCodeSystemVersion)) {
 			if (isNoneBlank(theCodeSystem, theCode)) {
 				if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
-					theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, null, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
+					theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
 				}
 
 				if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
@@ -295,7 +302,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			}
 		} else {
 			if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
-				theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, null, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
+				theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
 			}
 
 			if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
@@ -560,8 +567,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			String systemVersion = conceptView.getConceptSystemVersion();
 
 			//-- this is quick solution, may need to revisit
-			if (!applyFilter(display, filterDisplayValue))
+			if (!applyFilter(display, filterDisplayValue)) {
-				continue;
+				continue;}
 
 			Long conceptPid = conceptView.getConceptPid();
 			if (!pidToConcept.containsKey(conceptPid)) {
@@ -793,7 +800,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system);
 			if (cs != null) {
 
-				return expandValueSetHandleIncludeOrExcludeUsingDatabase(theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, theAdd, theQueryIndex, theExpansionFilter, system, cs);
+				return expandValueSetHandleIncludeOrExcludeUsingDatabase(theExpansionOptions, theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, theAdd, theQueryIndex, theExpansionFilter, system, cs);
 
 			} else {
 
@@ -854,11 +861,11 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 	}
 
 	private boolean isHibernateSearchEnabled() {
-		return myFulltextSearchSvc != null;
+		return myFulltextSearchSvc != null && !ourForceDisableHibernateSearchForUnitTest;
 	}
 
 	@Nonnull
-	private Boolean expandValueSetHandleIncludeOrExcludeUsingDatabase(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter, String theSystem, TermCodeSystem theCs) {
+	private Boolean expandValueSetHandleIncludeOrExcludeUsingDatabase(ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter, String theSystem, TermCodeSystem theCs) {
 		String includeOrExcludeVersion = theIncludeOrExclude.getVersion();
 		TermCodeSystemVersion csv;
 		if (isEmpty(includeOrExcludeVersion)) {
@@ -948,11 +955,20 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		StopWatch swForBatch = new StopWatch();
 		AtomicInteger countForBatch = new AtomicInteger(0);
 
-		SearchQuery<TermConcept> termConceptsQuery = searchSession.search(TermConcept.class)
+		SearchQuery<EntityReference> termConceptsQuery = searchSession
-			.where(f -> finishedQuery).toQuery();
+			.search(TermConcept.class)
+			.selectEntityReference()
+			.where(f -> finishedQuery)
+			.toQuery();
 
 		ourLog.trace("About to query: {}", termConceptsQuery.queryString());
-		List<TermConcept> termConcepts = termConceptsQuery.fetchHits(theQueryIndex * maxResultsPerBatch, maxResultsPerBatch);
+		List<EntityReference> termConceptRefs = termConceptsQuery.fetchHits(theQueryIndex * maxResultsPerBatch, maxResultsPerBatch);
+		List<Long> pids = termConceptRefs
+			.stream()
+			.map(t -> (Long) t.id())
+			.collect(Collectors.toList());
+
+		List<TermConcept> termConcepts = myTermConceptDao.fetchConceptsAndDesignationsByPid(pids);
 
 		// If the include section had multiple codes, return the codes in the same order
 		if (codes.size() > 1) {
@@ -980,7 +996,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 					concept.setDisplay(theIncludeConcept.getDisplay());
 				}
 			}
-			boolean added = addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, includeOrExcludeVersion);
+			boolean added = addCodeIfNotAlreadyAdded(theExpansionOptions, theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, includeOrExcludeVersion);
 			if (added) {
 				delta++;
 			}
@@ -1259,7 +1275,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		}
 	}
 
-
 	private void isCodeSystemLoincOrThrowInvalidRequestException(String theSystemIdentifier, String theProperty) {
 		String systemUrl = getUrlFromIdentifier(theSystemIdentifier);
 		if (!isCodeSystemLoinc(systemUrl)) {
@@ -1287,7 +1302,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		bool.must(f.phrase().field("myDisplay").matching(nextFilter.getValue()));
 	}
 
-
 	private void addDisplayFilterInexact(SearchPredicateFactory f, BooleanPredicateClausesStep<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
 		bool.must(f.phrase()
 			.field("myDisplay").boost(4.0f)
@@ -1328,7 +1342,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		}
 	}
 
-
 	private void addLoincFilterDescendantEqual(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
 		addLoincFilterDescendantEqual(theSystem, f, b, theFilter.getProperty(), theFilter.getValue());
 	}
@@ -1358,7 +1371,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		return theTerms.stream().map(Term::text).map(Long::valueOf).collect(Collectors.toList());
 	}
 
-
 	private List<Term> getDescendantTerms(String theSystem, String theProperty, String theValue) {
 		List<Term> retVal = new ArrayList<>();
 
@@ -1376,7 +1388,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		return retVal;
 	}
 
-
 	private void logFilteringValueOnProperty(String theValue, String theProperty) {
 		ourLog.debug(" * Filtering with value={} on property {}", theValue, theProperty);
 	}
@@ -1400,8 +1411,10 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		Validate.isTrue(isNotBlank(theSystem), "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server.");
 
 		if (theInclude.getConcept().isEmpty()) {
-			for (TermConcept next : theVersion.getConcepts()) {
+
-				addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), next.getId(), next.getParentPidsAsString());
+			Collection<TermConcept> concepts = myConceptDao.fetchConceptsAndDesignationsByVersionPid(theVersion.getPid());
+			for (TermConcept next : concepts) {
+				addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), next.getId(), next.getParentPidsAsString(), next.getDesignations());
 			}
 		}
 
@@ -1409,7 +1422,18 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			if (!theSystem.equals(theInclude.getSystem()) && isNotBlank(theSystem)) {
 				continue;
 			}
-			addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), null, null);
+			Collection<TermConceptDesignation> designations = next
+				.getDesignation()
+				.stream()
+				.map(t->new TermConceptDesignation()
+					.setValue(t.getValue())
+					.setLanguage(t.getLanguage())
+					.setUseCode(t.getUse().getCode())
+					.setUseSystem(t.getUse().getSystem())
+					.setUseDisplay(t.getUse().getDisplay())
+				)
+				.collect(Collectors.toList());
+			addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), null, null, designations);
 		}
 
 
@@ -1442,7 +1466,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetPreExpansionInvalidated", termValueSet.getUrl(), totalConcepts);
 	}
 
-
 	@Override
 	@Transactional
 	public boolean isValueSetPreExpandedForCodeValidation(ValueSet theValueSet) {
@@ -1744,7 +1767,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		mySchedulerService.scheduleClusteredJob(10 * DateUtils.MILLIS_PER_MINUTE, vsJobDefinition);
 	}
 
-
 	@Override
 	public synchronized void preExpandDeferredValueSetsToTerminologyTables() {
 		if (!myDaoConfig.isEnableTaskPreExpandValueSets()) {
@@ -1781,7 +1803,9 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 				assert valueSet != null;
 
 				ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator(valueSetToExpand, myTermValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao);
-				expandValueSet(null, valueSet, accumulator);
+				ValueSetExpansionOptions options = new ValueSetExpansionOptions();
+				options.setIncludeHierarchy(true);
+				expandValueSet(options, valueSet, accumulator);
 
 				// We are done with this ValueSet.
 				txTemplate.execute(t -> {
@@ -2045,7 +2069,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		});
 	}
 
-
 	@Nullable
 	private ConceptSubsumptionOutcome testForSubsumption(SearchSession theSearchSession, TermConcept theLeft, TermConcept theRight, ConceptSubsumptionOutcome theOutput) {
 		List<TermConcept> fetch = theSearchSession.search(TermConcept.class)
@@ -2346,7 +2369,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		return codeSystemValidateCode(codeSystemUrl, theVersion, code, display);
 	}
 
-
 	/**
 	 * When the search is for unversioned loinc system it uses the forcedId to obtain the current
 	 * version, as it is not necessarily the last  one anymore.
@@ -2356,7 +2378,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 	public Optional<TermValueSet> findCurrentTermValueSet(String theUrl) {
 		if (TermReadSvcUtil.isLoincUnversionedValueSet(theUrl)) {
 			Optional<String> vsIdOpt = TermReadSvcUtil.getValueSetId(theUrl);
-			if (! vsIdOpt.isPresent()) {
+			if (!vsIdOpt.isPresent()) {
 				return Optional.empty();
 			}
 
@@ -2371,7 +2393,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		return Optional.of(termValueSetList.get(0));
 	}
 
-
 	@SuppressWarnings("unchecked")
 	private CodeValidationResult codeSystemValidateCode(String theCodeSystemUrl, String theCodeSystemVersion, String theCode, String theDisplay) {
 
@@ -2436,7 +2457,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 				"where f.myResourceType = 'CodeSystem' and f.myForcedId = '" + theForcedId + "'").getResultList();
 		if (resultList.isEmpty()) return Optional.empty();
 
-		if (resultList.size() > 1) throw new NonUniqueResultException(Msg.code(911) + "More than one CodeSystem is pointed by forcedId: " + theForcedId + ". Was constraint "
+		if (resultList.size() > 1)
+			throw new NonUniqueResultException(Msg.code(911) + "More than one CodeSystem is pointed by forcedId: " + theForcedId + ". Was constraint "
 				+ ForcedId.IDX_FORCEDID_TYPE_FID + " removed?");
 
 		IFhirResourceDao<CodeSystem> csDao = myDaoRegistry.getResourceDao("CodeSystem");
@@ -2444,14 +2466,9 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		return Optional.of(cs);
 	}
 
-	public static class Job implements HapiJob {
+	@VisibleForTesting
-		@Autowired
+	public static void setForceDisableHibernateSearchForUnitTest(boolean theForceDisableHibernateSearchForUnitTest) {
-		private ITermReadSvc myTerminologySvc;
+		ourForceDisableHibernateSearchForUnitTest = theForceDisableHibernateSearchForUnitTest;
-
-		@Override
-		public void execute(JobExecutionContext theContext) {
-			myTerminologySvc.preExpandDeferredValueSetsToTerminologyTables();
-		}
 	}
 
 	static boolean isPlaceholder(DomainResource theResource) {
@@ -2540,12 +2557,22 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		return termConcept;
 	}
 
-    static boolean isDisplayLanguageMatch(String theReqLang, String theStoredLang) {
+	static boolean isDisplayLanguageMatch(String theReqLang, String theStoredLang) {
 		// NOTE: return the designation when one of then is not specified.
 		if (theReqLang == null || theStoredLang == null)
 			return true;
 
 		return theReqLang.equalsIgnoreCase(theStoredLang);
-    }
+	}
+
+	public static class Job implements HapiJob {
+		@Autowired
+		private ITermReadSvc myTerminologySvc;
+
+		@Override
+		public void execute(JobExecutionContext theContext) {
+			myTerminologySvc.preExpandDeferredValueSetsToTerminologyTables();
+		}
+	}
 
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/JpaValidationSupportChain.java

@@ -56,6 +56,9 @@ public class JpaValidationSupportChain extends ValidationSupportChain {
 	@Autowired
 	private UnknownCodeSystemWarningValidationSupport myUnknownCodeSystemWarningValidationSupport;
 
+	/**
+	 * Constructor
+	 */
 	public JpaValidationSupportChain(FhirContext theFhirContext) {
 		myFhirContext = theFhirContext;
 	}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java

@@ -20,8 +20,14 @@ import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamTokenDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
+import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
+import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
+import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
+import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao;
 import ca.uhn.fhir.jpa.dao.index.IdHelperService;
 import ca.uhn.fhir.jpa.entity.TermConcept;
+import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
+import ca.uhn.fhir.jpa.entity.TermConceptProperty;
 import ca.uhn.fhir.jpa.entity.TermValueSet;
 import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
 import ca.uhn.fhir.jpa.entity.TermValueSetConceptDesignation;
@@ -32,7 +38,6 @@ import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.partition.IPartitionLookupSvc;
 import ca.uhn.fhir.jpa.provider.SystemProviderDstu2Test;
 import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
-import ca.uhn.fhir.jpa.search.HapiLuceneAnalysisConfigurer;
 import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider;
 import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc;
 import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc;
@@ -62,11 +67,7 @@ import org.apache.commons.io.IOUtils;
 import org.hibernate.HibernateException;
 import org.hibernate.Session;
 import org.hibernate.SessionFactory;
-import org.hibernate.search.backend.lucene.cfg.LuceneBackendSettings;
-import org.hibernate.search.backend.lucene.cfg.LuceneIndexSettings;
-import org.hibernate.search.engine.cfg.BackendSettings;
 import org.hibernate.search.mapper.orm.Search;
-import org.hibernate.search.mapper.orm.cfg.HibernateOrmMapperSettings;
 import org.hibernate.search.mapper.orm.session.SearchSession;
 import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator;
 import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
@@ -105,10 +106,8 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.TimeZone;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -179,6 +178,18 @@ public abstract class BaseJpaTest extends BaseTest {
 	protected IResourceIndexedSearchParamDateDao myResourceIndexedSearchParamDateDao;
 	@Autowired
 	protected IResourceIndexedComboTokensNonUniqueDao myResourceIndexedComboTokensNonUniqueDao;
+	@Autowired(required = false)
+	protected IFulltextSearchSvc myFulltestSearchSvc;
+	@Autowired(required = false)
+	protected BatchJobHelper myBatchJobHelper;
+	@Autowired
+	protected ITermConceptDao myTermConceptDao;
+	@Autowired
+	protected ITermValueSetConceptDao myTermValueSetConceptDao;
+	@Autowired
+	protected ITermConceptDesignationDao myTermConceptDesignationDao;
+	@Autowired
+	protected ITermConceptPropertyDao myTermConceptPropertyDao;
 	@Autowired
 	private IdHelperService myIdHelperService;
 	@Autowired
@@ -197,10 +208,6 @@ public abstract class BaseJpaTest extends BaseTest {
 	@Autowired
 	private IForcedIdDao myForcedIdDao;
 	@Autowired(required = false)
-	protected IFulltextSearchSvc myFulltestSearchSvc;
-	@Autowired(required = false)
-	protected BatchJobHelper myBatchJobHelper;
-	@Autowired(required = false)
 	private JobExecutionDao myMapJobExecutionDao;
 	@Autowired(required = false)
 	private JobInstanceDao myMapJobInstanceDao;
@@ -308,33 +315,6 @@ public abstract class BaseJpaTest extends BaseTest {
 		});
 	}
 
-	@SuppressWarnings("BusyWait")
-	public static void waitForSize(int theTarget, List<?> theList) {
-		StopWatch sw = new StopWatch();
-		while (theList.size() != theTarget && sw.getMillis() <= 16000) {
-			try {
-				Thread.sleep(50);
-			} catch (InterruptedException theE) {
-				throw new Error(theE);
-			}
-		}
-		if (sw.getMillis() >= 16000 || theList.size() > theTarget) {
-			String describeResults = theList
-				.stream()
-				.map(t -> {
-					if (t == null) {
-						return "null";
-					}
-					if (t instanceof IBaseResource) {
-						return ((IBaseResource) t).getIdElement().getValue();
-					}
-					return t.toString();
-				})
-				.collect(Collectors.joining(", "));
-			fail("Size " + theList.size() + " is != target " + theTarget + " - Got: " + describeResults);
-		}
-	}
-
 	protected int logAllResources() {
 		return runInTransaction(() -> {
 			List<ResourceTable> resources = myResourceTableDao.findAll();
@@ -343,6 +323,38 @@ public abstract class BaseJpaTest extends BaseTest {
 		});
 	}
 
+	protected int logAllConceptDesignations() {
+		return runInTransaction(() -> {
+			List<TermConceptDesignation> resources = myTermConceptDesignationDao.findAll();
+			ourLog.info("Concept Designations:\n * {}", resources.stream().map(t -> t.toString()).collect(Collectors.joining("\n * ")));
+			return resources.size();
+		});
+	}
+
+	protected int logAllConceptProperties() {
+		return runInTransaction(() -> {
+			List<TermConceptProperty> resources = myTermConceptPropertyDao.findAll();
+			ourLog.info("Concept Designations:\n * {}", resources.stream().map(t -> t.toString()).collect(Collectors.joining("\n * ")));
+			return resources.size();
+		});
+	}
+
+	protected int logAllConcepts() {
+		return runInTransaction(() -> {
+			List<TermConcept> resources = myTermConceptDao.findAll();
+			ourLog.info("Concepts:\n * {}", resources.stream().map(t -> t.toString()).collect(Collectors.joining("\n * ")));
+			return resources.size();
+		});
+	}
+
+	protected int logAllValueSetConcepts() {
+		return runInTransaction(() -> {
+			List<TermValueSetConcept> resources = myTermValueSetConceptDao.findAll();
+			ourLog.info("Concepts:\n * {}", resources.stream().map(t -> t.toString()).collect(Collectors.joining("\n * ")));
+			return resources.size();
+		});
+	}
+
 	protected int logAllForcedIds() {
 		return runInTransaction(() -> {
 			List<ForcedId> forcedIds = myForcedIdDao.findAll();
@@ -375,7 +387,7 @@ public abstract class BaseJpaTest extends BaseTest {
 			String message = myResourceIndexedSearchParamStringDao
 				.findAll()
 				.stream()
-				.filter(t->theParamNames.length == 0 ? true : Arrays.asList(theParamNames).contains(t.getParamName()))
+				.filter(t -> theParamNames.length == 0 ? true : Arrays.asList(theParamNames).contains(t.getParamName()))
 				.map(t -> t.toString())
 				.collect(Collectors.joining("\n * "));
 			ourLog.info("String indexes{}:\n * {}", messageSuffix, message);
@@ -649,6 +661,62 @@ public abstract class BaseJpaTest extends BaseTest {
 		}
 	}
 
+	protected TermValueSetConceptDesignation assertTermConceptContainsDesignation(TermValueSetConcept theConcept, String theLanguage, String theUseSystem, String theUseCode, String theUseDisplay, String theDesignationValue) {
+		Stream<TermValueSetConceptDesignation> stream = theConcept.getDesignations().stream();
+		if (theLanguage != null) {
+			stream = stream.filter(designation -> theLanguage.equalsIgnoreCase(designation.getLanguage()));
+		}
+		if (theUseSystem != null) {
+			stream = stream.filter(designation -> theUseSystem.equalsIgnoreCase(designation.getUseSystem()));
+		}
+		if (theUseCode != null) {
+			stream = stream.filter(designation -> theUseCode.equalsIgnoreCase(designation.getUseCode()));
+		}
+		if (theUseDisplay != null) {
+			stream = stream.filter(designation -> theUseDisplay.equalsIgnoreCase(designation.getUseDisplay()));
+		}
+		if (theDesignationValue != null) {
+			stream = stream.filter(designation -> theDesignationValue.equalsIgnoreCase(designation.getValue()));
+		}
+
+		Optional<TermValueSetConceptDesignation> first = stream.findFirst();
+		if (!first.isPresent()) {
+			String failureMessage = String.format("Concept %s did not contain designation [%s|%s|%s|%s|%s] ", theConcept, theLanguage, theUseSystem, theUseCode, theUseDisplay, theDesignationValue);
+			fail(failureMessage);
+			return null;
+		} else {
+			return first.get();
+		}
+
+	}
+
+	@SuppressWarnings("BusyWait")
+	public static void waitForSize(int theTarget, List<?> theList) {
+		StopWatch sw = new StopWatch();
+		while (theList.size() != theTarget && sw.getMillis() <= 16000) {
+			try {
+				Thread.sleep(50);
+			} catch (InterruptedException theE) {
+				throw new Error(theE);
+			}
+		}
+		if (sw.getMillis() >= 16000 || theList.size() > theTarget) {
+			String describeResults = theList
+				.stream()
+				.map(t -> {
+					if (t == null) {
+						return "null";
+					}
+					if (t instanceof IBaseResource) {
+						return ((IBaseResource) t).getIdElement().getValue();
+					}
+					return t.toString();
+				})
+				.collect(Collectors.joining(", "));
+			fail("Size " + theList.size() + " is != target " + theTarget + " - Got: " + describeResults);
+		}
+	}
+
 	@BeforeAll
 	public static void beforeClassRandomizeLocale() {
 		doRandomizeLocaleAndTimezone();
@@ -723,35 +791,6 @@ public abstract class BaseJpaTest extends BaseTest {
 		return retVal;
 	}
 
-	protected TermValueSetConceptDesignation assertTermConceptContainsDesignation(TermValueSetConcept theConcept, String theLanguage, String theUseSystem, String theUseCode, String theUseDisplay, String theDesignationValue) {
-		Stream<TermValueSetConceptDesignation> stream = theConcept.getDesignations().stream();
-		if (theLanguage != null) {
-			stream = stream.filter(designation -> theLanguage.equalsIgnoreCase(designation.getLanguage()));
-		}
-		if (theUseSystem != null) {
-			stream = stream.filter(designation -> theUseSystem.equalsIgnoreCase(designation.getUseSystem()));
-		}
-		if (theUseCode != null) {
-			stream = stream.filter(designation -> theUseCode.equalsIgnoreCase(designation.getUseCode()));
-		}
-		if (theUseDisplay != null) {
-			stream = stream.filter(designation -> theUseDisplay.equalsIgnoreCase(designation.getUseDisplay()));
-		}
-		if (theDesignationValue != null) {
-			stream = stream.filter(designation -> theDesignationValue.equalsIgnoreCase(designation.getValue()));
-		}
-
-		Optional<TermValueSetConceptDesignation> first = stream.findFirst();
-		if (!first.isPresent()) {
-			String failureMessage = String.format("Concept %s did not contain designation [%s|%s|%s|%s|%s] ", theConcept, theLanguage, theUseSystem, theUseCode, theUseDisplay, theDesignationValue);
-			fail(failureMessage);
-			return null;
-		} else {
-			return first.get();
-		}
-
-	}
-
 	public static void waitForSize(int theTarget, Callable<Number> theCallable, Callable<String> theFailureMessage) throws Exception {
 		waitForSize(theTarget, 10000, theCallable, theFailureMessage);
 	}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/BaseJpaDstu2Test.java

@@ -60,6 +60,8 @@ import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.rest.api.EncodingEnum;
 import ca.uhn.fhir.rest.server.provider.ResourceProviderFactory;
 import org.apache.commons.io.IOUtils;
+import org.hl7.fhir.common.hapi.validation.support.CachingValidationSupport;
+import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -218,6 +220,8 @@ public abstract class BaseJpaDstu2Test extends BaseJpaTest {
 	protected SubscriptionLoader mySubscriptionLoader;
 	@Autowired
 	private IBulkDataExportSvc myBulkDataExportSvc;
+	@Autowired
+	private ValidationSupportChain myJpaValidationSupportChain;
 
 	@BeforeEach
 	public void beforeFlushFT() {
@@ -271,5 +275,11 @@ public abstract class BaseJpaDstu2Test extends BaseJpaTest {
 		return retVal;
 	}
 
+	@AfterEach
+	public void afterEachClearCaches() {
+		myValueSetDao.purgeCaches();
+		myJpaValidationSupportChain.invalidateCaches();
+	}
+
 
 }

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/BaseJpaDstu3Test.java

@@ -426,14 +426,10 @@ public abstract class BaseJpaDstu3Test extends BaseJpaTest {
 		return retVal;
 	}
 
-	@AfterAll
+	@AfterEach
-	public static void afterClassClearContextBaseJpaDstu3Test() {
+	public void afterEachClearCaches() {
-		if (ourValueSetDao != null) {
+		myValueSetDao.purgeCaches();
-			ourValueSetDao.purgeCaches();
+		myJpaValidationSupportChainDstu3.invalidateCaches();
-		}
-		if (ourJpaValidationSupportChainDstu3 != null) {
-			ourJpaValidationSupportChainDstu3.invalidateCaches();
-		}
 	}
 
 	/**

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java

@@ -806,7 +806,7 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test {
 			assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty());
 		} catch (ResourceNotFoundException e) {
 			//noinspection SpellCheckingInspection
-			assertEquals(Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fexample.com%2Fmy_value_set", e.getMessage());
+			assertEquals(Msg.code(2024) + "Unknown ValueSet: http%3A%2F%2Fexample.com%2Fmy_value_set", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java

@@ -87,9 +87,7 @@ import ca.uhn.fhir.rest.api.EncodingEnum;
 import ca.uhn.fhir.rest.server.BasePagingProvider;
 import ca.uhn.fhir.rest.server.interceptor.IServerInterceptor;
 import ca.uhn.fhir.rest.server.provider.ResourceProviderFactory;
-import ca.uhn.fhir.test.utilities.BatchJobHelper;
 import ca.uhn.fhir.test.utilities.ITestDataBuilder;
-import ca.uhn.fhir.test.utilities.ProxyUtil;
 import ca.uhn.fhir.util.ClasspathUtil;
 import ca.uhn.fhir.util.UrlUtil;
 import ca.uhn.fhir.validation.FhirValidator;
@@ -153,6 +151,7 @@ import org.hl7.fhir.r4.model.QuestionnaireResponse;
 import org.hl7.fhir.r4.model.RiskAssessment;
 import org.hl7.fhir.r4.model.SearchParameter;
 import org.hl7.fhir.r4.model.ServiceRequest;
+import org.hl7.fhir.r4.model.StringType;
 import org.hl7.fhir.r4.model.StructureDefinition;
 import org.hl7.fhir.r4.model.Subscription;
 import org.hl7.fhir.r4.model.Substance;
@@ -173,10 +172,6 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.slf4j.event.Level;
-import org.springframework.batch.core.repository.dao.JobExecutionDao;
-import org.springframework.batch.core.repository.dao.JobInstanceDao;
-import org.springframework.batch.core.repository.dao.MapJobExecutionDao;
-import org.springframework.batch.core.repository.dao.MapJobInstanceDao;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.ApplicationContext;
@@ -205,8 +200,7 @@ import static org.mockito.Mockito.mock;
 @ExtendWith(SpringExtension.class)
 @ContextConfiguration(classes = {TestR4Config.class})
 public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuilder {
-	private static IValidationSupport ourJpaValidationSupportChainR4;
+	public static final String MY_VALUE_SET = "my-value-set";
-	private static IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> ourValueSetDao;
 
 	@Autowired
 	protected IPackageInstallerSvc myPackageInstallerSvc;
@@ -541,12 +535,6 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil
 		termDeferredStorageSvc.clearDeferred();
 	}
 
-	@AfterEach()
-	public void afterGrabCaches() {
-		ourValueSetDao = myValueSetDao;
-		ourJpaValidationSupportChainR4 = myJpaValidationSupportChainR4;
-	}
-
 	@BeforeEach
 	public void beforeCreateInterceptor() {
 		myInterceptor = mock(IServerInterceptor.class);
@@ -721,6 +709,38 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil
 		});
 	}
 
+	protected void createLocalCsAndVs() {
+		//@formatter:off
+		CodeSystem codeSystem = new CodeSystem();
+		codeSystem.setUrl(FhirResourceDaoR4TerminologyTest.URL_MY_CODE_SYSTEM);
+		codeSystem.setContent(CodeSystem.CodeSystemContentMode.COMPLETE);
+		codeSystem
+			.addConcept().setCode("A").setDisplay("Code A").addDesignation(
+				new CodeSystem.ConceptDefinitionDesignationComponent().setLanguage("en").setValue("CodeADesignation")).addProperty(
+				new CodeSystem.ConceptPropertyComponent().setCode("CodeAProperty").setValue(new StringType("CodeAPropertyValue"))
+			)
+			.addConcept(new CodeSystem.ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA")
+				.addConcept(new CodeSystem.ConceptDefinitionComponent().setCode("AAA").setDisplay("Code AAA"))
+			)
+			.addConcept(new CodeSystem.ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB"));
+		codeSystem
+			.addConcept().setCode("B").setDisplay("Code B")
+			.addConcept(new CodeSystem.ConceptDefinitionComponent().setCode("BA").setDisplay("Code BA"))
+			.addConcept(new CodeSystem.ConceptDefinitionComponent().setCode("BB").setDisplay("Code BB"));
+		//@formatter:on
+		myCodeSystemDao.create(codeSystem, mySrd);
+
+		createLocalVs(codeSystem);
+	}
+
+	protected void createLocalVs(CodeSystem codeSystem) {
+		ValueSet valueSet = new ValueSet();
+		valueSet.setId(MY_VALUE_SET);
+		valueSet.setUrl(FhirResourceDaoR4TerminologyTest.URL_MY_VALUE_SET);
+		valueSet.getCompose().addInclude().setSystem(codeSystem.getUrl());
+		myValueSetDao.update(valueSet, mySrd);
+	}
+
 	private static void flattenExpansionHierarchy(List<String> theFlattenedHierarchy, List<TermConcept> theCodes, String thePrefix) {
 		theCodes.sort((o1, o2) -> {
 			int s1 = o1.getSequence() != null ? o1.getSequence() : o1.getCode().hashCode();
@@ -738,10 +758,10 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil
 		}
 	}
 
-	@AfterAll
+	@AfterEach
-	public static void afterClassClearContextBaseJpaR4Test() {
+	public void afterEachClearCaches() {
-		ourValueSetDao.purgeCaches();
+		myValueSetDao.purgeCaches();
-		ourJpaValidationSupportChainR4.invalidateCaches();
+		myJpaValidationSupportChainR4.invalidateCaches();
 	}
 
 	/**

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboUniqueParamTest.java

@@ -57,6 +57,7 @@ import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.stringContainsInOrder;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test {
@@ -796,6 +797,41 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 
 	}
 
+	@Test
+	public void testUpdateConditionalOverExistingUnique() {
+		createUniqueIndexPatientIdentifierCount1();
+
+		Patient pt = new Patient();
+		pt.addIdentifier().setSystem("urn").setValue("111");
+		IIdType id = myPatientDao.create(pt).getId().toUnqualifiedVersionless();
+
+		new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
+			@Override
+			protected void doInTransactionWithoutResult(@Nonnull TransactionStatus status) {
+				List<ResourceIndexedComboStringUnique> all = myResourceIndexedCompositeStringUniqueDao.findAll();
+				assertEquals(1, all.size());
+			}
+		});
+
+		pt = new Patient();
+		pt.addIdentifier().setSystem("urn").setValue("111");
+		pt.setActive(true);
+		String version = myPatientDao.update(pt, "Patient?first-identifier=urn|111").getId().getVersionIdPart();
+		assertEquals("2", version);
+
+		new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
+			@Override
+			protected void doInTransactionWithoutResult(@Nonnull TransactionStatus status) {
+				List<ResourceIndexedComboStringUnique> all = myResourceIndexedCompositeStringUniqueDao.findAll();
+				assertEquals(1, all.size());
+			}
+		});
+
+		pt = myPatientDao.read(id);
+		assertTrue(pt.getActive());
+
+	}
+
 	@Test
 	public void testIndexTransactionWithMatchUrl() {
 		Patient pt2 = new Patient();
@@ -1255,7 +1291,7 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 		pt1.setManagingOrganization(new Reference("Organization/ORG"));
 		myPatientDao.update(pt1, "Patient?name=FAMILY1&organization.name=ORG").getId().toUnqualifiedVersionless();
 
-		runInTransaction(()->{
+		runInTransaction(() -> {
 			List<ResourceIndexedComboStringUnique> uniques = myResourceIndexedCompositeStringUniqueDao.findAll();
 			assertEquals(1, uniques.size());
 			assertEquals("Patient/" + id1.getIdPart(), uniques.get(0).getResource().getIdDt().toUnqualifiedVersionless().getValue());

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java

@@ -1,7 +1,11 @@
 package ca.uhn.fhir.jpa.dao.r4;
 
+import ca.uhn.fhir.context.support.ValidationSupportContext;
+import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.api.model.HistoryCountModeEnum;
+import ca.uhn.fhir.jpa.entity.TermValueSet;
+import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum;
 import ca.uhn.fhir.jpa.model.entity.ForcedId;
 import ca.uhn.fhir.jpa.model.entity.ModelConfig;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
@@ -9,6 +13,7 @@ import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.partition.SystemRequestDetails;
 import ca.uhn.fhir.jpa.provider.r4.BaseResourceProviderR4Test;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl;
 import ca.uhn.fhir.jpa.util.SqlQuery;
 import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
 import ca.uhn.fhir.rest.api.SortSpec;
@@ -39,11 +44,14 @@ import org.hl7.fhir.r4.model.Quantity;
 import org.hl7.fhir.r4.model.Reference;
 import org.hl7.fhir.r4.model.ServiceRequest;
 import org.hl7.fhir.r4.model.StringType;
+import org.hl7.fhir.r4.model.ValueSet;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Slice;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
@@ -53,6 +61,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static org.awaitility.Awaitility.await;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
@@ -81,6 +90,8 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
 		myDaoConfig.setAutoCreatePlaceholderReferenceTargets(new DaoConfig().isAutoCreatePlaceholderReferenceTargets());
 		myDaoConfig.setPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets(new DaoConfig().isPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets());
 		myDaoConfig.setResourceClientIdStrategy(new DaoConfig().getResourceClientIdStrategy());
+
+		BaseTermReadSvcImpl.setForceDisableHibernateSearchForUnitTest(false);
 	}
 
 	@Override
@@ -2276,6 +2287,128 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
 	}
 
 
+	@Test
+	public void testValueSetExpand_NotPreExpanded_UseHibernateSearch() {
+		createLocalCsAndVs();
+
+		logAllConcepts();
+		logAllConceptDesignations();
+		logAllConceptProperties();
+
+		ValueSet valueSet = myValueSetDao.read(new IdType(MY_VALUE_SET), mySrd);
+
+		myCaptureQueriesListener.clear();
+		ValueSet expansion = (ValueSet) myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), new ValueSetExpansionOptions(), valueSet).getValueSet();
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
+		assertEquals(7, expansion.getExpansion().getContains().size());
+		assertEquals(1, expansion.getExpansion().getContains().stream().filter(t->t.getCode().equals("A")).findFirst().orElseThrow(()->new IllegalArgumentException()).getDesignation().size());
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread();
+		assertEquals(5, myCaptureQueriesListener.countSelectQueries());
+		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
+		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
+		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
+		assertEquals(1, myCaptureQueriesListener.countCommits());
+		assertEquals(0, myCaptureQueriesListener.countRollbacks());
+
+		// Second time - Should reuse cache
+		myCaptureQueriesListener.clear();
+		expansion = (ValueSet) myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), new ValueSetExpansionOptions(), valueSet).getValueSet();
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
+		assertEquals(7, expansion.getExpansion().getContains().size());
+		assertEquals(1, expansion.getExpansion().getContains().stream().filter(t->t.getCode().equals("A")).findFirst().orElseThrow(()->new IllegalArgumentException()).getDesignation().size());
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread();
+		assertEquals(0, myCaptureQueriesListener.countSelectQueries());
+		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
+		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
+		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
+		assertEquals(0, myCaptureQueriesListener.countCommits());
+		assertEquals(0, myCaptureQueriesListener.countRollbacks());
+	}
+
+	@Test
+	public void testValueSetExpand_NotPreExpanded_DontUseHibernateSearch() {
+		BaseTermReadSvcImpl.setForceDisableHibernateSearchForUnitTest(true);
+
+		createLocalCsAndVs();
+
+		logAllConcepts();
+		logAllConceptDesignations();
+		logAllConceptProperties();
+
+		ValueSet valueSet = myValueSetDao.read(new IdType(MY_VALUE_SET), mySrd);
+
+		myCaptureQueriesListener.clear();
+		ValueSet expansion = (ValueSet) myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), new ValueSetExpansionOptions(), valueSet).getValueSet();
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
+		assertEquals(7, expansion.getExpansion().getContains().size());
+		assertEquals(1, expansion.getExpansion().getContains().stream().filter(t->t.getCode().equals("A")).findFirst().orElseThrow(()->new IllegalArgumentException()).getDesignation().size());
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread();
+		assertEquals(5, myCaptureQueriesListener.countSelectQueries());
+		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
+		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
+		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
+		assertEquals(1, myCaptureQueriesListener.countCommits());
+		assertEquals(0, myCaptureQueriesListener.countRollbacks());
+
+		// Second time - Should reuse cache
+		myCaptureQueriesListener.clear();
+		expansion = (ValueSet) myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), new ValueSetExpansionOptions(), valueSet).getValueSet();
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
+		assertEquals(7, expansion.getExpansion().getContains().size());
+		assertEquals(1, expansion.getExpansion().getContains().stream().filter(t->t.getCode().equals("A")).findFirst().orElseThrow(()->new IllegalArgumentException()).getDesignation().size());
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread();
+		assertEquals(0, myCaptureQueriesListener.countSelectQueries());
+		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
+		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
+		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
+		assertEquals(0, myCaptureQueriesListener.countCommits());
+		assertEquals(0, myCaptureQueriesListener.countRollbacks());
+	}
+
+	@Test
+	public void testValueSetExpand_PreExpanded_UseHibernateSearch() {
+		createLocalCsAndVs();
+
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+		runInTransaction(()->{
+			Slice<TermValueSet> page = myTermValueSetDao.findByExpansionStatus(PageRequest.of(0, 10), TermValueSetPreExpansionStatusEnum.EXPANDED);
+			assertEquals(1, page.getContent().size());
+		});
+
+		logAllConcepts();
+		logAllConceptDesignations();
+		logAllConceptProperties();
+
+		ValueSet valueSet = myValueSetDao.read(new IdType(MY_VALUE_SET), mySrd);
+
+		myCaptureQueriesListener.clear();
+		ValueSet expansion = (ValueSet) myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), new ValueSetExpansionOptions(), valueSet).getValueSet();
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
+		assertEquals(7, expansion.getExpansion().getContains().size());
+		assertEquals(1, expansion.getExpansion().getContains().stream().filter(t->t.getCode().equals("A")).findFirst().orElseThrow(()->new IllegalArgumentException()).getDesignation().size());
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread();
+		assertEquals(3, myCaptureQueriesListener.countSelectQueries());
+		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
+		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
+		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
+		assertEquals(1, myCaptureQueriesListener.countCommits());
+		assertEquals(0, myCaptureQueriesListener.countRollbacks());
+
+		// Second time - Should reuse cache
+		myCaptureQueriesListener.clear();
+		expansion = (ValueSet) myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), new ValueSetExpansionOptions(), valueSet).getValueSet();
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
+		assertEquals(7, expansion.getExpansion().getContains().size());
+		assertEquals(1, expansion.getExpansion().getContains().stream().filter(t->t.getCode().equals("A")).findFirst().orElseThrow(()->new IllegalArgumentException()).getDesignation().size());
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread();
+		assertEquals(0, myCaptureQueriesListener.countSelectQueries());
+		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
+		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
+		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
+		assertEquals(0, myCaptureQueriesListener.countCommits());
+		assertEquals(0, myCaptureQueriesListener.countRollbacks());
+	}
+
 	@Test
 	public void testMassIngestionMode_TransactionWithChanges() {
 		myDaoConfig.setDeleteEnabled(false);

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java

@@ -168,7 +168,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
 		codeSystem.setContent(CodeSystemContentMode.NOTPRESENT);
 		IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified();
 
-		ResourceTable table = runInTransaction(()->myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new));
+		ResourceTable table = runInTransaction(() -> myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new));
 
 		TermCodeSystemVersion cs = new TermCodeSystemVersion();
 		cs.setResource(table);
@@ -194,34 +194,6 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
 		myTerminologyDeferredStorageSvc.saveAllDeferred();
 	}
 
-	private void createLocalCsAndVs() {
-		//@formatter:off
-		CodeSystem codeSystem = new CodeSystem();
-		codeSystem.setUrl(URL_MY_CODE_SYSTEM);
-		codeSystem.setContent(CodeSystemContentMode.COMPLETE);
-		codeSystem
-			.addConcept().setCode("A").setDisplay("Code A")
-			.addConcept(new ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA")
-				.addConcept(new ConceptDefinitionComponent().setCode("AAA").setDisplay("Code AAA"))
-			)
-			.addConcept(new ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB"));
-		codeSystem
-			.addConcept().setCode("B").setDisplay("Code B")
-			.addConcept(new ConceptDefinitionComponent().setCode("BA").setDisplay("Code BA"))
-			.addConcept(new ConceptDefinitionComponent().setCode("BB").setDisplay("Code BB"));
-		//@formatter:on
-		myCodeSystemDao.create(codeSystem, mySrd);
-
-		createLocalVs(codeSystem);
-	}
-
-	private void createLocalVs(CodeSystem codeSystem) {
-		ValueSet valueSet = new ValueSet();
-		valueSet.setUrl(URL_MY_VALUE_SET);
-		valueSet.getCompose().addInclude().setSystem(codeSystem.getUrl());
-		myValueSetDao.create(valueSet, mySrd);
-	}
-
 	private void logAndValidateValueSet(ValueSet theResult) {
 		IParser parser = myFhirCtx.newXmlParser().setPrettyPrint(true);
 		String encoded = parser.encodeResourceToString(theResult);
@@ -531,7 +503,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
 		codeSystem.setContent(CodeSystemContentMode.NOTPRESENT);
 		IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified();
 
-		ResourceTable table = runInTransaction(()->myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new));
+		ResourceTable table = runInTransaction(() -> myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new));
 
 		TermCodeSystemVersion cs = new TermCodeSystemVersion();
 		cs.setResource(table);
@@ -857,7 +829,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
 		codeSystem.setContent(CodeSystemContentMode.NOTPRESENT);
 		IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified();
 
-		ResourceTable table = runInTransaction(()->myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new));
+		ResourceTable table = runInTransaction(() -> myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new));
 
 		TermCodeSystemVersion cs = new TermCodeSystemVersion();
 		cs.setResource(table);
@@ -1250,15 +1222,44 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
 		obsCA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("CA");
 		myObservationDao.create(obsCA, mySrd).getId().toUnqualifiedVersionless();
 
-		SearchParameterMap params = new SearchParameterMap();
+		SearchParameterMap params = SearchParameterMap.newSynchronous();
 		params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN));
 		assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), containsInAnyOrder(idAA.getValue(), idBA.getValue()));
 
+		myCaptureQueriesListener.clear();
+		assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), containsInAnyOrder(idAA.getValue(), idBA.getValue()));
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread();
+
+	}
+
+	@Test
+	public void testSearchCodeNotInLocalCodesystem() {
+		createLocalCsAndVs();
+
+		Observation obsAA = new Observation();
+		obsAA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("AA");
+		IIdType idAA = myObservationDao.create(obsAA, mySrd).getId().toUnqualifiedVersionless();
+
+		Observation obsBA = new Observation();
+		obsBA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("BA");
+		IIdType idBA = myObservationDao.create(obsBA, mySrd).getId().toUnqualifiedVersionless();
+
+		Observation obsCA = new Observation();
+		obsCA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("CA");
+		IIdType idCA = myObservationDao.create(obsCA, mySrd).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap params = SearchParameterMap.newSynchronous();
+		params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.NOT_IN));
+		assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), containsInAnyOrder(idCA.getValue()));
+
+		myCaptureQueriesListener.clear();
+		assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), containsInAnyOrder(idCA.getValue()));
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread();
+
 	}
 
 	@Test
 	public void testSearchCodeInUnknownCodeSystem() {
-
 		SearchParameterMap params = new SearchParameterMap();
 
 		try {
@@ -1266,7 +1267,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
 			assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty());
 		} catch (ResourceNotFoundException e) {
 			//noinspection SpellCheckingInspection
-			assertEquals(Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fexample.com%2Fmy_value_set", e.getMessage());
+			assertEquals(Msg.code(2024) + "Unknown ValueSet: http%3A%2F%2Fexample.com%2Fmy_value_set", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValueSetMultiVersionTest.java

@@ -12,6 +12,7 @@ import org.hl7.fhir.r4.model.ValueSet;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.PageRequest;
+import org.springframework.test.annotation.DirtiesContext;
 
 import java.util.HashMap;
 import java.util.HashSet;

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java

@@ -1319,13 +1319,12 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 
 	@Test
 	public void testSearch_IdParamSecond_ForcedId_SpecificPartition() {
-		// FIXME: move down
 		IIdType patientId1 = createPatient(withPartition(1), withId("PT-1"), withActiveTrue());
-		logAllTokenIndexes();
-
 		IIdType patientIdNull = createPatient(withPartition(null), withId("PT-NULL"), withActiveTrue());
 		IIdType patientId2 = createPatient(withPartition(2), withId("PT-2"), withActiveTrue());
 
+		logAllTokenIndexes();
+
 		/* *******************************
 		 * _id param is second parameter
 		 * *******************************/

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java

@@ -566,10 +566,10 @@ public abstract class BaseJpaR5Test extends BaseJpaTest implements ITestDataBuil
 		});
 	}
 
-	@AfterAll
+	@AfterEach
-	public static void afterClassClearContextBaseJpaR5Test() {
+	public void afterEachClearCaches() {
-		ourValueSetDao.purgeCaches();
+		myValueSetDao.purgeCaches();
-		ourJpaValidationSupportChainR5.invalidateCaches();
+		myJpaValidationSupportChain.invalidateCaches();
 	}
 
 	/**

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java

@@ -441,7 +441,7 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
 		}
 	}
 
@@ -484,7 +484,7 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetVersionedTest.java

@@ -576,7 +576,7 @@ public class ResourceProviderDstu3ValueSetVersionedTest extends BaseResourceProv
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
 		}
 	}
 
@@ -657,7 +657,7 @@ public class ResourceProviderDstu3ValueSetVersionedTest extends BaseResourceProv
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/AuthorizationInterceptorJpaR4Test.java

@@ -4,6 +4,7 @@ import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportSvc;
 import ca.uhn.fhir.jpa.bulk.export.model.BulkExportJobStatusEnum;
 import ca.uhn.fhir.jpa.bulk.export.provider.BulkDataExportProvider;
+import ca.uhn.fhir.jpa.dao.r4.FhirResourceDaoR4TerminologyTest;
 import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor;
 import ca.uhn.fhir.model.primitive.IdDt;
 import ca.uhn.fhir.rest.api.Constants;
@@ -61,6 +62,7 @@ import org.springframework.mock.web.MockHttpServletRequest;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -68,6 +70,7 @@ import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.startsWith;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 public class AuthorizationInterceptorJpaR4Test extends BaseResourceProviderR4Test {
@@ -678,6 +681,34 @@ public class AuthorizationInterceptorJpaR4Test extends BaseResourceProviderR4Tes
 
 	}
 
+	@Test
+	public void testSearchCodeIn() {
+		createLocalCsAndVs();
+
+		createObservation(withId("allowed"), withObservationCode(FhirResourceDaoR4TerminologyTest.URL_MY_CODE_SYSTEM, "A"));
+		createObservation(withId("disallowed"), withObservationCode(FhirResourceDaoR4TerminologyTest.URL_MY_CODE_SYSTEM, "foo"));
+
+		ourRestServer.registerInterceptor(new AuthorizationInterceptor(PolicyEnum.DENY) {
+			@Override
+			public List<IAuthRule> buildRuleList(RequestDetails theRequestDetails) {
+				return new RuleBuilder()
+					.allow().read().resourcesOfType("Observation").withCodeInValueSet("code", FhirResourceDaoR4TerminologyTest.URL_MY_VALUE_SET).andThen()
+					.build();
+			}
+		}.setValidationSupport(myValidationSupport));
+
+		// Should be ok
+		myClient.read().resource(Observation.class).withId("Observation/allowed").execute();
+
+		try {
+			myClient.read().resource(Observation.class).withId("Observation/disallowed").execute();
+			fail();
+		} catch (ForbiddenOperationException e) {
+			// good
+		}
+
+	}
+
 	/**
 	 * See #751
 	 */

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ValueSetNoVerCSNoVerTest.java

@@ -159,7 +159,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 	private void createExternalCsAndLocalVs() {
 		runInTransaction(() -> {
 			CodeSystem codeSystem = createExternalCs();
-			createLocalVs(codeSystem);
+			createLocalVsForCodeSystem(codeSystem);
 		});
 	}
 
@@ -197,7 +197,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 		myLocalValueSetId = myValueSetDao.create(myLocalVs, mySrd).getId().toUnqualifiedVersionless();
 	}
 
-	private void createLocalVs(CodeSystem codeSystem) {
+	private void createLocalVsForCodeSystem(CodeSystem codeSystem) {
 		myLocalVs = new ValueSet();
 		myLocalVs.setUrl(URL_MY_VALUE_SET);
 		ConceptSetComponent include = myLocalVs.getCompose().addInclude();
@@ -445,7 +445,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
 		}
 	}
 
@@ -493,7 +493,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
 		}
 	}
 
@@ -1075,7 +1075,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 			.execute();
 		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
 		assertThat(toDirectCodes(expansion.getExpansion().getContains()), containsInAnyOrder("A", "AA", "AB", "AAA"));
-		assertEquals(19, myCaptureQueriesListener.getSelectQueries().size());
+		assertEquals(11, myCaptureQueriesListener.getSelectQueries().size());
 		assertEquals("ValueSet \"ValueSet.url[http://example.com/my_value_set]\" has not yet been pre-expanded. Performing in-memory expansion without parameters. Current status: NOT_EXPANDED | The ValueSet is waiting to be picked up and pre-expanded by a scheduled task.", expansion.getMeta().getExtensionString(EXT_VALUESET_EXPANSION_MESSAGE));
 
 		// Hierarchical
@@ -1092,7 +1092,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 		assertThat(toDirectCodes(expansion.getExpansion().getContains()), containsInAnyOrder("A"));
 		assertThat(toDirectCodes(expansion.getExpansion().getContains().get(0).getContains()), containsInAnyOrder("AA", "AB"));
 		assertThat(toDirectCodes(expansion.getExpansion().getContains().get(0).getContains().stream().filter(t -> t.getCode().equals("AA")).findFirst().orElseThrow(() -> new IllegalArgumentException()).getContains()), containsInAnyOrder("AAA"));
-		assertEquals(16, myCaptureQueriesListener.getSelectQueries().size());
+		assertEquals(12, myCaptureQueriesListener.getSelectQueries().size());
 
 	}
 
@@ -1115,7 +1115,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 			.execute();
 		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
 		assertThat(toDirectCodes(expansion.getExpansion().getContains()), containsInAnyOrder("A", "AA", "AB", "AAA"));
-		assertEquals(15, myCaptureQueriesListener.getSelectQueries().size());
+		assertEquals(7, myCaptureQueriesListener.getSelectQueries().size());
 		assertEquals("ValueSet with URL \"Unidentified ValueSet\" was expanded using an in-memory expansion", expansion.getMeta().getExtensionString(EXT_VALUESET_EXPANSION_MESSAGE));
 
 		// Hierarchical
@@ -1132,7 +1132,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 		assertThat(toDirectCodes(expansion.getExpansion().getContains()), containsInAnyOrder("A"));
 		assertThat(toDirectCodes(expansion.getExpansion().getContains().get(0).getContains()), containsInAnyOrder("AA", "AB"));
 		assertThat(toDirectCodes(expansion.getExpansion().getContains().get(0).getContains().stream().filter(t -> t.getCode().equals("AA")).findFirst().orElseThrow(() -> new IllegalArgumentException()).getContains()), containsInAnyOrder("AAA"));
-		assertEquals(14, myCaptureQueriesListener.getSelectQueries().size());
+		assertEquals(10, myCaptureQueriesListener.getSelectQueries().size());
 
 	}
 
@@ -1147,6 +1147,8 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 
 		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
 
+		logAllValueSetConcepts();
+
 		// Do a warm-up pass to precache anything that can be pre-cached
 		myClient
 			.operation()
@@ -1156,7 +1158,7 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 			.returnResourceType(ValueSet.class)
 			.execute();
 
-		// Non-hierarchical
+		// Non-hierarchical (Should reuse cache)
 		myCaptureQueriesListener.clear();
 		expansion = myClient
 			.operation()
@@ -1167,10 +1169,10 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
 			.execute();
 		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
 		assertThat(toDirectCodes(expansion.getExpansion().getContains()), containsInAnyOrder("A", "AA", "AB", "AAA"));
-		assertEquals(3, myCaptureQueriesListener.getSelectQueries().size());
+		assertEquals(0, myCaptureQueriesListener.getSelectQueries().size());
 		assertThat(expansion.getMeta().getExtensionString(EXT_VALUESET_EXPANSION_MESSAGE), containsString("ValueSet was expanded using an expansion that was pre-calculated"));
 
-		// Hierarchical
+		// Hierarchical (shouldn't reuse cache)
 		myCaptureQueriesListener.clear();
 		expansion = myClient
 			.operation()

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ValueSetVerCSNoVerTest.java

@@ -152,7 +152,7 @@ public class ResourceProviderR4ValueSetVerCSNoVerTest extends BaseResourceProvid
 	private void createExternalCsAndLocalVs() {
 		runInTransaction(()-> {
 			CodeSystem codeSystem = createExternalCs();
-			createLocalVs(codeSystem);
+			createLocalVsForCodeSystem(codeSystem);
 		});
 	}
 
@@ -163,7 +163,7 @@ public class ResourceProviderR4ValueSetVerCSNoVerTest extends BaseResourceProvid
 		});
 	}
 
-	private void createLocalVs(CodeSystem codeSystem) {
+	private void createLocalVsForCodeSystem(CodeSystem codeSystem) {
 		myLocalVs = new ValueSet();
 		myLocalVs.setUrl(URL_MY_VALUE_SET);
 		ConceptSetComponent include = myLocalVs.getCompose().addInclude();
@@ -357,7 +357,7 @@ public class ResourceProviderR4ValueSetVerCSNoVerTest extends BaseResourceProvid
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
 		}
 	}
 
@@ -405,7 +405,7 @@ public class ResourceProviderR4ValueSetVerCSNoVerTest extends BaseResourceProvid
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ValueSetVerCSVerTest.java

@@ -526,7 +526,7 @@ public class ResourceProviderR4ValueSetVerCSVerTest extends BaseResourceProvider
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
 		}
 	}
 
@@ -610,7 +610,7 @@ public class ResourceProviderR4ValueSetVerCSVerTest extends BaseResourceProvider
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java

@@ -485,7 +485,7 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test {
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
 		}
 	}
 
@@ -622,7 +622,7 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test {
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetVersionedTest.java

@@ -558,7 +558,7 @@ public class ResourceProviderR5ValueSetVersionedTest extends BaseResourceProvide
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
 		}
 	}
 
@@ -640,7 +640,7 @@ public class ResourceProviderR5ValueSetVersionedTest extends BaseResourceProvide
 				.execute();
 		} catch (ResourceNotFoundException e) {
 			assertEquals(404, e.getStatusCode());
-			assertEquals("HTTP 404 Not Found: " + Msg.code(886) + "Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
+			assertEquals("HTTP 404 Not Found: HAPI-2030: Can not find ValueSet with URL: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fextensional-case-2%7C3", e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-cql/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-mdm/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-model/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-searchparam/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-subscription/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-utilities/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-uhnfhirtest/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server-mdm/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server-openapi/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/AllowedCodeInValueSet.java

@@ -0,0 +1,59 @@
+package ca.uhn.fhir.rest.server.interceptor.auth;
+
+/*-
+ * #%L
+ * HAPI FHIR - Server Framework
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import javax.annotation.Nonnull;
+
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
+class AllowedCodeInValueSet {
+	private final String myResourceName;
+	private final String mySearchParameterName;
+	private final String myValueSetUrl;
+	private final boolean myNegate;
+
+	public AllowedCodeInValueSet(@Nonnull String theResourceName, @Nonnull String theSearchParameterName, @Nonnull String theValueSetUrl, boolean theNegate) {
+		assert isNotBlank(theResourceName);
+		assert isNotBlank(theSearchParameterName);
+		assert isNotBlank(theValueSetUrl);
+
+		myResourceName = theResourceName;
+		mySearchParameterName = theSearchParameterName;
+		myValueSetUrl = theValueSetUrl;
+		myNegate = theNegate;
+	}
+
+	public String getResourceName() {
+		return myResourceName;
+	}
+
+	public String getSearchParameterName() {
+		return mySearchParameterName;
+	}
+
+	public String getValueSetUrl() {
+		return myValueSetUrl;
+	}
+
+	public boolean isNegate() {
+		return myNegate;
+	}
+}

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/AuthorizationInterceptor.java

@@ -20,8 +20,9 @@ package ca.uhn.fhir.rest.server.interceptor.auth;
  * #L%
  */
 
-import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.context.support.IValidationSupport;
+import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.interceptor.api.Hook;
 import ca.uhn.fhir.interceptor.api.Interceptor;
 import ca.uhn.fhir.interceptor.api.Pointcut;
@@ -43,6 +44,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -78,12 +80,15 @@ public class AuthorizationInterceptor implements IRuleApplier {
 	private final String myRequestRuleListKey = AuthorizationInterceptor.class.getName() + "_" + myInstanceIndex + "_RULELIST";
 	private PolicyEnum myDefaultPolicy = PolicyEnum.DENY;
 	private Set<AuthorizationFlagsEnum> myFlags = Collections.emptySet();
+	private IValidationSupport myValidationSupport;
+	private Logger myTroubleshootingLog;
 
 	/**
 	 * Constructor
 	 */
 	public AuthorizationInterceptor() {
 		super();
+		setTroubleshootingLog(ourLog);
 	}
 
 	/**
@@ -96,6 +101,17 @@ public class AuthorizationInterceptor implements IRuleApplier {
 		setDefaultPolicy(theDefaultPolicy);
 	}
 
+	@Nonnull
+	@Override
+	public Logger getTroubleshootingLog() {
+		return myTroubleshootingLog;
+	}
+
+	public void setTroubleshootingLog(@Nonnull Logger theTroubleshootingLog) {
+		Validate.notNull(theTroubleshootingLog, "theTroubleshootingLog must not be null");
+		myTroubleshootingLog = theTroubleshootingLog;
+	}
+
 	private void applyRulesAndFailIfDeny(RestOperationTypeEnum theOperation, RequestDetails theRequestDetails, IBaseResource theInputResource, IIdType theInputResourceId,
 													 IBaseResource theOutputResource, Pointcut thePointcut) {
 		Verdict decision = applyRulesAndReturnDecision(theOperation, theRequestDetails, theInputResource, theInputResourceId, theOutputResource, thePointcut);
@@ -139,6 +155,28 @@ public class AuthorizationInterceptor implements IRuleApplier {
 		return verdict;
 	}
 
+	/**
+	 * @since 6.0.0
+	 */
+	@Nullable
+	@Override
+	public IValidationSupport getValidationSupport() {
+		return myValidationSupport;
+	}
+
+	/**
+	 * Sets a validation support module that will be used for terminology-based rules
+	 *
+	 * @param theValidationSupport The validation support. Null is also acceptable (this is the default),
+	 *                             in which case the validation support module associated with the {@link FhirContext}
+	 *                             will be used.
+	 * @since 6.0.0
+	 */
+	public AuthorizationInterceptor setValidationSupport(IValidationSupport theValidationSupport) {
+		myValidationSupport = theValidationSupport;
+		return this;
+	}
+
 	/**
 	 * Subclasses should override this method to supply the set of rules to be applied to
 	 * this individual request.
@@ -363,7 +401,6 @@ public class AuthorizationInterceptor implements IRuleApplier {
 		applyRulesAndFailIfDeny(restOperationType, theRequestDetails, null, null, null, thePointcut);
 	}
 
-
 	private void checkPointcutAndFailIfDeny(RequestDetails theRequestDetails, Pointcut thePointcut, @Nonnull IBaseResource theInputResource) {
 		applyRulesAndFailIfDeny(theRequestDetails.getRestOperationType(), theRequestDetails, theInputResource, theInputResource.getIdElement(), null, thePointcut);
 	}
@@ -442,6 +479,34 @@ public class AuthorizationInterceptor implements IRuleApplier {
 		OUT,
 	}
 
+	static List<IBaseResource> toListOfResourcesAndExcludeContainer(IBaseResource theResponseObject, FhirContext fhirContext) {
+		if (theResponseObject == null) {
+			return Collections.emptyList();
+		}
+
+		List<IBaseResource> retVal;
+
+		boolean isContainer = false;
+		if (theResponseObject instanceof IBaseBundle) {
+			isContainer = true;
+		} else if (theResponseObject instanceof IBaseParameters) {
+			isContainer = true;
+		}
+
+		if (!isContainer) {
+			return Collections.singletonList(theResponseObject);
+		}
+
+		retVal = fhirContext.newTerser().getAllPopulatedChildElementsOfType(theResponseObject, IBaseResource.class);
+
+		// Exclude the container
+		if (retVal.size() > 0 && retVal.get(0) == theResponseObject) {
+			retVal = retVal.subList(1, retVal.size());
+		}
+
+		return retVal;
+	}
+
 	public static class Verdict {
 
 		private final IAuthRule myDecidingRule;
@@ -478,32 +543,4 @@ public class AuthorizationInterceptor implements IRuleApplier {
 
 	}
 
-	static List<IBaseResource> toListOfResourcesAndExcludeContainer(IBaseResource theResponseObject, FhirContext fhirContext) {
-		if (theResponseObject == null) {
-			return Collections.emptyList();
-		}
-
-		List<IBaseResource> retVal;
-
-		boolean isContainer = false;
-		if (theResponseObject instanceof IBaseBundle) {
-			isContainer = true;
-		} else if (theResponseObject instanceof IBaseParameters) {
-			isContainer = true;
-		}
-
-		if (!isContainer) {
-			return Collections.singletonList(theResponseObject);
-		}
-
-		retVal = fhirContext.newTerser().getAllPopulatedChildElementsOfType(theResponseObject, IBaseResource.class);
-
-		// Exclude the container
-		if (retVal.size() > 0 && retVal.get(0) == theResponseObject) {
-			retVal = retVal.subList(1, retVal.size());
-		}
-
-		return retVal;
-	}
-
 }

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/AuthorizedList.java

@@ -23,6 +23,8 @@ package ca.uhn.fhir.rest.server.interceptor.auth;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import org.apache.commons.lang3.Validate;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -33,11 +35,19 @@ public class AuthorizedList {
 
 	private List<String> myAllowedCompartments;
 	private List<String> myAllowedInstances;
+	private List<AllowedCodeInValueSet> myAllowedCodeInValueSets;
 
+	@Nullable
 	List<String> getAllowedCompartments() {
 		return myAllowedCompartments;
 	}
 
+	@Nullable
+	List<AllowedCodeInValueSet> getAllowedCodeInValueSets() {
+		return myAllowedCodeInValueSets;
+	}
+
+	@Nullable
 	List<String> getAllowedInstances() {
 		return myAllowedInstances;
 	}
@@ -101,4 +111,51 @@ public class AuthorizedList {
 		}
 		return this;
 	}
+
+	/**
+	 * If specified, any search for <code>theResourceName</code> will automatically include a parameter indicating that
+	 * the token search parameter <code>theSearchParameterName</code> must have a value in the ValueSet with URL <code>theValueSetUrl</code>.
+	 *
+	 * @param theResourceName        The resource name, e.g. <code>Observation</code>
+	 * @param theSearchParameterName The search parameter name, e.g. <code>code</code>
+	 * @param theValueSetUrl         The valueset URL, e.g. <code>http://my-value-set</code>
+	 * @return Returns a reference to <code>this</code> for easy chaining
+	 * @see AuthorizationInterceptor If search narrowing by code is being used for security reasons, consider also using AuthorizationInterceptor as a failsafe to ensure that no inapproproiate resources are returned
+	 * @since 6.0.0
+	 */
+	public AuthorizedList addCodeInValueSet(@Nonnull String theResourceName, @Nonnull String theSearchParameterName, @Nonnull String theValueSetUrl) {
+		Validate.notBlank(theResourceName, "theResourceName must not be missing or null");
+		Validate.notBlank(theSearchParameterName, "theSearchParameterName must not be missing or null");
+		Validate.notBlank(theValueSetUrl, "theResourceUrl must not be missing or null");
+
+		return doAddCodeInValueSet(theResourceName, theSearchParameterName, theValueSetUrl, false);
+	}
+
+	/**
+	 * If specified, any search for <code>theResourceName</code> will automatically include a parameter indicating that
+	 * the token search parameter <code>theSearchParameterName</code> must have a value not in the ValueSet with URL <code>theValueSetUrl</code>.
+	 *
+	 * @param theResourceName        The resource name, e.g. <code>Observation</code>
+	 * @param theSearchParameterName The search parameter name, e.g. <code>code</code>
+	 * @param theValueSetUrl         The valueset URL, e.g. <code>http://my-value-set</code>
+	 * @return Returns a reference to <code>this</code> for easy chaining
+	 * @see AuthorizationInterceptor If search narrowing by code is being used for security reasons, consider also using AuthorizationInterceptor as a failsafe to ensure that no inapproproiate resources are returned
+	 * @since 6.0.0
+	 */
+	public AuthorizedList addCodeNotInValueSet(@Nonnull String theResourceName, @Nonnull String theSearchParameterName, @Nonnull String theValueSetUrl) {
+		Validate.notBlank(theResourceName, "theResourceName must not be missing or null");
+		Validate.notBlank(theSearchParameterName, "theSearchParameterName must not be missing or null");
+		Validate.notBlank(theValueSetUrl, "theResourceUrl must not be missing or null");
+
+		return doAddCodeInValueSet(theResourceName, theSearchParameterName, theValueSetUrl, true);
+	}
+
+	private AuthorizedList doAddCodeInValueSet(String theResourceName, String theSearchParameterName, String theValueSetUrl, boolean negate) {
+		if (myAllowedCodeInValueSets == null) {
+			myAllowedCodeInValueSets = new ArrayList<>();
+		}
+		myAllowedCodeInValueSets.add(new AllowedCodeInValueSet(theResourceName, theSearchParameterName, theValueSetUrl, negate));
+
+		return this;
+	}
 }

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/IAuthRuleBuilderRuleOpClassifier.java

@@ -25,6 +25,8 @@ import java.util.List;
 
 import org.hl7.fhir.instance.model.api.IIdType;
 
+import javax.annotation.Nonnull;
+
 public interface IAuthRuleBuilderRuleOpClassifier {
 
 	/**
@@ -116,4 +118,20 @@ public interface IAuthRuleBuilderRuleOpClassifier {
 	 * </p>
 	 */
 	IAuthRuleBuilderRuleOpClassifierFinished withAnyId();
+
+	/**
+	 * Rule applies to resources where the given search parameter would be satisfied by a code in the given ValueSet
+	 * @param theSearchParameterName The search parameter name, e.g. <code>"code"</code>
+	 * @param theValueSetUrl The valueset URL, e.g. <code>"http://my-value-set"</code>
+	 * @since 6.0.0
+	 */
+	IAuthRuleBuilderRuleOpClassifierFinished withCodeInValueSet(@Nonnull String theSearchParameterName, @Nonnull String theValueSetUrl);
+
+	/**
+	 * Rule applies to resources where the given search parameter would be satisfied by a code not in the given ValueSet
+	 * @param theSearchParameterName The search parameter name, e.g. <code>"code"</code>
+	 * @param theValueSetUrl The valueset URL, e.g. <code>"http://my-value-set"</code>
+	 * @since 6.0.0
+	 */
+	IAuthRuleFinished withCodeNotInValueSet(@Nonnull String theSearchParameterName, @Nonnull String theValueSetUrl);
 }

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/IRuleApplier.java

@@ -20,6 +20,7 @@ package ca.uhn.fhir.rest.server.interceptor.auth;
  * #L%
  */
 
+import ca.uhn.fhir.context.support.IValidationSupport;
 import ca.uhn.fhir.interceptor.api.Pointcut;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
@@ -27,9 +28,18 @@ import org.hl7.fhir.instance.model.api.IIdType;
 import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.server.interceptor.auth.AuthorizationInterceptor.Verdict;
+import org.slf4j.Logger;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 public interface IRuleApplier {
 
+	@Nonnull
+	Logger getTroubleshootingLog();
+
 	Verdict applyRulesAndReturnDecision(RestOperationTypeEnum theOperation, RequestDetails theRequestDetails, IBaseResource theInputResource, IIdType theInputResourceId, IBaseResource theOutputResource, Pointcut thePointcut);
 
+	@Nullable
+	IValidationSupport getValidationSupport();
 }

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/RuleBuilder.java

@@ -472,22 +472,26 @@ public class RuleBuilder implements IAuthRuleBuilder {
 				}
 
 				private RuleBuilderFinished finished() {
-					Validate.isTrue(myRule == null, "Can not call finished() twice");
+					return finished(new RuleImplOp(myRuleName));
-					myRule = new RuleImplOp(myRuleName);
+				}
-					myRule.setMode(myRuleMode);
-					myRule.setOp(myRuleOp);
-					myRule.setAppliesTo(myAppliesTo);
-					myRule.setAppliesToTypes(myAppliesToTypes);
-					myRule.setAppliesToInstances(myAppliesToInstances);
-					myRule.setClassifierType(myClassifierType);
-					myRule.setClassifierCompartmentName(myInCompartmentName);
-					myRule.setClassifierCompartmentOwners(myInCompartmentOwners);
-					myRule.setAppliesToDeleteCascade(myOnCascade);
-					myRule.setAppliesToDeleteExpunge(myOnExpunge);
-					myRule.setAdditionalSearchParamsForCompartmentTypes(myAdditionalSearchParamsForCompartmentTypes);
-					myRules.add(myRule);
 
-					return new RuleBuilderFinished(myRule);
+				private RuleBuilderFinished finished(RuleImplOp theRule) {
+					Validate.isTrue(myRule == null, "Can not call finished() twice");
+					myRule = theRule;
+					theRule.setMode(myRuleMode);
+					theRule.setOp(myRuleOp);
+					theRule.setAppliesTo(myAppliesTo);
+					theRule.setAppliesToTypes(myAppliesToTypes);
+					theRule.setAppliesToInstances(myAppliesToInstances);
+					theRule.setClassifierType(myClassifierType);
+					theRule.setClassifierCompartmentName(myInCompartmentName);
+					theRule.setClassifierCompartmentOwners(myInCompartmentOwners);
+					theRule.setAppliesToDeleteCascade(myOnCascade);
+					theRule.setAppliesToDeleteExpunge(myOnExpunge);
+					theRule.setAdditionalSearchParamsForCompartmentTypes(myAdditionalSearchParamsForCompartmentTypes);
+					myRules.add(theRule);
+
+					return new RuleBuilderFinished(theRule);
 				}
 
 				@Override
@@ -554,6 +558,24 @@ public class RuleBuilder implements IAuthRuleBuilder {
 					return finished();
 				}
 
+				@Override
+				public IAuthRuleBuilderRuleOpClassifierFinished withCodeInValueSet(@Nonnull String theSearchParameterName, @Nonnull String theValueSetUrl) {
+					SearchParameterAndValueSetRuleImpl rule = new SearchParameterAndValueSetRuleImpl(myRuleName);
+					rule.setSearchParameterName(theSearchParameterName);
+					rule.setValueSetUrl(theValueSetUrl);
+					rule.setWantCode(true);
+					return finished(rule);
+				}
+
+				@Override
+				public IAuthRuleFinished withCodeNotInValueSet(@Nonnull String theSearchParameterName, @Nonnull String theValueSetUrl) {
+					SearchParameterAndValueSetRuleImpl rule = new SearchParameterAndValueSetRuleImpl(myRuleName);
+					rule.setSearchParameterName(theSearchParameterName);
+					rule.setValueSetUrl(theValueSetUrl);
+					rule.setWantCode(false);
+					return finished(rule);
+				}
+
 				RuleBuilderFinished addInstances(Collection<IIdType> theInstances) {
 					myAppliesToInstances.addAll(theInstances);
 					return new RuleBuilderFinished(myRule);

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/RuleImplOp.java

@@ -298,11 +298,21 @@ class RuleImplOp extends BaseRule /* implements IAuthRule */ {
 				throw new IllegalStateException(Msg.code(336) + "Unable to apply security to event of applies to type " + myAppliesTo);
 		}
 
+		return applyRuleLogic(theOperation, theRequestDetails, theInputResource, theInputResourceId, theOutputResource, theFlags, ctx, target, theRuleApplier);
+	}
+
+	/**
+	 * Apply any special processing logic specific to this rule.
+	 * This is intended to be overridden.
+	 *
+	 * TODO: At this point {@link RuleImplOp} handles "any ID" and "in compartment" logic - It would be nice to split these into separate classes.
+	 */
+	protected Verdict applyRuleLogic(RestOperationTypeEnum theOperation, RequestDetails theRequestDetails, IBaseResource theInputResource, IIdType theInputResourceId, IBaseResource theOutputResource, Set<AuthorizationFlagsEnum> theFlags, FhirContext theFhirContext, RuleTarget theRuleTarget, IRuleApplier theRuleApplier) {
 		switch (myClassifierType) {
 			case ANY_ID:
 				break;
 			case IN_COMPARTMENT:
-				return applyRuleToCompartment(theOperation, theRequestDetails, theInputResource, theInputResourceId, theOutputResource, theFlags, ctx, target);
+				return applyRuleToCompartment(theOperation, theRequestDetails, theInputResource, theInputResourceId, theOutputResource, theFlags, theFhirContext, theRuleTarget);
 			default:
 				throw new IllegalStateException(Msg.code(337) + "Unable to apply security to event of applies to type " + myAppliesTo);
 		}
@@ -704,4 +714,5 @@ class RuleImplOp extends BaseRule /* implements IAuthRule */ {
 	public void setAdditionalSearchParamsForCompartmentTypes(AdditionalCompartmentSearchParameters theAdditionalParameters) {
 		myAdditionalCompartmentSearchParamMap = theAdditionalParameters;
 	}
+
 }

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/SearchNarrowingInterceptor.java

@@ -23,6 +23,7 @@ package ca.uhn.fhir.rest.server.interceptor.auth;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
 import ca.uhn.fhir.context.RuntimeSearchParam;
+import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.interceptor.api.Hook;
 import ca.uhn.fhir.interceptor.api.Pointcut;
 import ca.uhn.fhir.rest.api.Constants;
@@ -31,23 +32,31 @@ import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.param.ParameterUtil;
 import ca.uhn.fhir.rest.server.exceptions.AuthenticationException;
+import ca.uhn.fhir.rest.server.exceptions.ForbiddenOperationException;
 import ca.uhn.fhir.rest.server.method.BaseMethodBinding;
 import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
 import ca.uhn.fhir.rest.server.servlet.ServletSubRequestDetails;
 import ca.uhn.fhir.rest.server.util.ServletRequestUtil;
 import ca.uhn.fhir.util.BundleUtil;
+import ca.uhn.fhir.util.UrlUtil;
 import ca.uhn.fhir.util.bundle.ModifiableBundleEntry;
 import com.google.common.collect.ArrayListMultimap;
 import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 import org.hl7.fhir.instance.model.api.IBaseBundle;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -75,8 +84,6 @@ import java.util.stream.Collectors;
  * @see AuthorizationInterceptor
  */
 public class SearchNarrowingInterceptor {
-	private static final Logger ourLog = LoggerFactory.getLogger(SearchNarrowingInterceptor.class);
-
 
 	/**
 	 * Subclasses should override this method to supply the set of compartments that
@@ -106,7 +113,6 @@ public class SearchNarrowingInterceptor {
 
 		FhirContext ctx = theRequestDetails.getServer().getFhirContext();
 		RuntimeResourceDefinition resDef = ctx.getResourceDefinition(theRequestDetails.getResourceName());
-		HashMap<String, List<String>> parameterToOrValues = new HashMap<>();
 		AuthorizedList authorizedList = buildAuthorizedList(theRequestDetails);
 		if (authorizedList == null) {
 			return true;
@@ -118,19 +124,27 @@ public class SearchNarrowingInterceptor {
 		 */
 		Collection<String> compartments = authorizedList.getAllowedCompartments();
 		if (compartments != null) {
-			processResourcesOrCompartments(theRequestDetails, resDef, parameterToOrValues, compartments, true);
+			Map<String, List<String>> parameterToOrValues =	processResourcesOrCompartments(theRequestDetails, resDef, compartments, true);
+			applyParametersToRequestDetails(theRequestDetails, parameterToOrValues, true);
 		}
 		Collection<String> resources = authorizedList.getAllowedInstances();
 		if (resources != null) {
-			processResourcesOrCompartments(theRequestDetails, resDef, parameterToOrValues, resources, false);
+			Map<String, List<String>> parameterToOrValues = processResourcesOrCompartments(theRequestDetails, resDef, resources, false);
+			applyParametersToRequestDetails(theRequestDetails, parameterToOrValues, true);
+		}
+		List<AllowedCodeInValueSet> allowedCodeInValueSet = authorizedList.getAllowedCodeInValueSets();
+		if (allowedCodeInValueSet != null) {
+			Map<String, List<String>> parameterToOrValues = processAllowedCodes(resDef, allowedCodeInValueSet);
+			applyParametersToRequestDetails(theRequestDetails, parameterToOrValues, false);
 		}
 
-		/*
+		return true;
-		 * Add any param values to the actual request
+	}
-		 */
+
-		if (parameterToOrValues.size() > 0) {
+	private void applyParametersToRequestDetails(RequestDetails theRequestDetails, @Nullable Map<String, List<String>> theParameterToOrValues, boolean thePatientIdMode) {
+		if (theParameterToOrValues != null) {
 			Map<String, String[]> newParameters = new HashMap<>(theRequestDetails.getParameters());
-			for (Map.Entry<String, List<String>> nextEntry : parameterToOrValues.entrySet()) {
+			for (Map.Entry<String, List<String>> nextEntry : theParameterToOrValues.entrySet()) {
 				String nextParamName = nextEntry.getKey();
 				List<String> nextAllowedValues = nextEntry.getValue();
 
@@ -151,43 +165,54 @@ public class SearchNarrowingInterceptor {
 					 * requested, and the values that the user is allowed to see
 					 */
 					String[] existingValues = newParameters.get(nextParamName);
-					List<String> nextAllowedValueIds = nextAllowedValues
-						.stream()
-						.map(t -> t.lastIndexOf("/") > -1 ? t.substring(t.lastIndexOf("/") + 1) : t)
-						.collect(Collectors.toList());
-					boolean restrictedExistingList = false;
-					for (int i = 0; i < existingValues.length; i++) {
 
-						String nextExistingValue = existingValues[i];
+					if (thePatientIdMode) {
-						List<String> nextRequestedValues = QualifiedParamList.splitQueryStringByCommasIgnoreEscape(null, nextExistingValue);
+						List<String> nextAllowedValueIds = nextAllowedValues
-						List<String> nextPermittedValues = ListUtils.union(
+							.stream()
-							ListUtils.intersection(nextRequestedValues, nextAllowedValues),
+							.map(t -> t.lastIndexOf("/") > -1 ? t.substring(t.lastIndexOf("/") + 1) : t)
-							ListUtils.intersection(nextRequestedValues, nextAllowedValueIds)
+							.collect(Collectors.toList());
-						);
+						boolean restrictedExistingList = false;
-						if (nextPermittedValues.size() > 0) {
+						for (int i = 0; i < existingValues.length; i++) {
-							restrictedExistingList = true;
+
-							existingValues[i] = ParameterUtil.escapeAndJoinOrList(nextPermittedValues);
+							String nextExistingValue = existingValues[i];
+							List<String> nextRequestedValues = QualifiedParamList.splitQueryStringByCommasIgnoreEscape(null, nextExistingValue);
+							List<String> nextPermittedValues = ListUtils.union(
+								ListUtils.intersection(nextRequestedValues, nextAllowedValues),
+								ListUtils.intersection(nextRequestedValues, nextAllowedValueIds)
+							);
+							if (nextPermittedValues.size() > 0) {
+								restrictedExistingList = true;
+								existingValues[i] = ParameterUtil.escapeAndJoinOrList(nextPermittedValues);
+							}
+
 						}
 
+						/*
+						 * If none of the values that were requested by the client overlap at all
+						 * with the values that the user is allowed to see, the client shouldn't
+						 * get *any* results back. We return an error code indicating that the
+						 * caller is forbidden from accessing the resources they requested.
+						 */
+						if (!restrictedExistingList) {
+							throw new ForbiddenOperationException(Msg.code(2026) + "Value not permitted for parameter " + UrlUtil.escapeUrlParam(nextParamName));
+						}
+
+					} else {
+
+						int existingValuesCount = existingValues.length;
+						String[] newValues = Arrays.copyOf(existingValues, existingValuesCount + nextAllowedValues.size());
+						for (int i = 0; i < nextAllowedValues.size(); i++) {
+							newValues[existingValuesCount + i] = nextAllowedValues.get(i);
+						}
+						newParameters.put(nextParamName, newValues);
+
 					}
 
-					/*
-					 * If none of the values that were requested by the client overlap at all
-					 * with the values that the user is allowed to see, the client shouldn't
-					 * get *any* results back. We return an error code indicating that the
-					 * caller is forbidden from accessing the resources they requested.
-					 */
-					if (!restrictedExistingList) {
-						theResponse.setStatus(Constants.STATUS_HTTP_403_FORBIDDEN);
-						return false;
-					}
 				}
 
 			}
 			theRequestDetails.setParameters(newParameters);
 		}
-
-		return true;
 	}
 
 	@Hook(Pointcut.SERVER_INCOMING_REQUEST_PRE_HANDLED)
@@ -202,37 +227,10 @@ public class SearchNarrowingInterceptor {
 		BundleUtil.processEntries(ctx, bundle, processor);
 	}
 
-	private class BundleEntryUrlProcessor implements Consumer<ModifiableBundleEntry> {
+	@Nullable
-		private final FhirContext myFhirContext;
+	private Map<String, List<String>> processResourcesOrCompartments(RequestDetails theRequestDetails, RuntimeResourceDefinition theResDef, Collection<String> theResourcesOrCompartments, boolean theAreCompartments) {
-		private final ServletRequestDetails myRequestDetails;
+		Map<String, List<String>> retVal = null;
-		private final HttpServletRequest myRequest;
-		private final HttpServletResponse myResponse;
 
-		public BundleEntryUrlProcessor(FhirContext theFhirContext, ServletRequestDetails theRequestDetails, HttpServletRequest theRequest, HttpServletResponse theResponse) {
-			myFhirContext = theFhirContext;
-			myRequestDetails = theRequestDetails;
-			myRequest = theRequest;
-			myResponse = theResponse;
-		}
-
-		@Override
-		public void accept(ModifiableBundleEntry theModifiableBundleEntry) {
-			ArrayListMultimap<String, String> paramValues = ArrayListMultimap.create();
-
-			String url = theModifiableBundleEntry.getRequestUrl();
-
-			ServletSubRequestDetails subServletRequestDetails = ServletRequestUtil.getServletSubRequestDetails(myRequestDetails, url, paramValues);
-			BaseMethodBinding<?> method = subServletRequestDetails.getServer().determineResourceMethod(subServletRequestDetails, url);
-			RestOperationTypeEnum restOperationType = method.getRestOperationType();
-			subServletRequestDetails.setRestOperationType(restOperationType);
-
-			incomingRequestPostProcessed(subServletRequestDetails, myRequest, myResponse);
-
-			theModifiableBundleEntry.setRequestUrl(myFhirContext, ServletRequestUtil.extractUrl(subServletRequestDetails));
-		}
-	}
-
-	private void processResourcesOrCompartments(RequestDetails theRequestDetails, RuntimeResourceDefinition theResDef, HashMap<String, List<String>> theParameterToOrValues, Collection<String> theResourcesOrCompartments, boolean theAreCompartments) {
 		String lastCompartmentName = null;
 		String lastSearchParamName = null;
 		for (String nextCompartment : theResourcesOrCompartments) {
@@ -262,12 +260,44 @@ public class SearchNarrowingInterceptor {
 			}
 
 			if (searchParamName != null) {
-				List<String> orValues = theParameterToOrValues.computeIfAbsent(searchParamName, t -> new ArrayList<>());
+				if (retVal == null) {
+					retVal = new HashMap<>();
+				}
+				List<String> orValues = retVal.computeIfAbsent(searchParamName, t -> new ArrayList<>());
 				orValues.add(nextCompartment);
 			}
 		}
+
+		return retVal;
 	}
 
+	@Nullable
+	private Map<String, List<String>> processAllowedCodes(RuntimeResourceDefinition theResDef, List<AllowedCodeInValueSet> theAllowedCodeInValueSet) {
+		Map<String, List<String>> retVal = null;
+
+		for (AllowedCodeInValueSet next : theAllowedCodeInValueSet) {
+			if (!next.getResourceName().equals(theResDef.getName())) {
+				continue;
+			}
+
+			String paramName;
+			if (next.isNegate()) {
+				paramName = next.getSearchParameterName() + Constants.PARAMQUALIFIER_TOKEN_NOT_IN;
+			} else {
+				paramName = next.getSearchParameterName() + Constants.PARAMQUALIFIER_TOKEN_IN;
+			}
+
+			if (retVal == null) {
+				retVal = new HashMap<>();
+			}
+			retVal.computeIfAbsent(paramName, k->new ArrayList<>()).add(next.getValueSetUrl());
+		}
+
+		return retVal;
+	}
+
+
+
 	private String selectBestSearchParameterForCompartment(RequestDetails theRequestDetails, RuntimeResourceDefinition theResDef, String compartmentName) {
 		String searchParamName = null;
 
@@ -333,4 +363,34 @@ public class SearchNarrowingInterceptor {
 		}
 	}
 
+	private class BundleEntryUrlProcessor implements Consumer<ModifiableBundleEntry> {
+		private final FhirContext myFhirContext;
+		private final ServletRequestDetails myRequestDetails;
+		private final HttpServletRequest myRequest;
+		private final HttpServletResponse myResponse;
+
+		public BundleEntryUrlProcessor(FhirContext theFhirContext, ServletRequestDetails theRequestDetails, HttpServletRequest theRequest, HttpServletResponse theResponse) {
+			myFhirContext = theFhirContext;
+			myRequestDetails = theRequestDetails;
+			myRequest = theRequest;
+			myResponse = theResponse;
+		}
+
+		@Override
+		public void accept(ModifiableBundleEntry theModifiableBundleEntry) {
+			ArrayListMultimap<String, String> paramValues = ArrayListMultimap.create();
+
+			String url = theModifiableBundleEntry.getRequestUrl();
+
+			ServletSubRequestDetails subServletRequestDetails = ServletRequestUtil.getServletSubRequestDetails(myRequestDetails, url, paramValues);
+			BaseMethodBinding<?> method = subServletRequestDetails.getServer().determineResourceMethod(subServletRequestDetails, url);
+			RestOperationTypeEnum restOperationType = method.getRestOperationType();
+			subServletRequestDetails.setRestOperationType(restOperationType);
+
+			incomingRequestPostProcessed(subServletRequestDetails, myRequest, myResponse);
+
+			theModifiableBundleEntry.setRequestUrl(myFhirContext, ServletRequestUtil.extractUrl(subServletRequestDetails));
+		}
+	}
+
 }

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/auth/SearchParameterAndValueSetRuleImpl.java

@@ -0,0 +1,163 @@
+package ca.uhn.fhir.rest.server.interceptor.auth;
+
+/*-
+ * #%L
+ * HAPI FHIR - Server Framework
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.context.RuntimeResourceDefinition;
+import ca.uhn.fhir.context.RuntimeSearchParam;
+import ca.uhn.fhir.context.support.ConceptValidationOptions;
+import ca.uhn.fhir.context.support.IValidationSupport;
+import ca.uhn.fhir.context.support.ValidationSupportContext;
+import ca.uhn.fhir.i18n.Msg;
+import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
+import ca.uhn.fhir.rest.api.server.RequestDetails;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import ca.uhn.fhir.util.FhirTerser;
+import org.apache.commons.lang3.Validate;
+import org.hl7.fhir.instance.model.api.IBase;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.instance.model.api.ICompositeType;
+import org.hl7.fhir.instance.model.api.IIdType;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
+class SearchParameterAndValueSetRuleImpl extends RuleImplOp {
+
+	private String mySearchParameterName;
+	private String myValueSetUrl;
+	private boolean myWantCode;
+
+	/**
+	 * Constructor
+	 *
+	 * @param theRuleName The rule name
+	 */
+	SearchParameterAndValueSetRuleImpl(String theRuleName) {
+		super(theRuleName);
+	}
+
+	void setWantCode(boolean theWantCode) {
+		myWantCode = theWantCode;
+	}
+
+	public void setSearchParameterName(String theSearchParameterName) {
+		mySearchParameterName = theSearchParameterName;
+	}
+
+	public void setValueSetUrl(String theValueSetUrl) {
+		myValueSetUrl = theValueSetUrl;
+	}
+
+
+	@Override
+	protected AuthorizationInterceptor.Verdict applyRuleLogic(RestOperationTypeEnum theOperation, RequestDetails theRequestDetails, IBaseResource theInputResource, IIdType theInputResourceId, IBaseResource theOutputResource, Set<AuthorizationFlagsEnum> theFlags, FhirContext theFhirContext, RuleTarget theRuleTarget, IRuleApplier theRuleApplier) {
+		// Sanity check
+		Validate.isTrue(theInputResource == null || theOutputResource == null);
+
+		if (theInputResource != null) {
+			return applyRuleLogic(theFhirContext, theRequestDetails, theInputResource, theOperation, theInputResource, theInputResourceId, theOutputResource, theRuleApplier);
+		}
+		if (theOutputResource != null) {
+			return applyRuleLogic(theFhirContext, theRequestDetails, theOutputResource, theOperation, theInputResource, theInputResourceId, theOutputResource, theRuleApplier);
+		}
+
+		// No resource present
+		if (theOperation == RestOperationTypeEnum.READ || theOperation == RestOperationTypeEnum.SEARCH_TYPE) {
+			return new AuthorizationInterceptor.Verdict(PolicyEnum.ALLOW, this);
+		}
+
+		return null;
+	}
+
+	private AuthorizationInterceptor.Verdict applyRuleLogic(FhirContext theFhirContext, RequestDetails theRequestDetails, IBaseResource theResource, RestOperationTypeEnum theOperation, IBaseResource theInputResource, IIdType theInputResourceId, IBaseResource theOutputResource, IRuleApplier theRuleApplier) {
+		IValidationSupport validationSupport = theRuleApplier.getValidationSupport();
+		if (validationSupport == null) {
+			validationSupport = theFhirContext.getValidationSupport();
+		}
+
+		FhirTerser terser = theFhirContext.newTerser();
+		ConceptValidationOptions conceptValidationOptions = new ConceptValidationOptions();
+		ValidationSupportContext validationSupportContext = new ValidationSupportContext(validationSupport);
+
+		RuntimeResourceDefinition resourceDefinition = theFhirContext.getResourceDefinition(theResource);
+		RuntimeSearchParam searchParameter = resourceDefinition.getSearchParam(mySearchParameterName);
+		if (searchParameter == null) {
+			throw new InternalErrorException(Msg.code(2025) + "Unknown SearchParameter for resource " + resourceDefinition.getName() + ": " + mySearchParameterName);
+		}
+
+		theRuleApplier
+			.getTroubleshootingLog()
+			.debug("Applying {}:{} rule for valueSet: {}", mySearchParameterName, myWantCode ? "in" : "not-in", myValueSetUrl);
+
+		List<String> paths = searchParameter.getPathsSplitForResourceType(resourceDefinition.getName());
+
+		for (String nextPath : paths) {
+			List<ICompositeType> foundCodeableConcepts = theFhirContext.newFhirPath().evaluate(theResource, nextPath, ICompositeType.class);
+			int codeCount = 0;
+			int matchCount = 0;
+			for (ICompositeType nextCodeableConcept : foundCodeableConcepts) {
+				for (IBase nextCoding : terser.getValues(nextCodeableConcept, "coding")) {
+					String system = terser.getSinglePrimitiveValueOrNull(nextCoding, "system");
+					String code = terser.getSinglePrimitiveValueOrNull(nextCoding, "code");
+					if (isNotBlank(system) && isNotBlank(code)) {
+						codeCount++;
+						IValidationSupport.CodeValidationResult validateCodeResult = validationSupport.validateCode(validationSupportContext, conceptValidationOptions, system, code, null, myValueSetUrl);
+						if (validateCodeResult != null) {
+							if (validateCodeResult.isOk()) {
+								if (myWantCode) {
+									AuthorizationInterceptor.Verdict verdict = newVerdict(theOperation, theRequestDetails, theInputResource, theInputResourceId, theOutputResource);
+									theRuleApplier
+										.getTroubleshootingLog()
+										.debug("Code {}:{} was found in VS - Verdict: {}", system, code, verdict);
+									return verdict;
+								} else {
+									matchCount++;
+									break;
+								}
+							} else {
+								theRuleApplier
+									.getTroubleshootingLog()
+									.debug("Code {}:{} was not found in VS", system, code);
+							}
+						}
+					}
+				}
+			}
+
+			if (!myWantCode) {
+				if ((getMode() == PolicyEnum.ALLOW && matchCount == 0) ||
+					(getMode() == PolicyEnum.DENY && matchCount < codeCount)) {
+					AuthorizationInterceptor.Verdict verdict = newVerdict(theOperation, theRequestDetails, theInputResource, theInputResourceId, theOutputResource);
+					theRuleApplier
+						.getTroubleshootingLog()
+						.debug("Code was found in VS - Verdict: {}", verdict);
+					return verdict;
+				}
+			}
+
+		}
+
+		return null;
+	}
+}

hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot-samples</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-sample-client-apache</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot-samples</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-sample-client-okhttp</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot-samples</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-sample-server-jersey</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-samples</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml

@@ -5,7 +5,7 @@
    <parent>
       <groupId>ca.uhn.hapi.fhir</groupId>
       <artifactId>hapi-deployable-pom</artifactId>
-      <version>6.0.0-PRE1-SNAPSHOT</version>
+      <version>6.0.0-PRE2-SNAPSHOT</version>
       <relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
    </parent>
 

hapi-fhir-spring-boot/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-sql-migrate/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-storage/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-dstu2.1/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-dstu2/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-dstu3/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-hl7org-dstu2/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.0.0-PRE1-SNAPSHOT</version>
+		<version>6.0.0-PRE2-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>