Merge remote-tracking branch 'origin/master' into 2161-refactor-empi

hapi-fhir-base/src/main/resources/ca/uhn/fhir/i18n/hapi-messages.properties

@@ -1,5 +1,8 @@
 
 ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl.expansionRefersToUnknownCs=Unknown CodeSystem URI "{0}" referenced from ValueSet
+ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl.valueSetNotYetExpanded=ValueSet "{0}" has not yet been pre-expanded. Performing in-memory expansion without parameters. Current status: {1} | {2}
+ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl.valueSetNotYetExpanded_OffsetNotAllowed=ValueSet expansion can not combine "offset" with "ValueSet.compose.exclude" unless the ValueSet has been pre-expanded. ValueSet "{0}" must be pre-expanded for this operation to work.
+
 
 # Core Library Messages
 ca.uhn.fhir.context.FhirContext.unknownResourceName=Unknown resource name "{0}" (this name is not known in FHIR version "{1}")

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_2_0/2162-improve-vs-expansion-filtering.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 2162
+title: "When expanding a pre-expanded ValueSet using a filter, the filter was ignored and the pre-expansion was not used
+   resulting in an inefficient and potentially incorrect expansion. This has been corrected."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_2_0/2164-package-description-longer-than-200-characters.yaml

@@ -0,0 +1,4 @@
+---
+type: fix
+issue: 2164
+title: "The JPA Package loader was failing if the package had a description longer than 200 characters. This has been fixed."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_plain/rest_operations.md

@@ -389,7 +389,7 @@ Tags have very specific semantics, which may not be obvious simply by using the
 
 ## Accessing Tags in a Read / VRead / Search Method
 
-Tags are stored within a resource object, in the Resource.meta element.
+Tags are stored within a resource object, in the Resource.meta element. It is important to note that changing a resource's tags will not cause a version update to that resource.
 
 In a server implementation, you may populate your tags into the returned resource(s) and HAPI will automatically place these tags into the response headers (for read/vread) or the bundle category tags (for search). The following example illustrates how to return tags from a server method. This example shows how to supply tags in a read method, but the same approach applies to vread and search operations as well.
 
@@ -409,12 +409,17 @@ Within a [Type Create](#type_create) or [Instance Update](#instance_update) meth
 
 Note that FHIR specifies that in an update method, any tags supplied by the client are copied to the newly saved version, as well as any tags the existing version had.
 
-To work with tags in a create/update method, the pattern used in the read examples above is simply revered. In a server, the resource which is passed in will be populated with any tags that the client supplied:
+To work with tags in a create/update method, the pattern used in the read examples above is simply reversed. In a server, the resource which is passed in will be populated with any tags that the client supplied:
 
 ```java
 {{snippet:classpath:/ca/uhn/hapi/fhir/docs/RestfulPatientResourceProviderMore.java|createTags}}
 ``` 
 
+## Removing Tags
+
+In order to remove a tag, it does not suffice to remove it from the resource. Tags can be removed using the [Resource Operation Meta Delete](https://www.hl7.org/fhir/resource-operation-meta-delete.html), which takes a Parameter 
+definining which tags to delete. 
+
 # Handling _summary and _elements
 
 The `_summary` and `_elements` parameters are automatically handled by the server, so no coding is required to make this work. 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptViewDao.java

@@ -1,6 +1,8 @@
 package ca.uhn.fhir.jpa.dao.data;
 
 import ca.uhn.fhir.jpa.entity.TermValueSetConceptView;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
@@ -32,4 +34,7 @@ public interface ITermValueSetConceptViewDao extends JpaRepository<TermValueSetC
 	@Query("SELECT v FROM TermValueSetConceptView v WHERE v.myConceptValueSetPid = :pid AND v.myConceptOrder >= :from AND v.myConceptOrder < :to ORDER BY v.myConceptOrder")
 	List<TermValueSetConceptView> findByTermValueSetId(@Param("from") int theFrom, @Param("to") int theTo, @Param("pid") Long theValueSetId);
 
+	@Query("SELECT v FROM TermValueSetConceptView v WHERE v.myConceptValueSetPid = :pid AND v.myConceptDisplay LIKE :display ORDER BY v.myConceptOrder")
+	List<TermValueSetConceptView> findByTermValueSetId(@Param("pid") Long theValueSetId, @Param("display") String theDisplay);
+
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java

@@ -27,6 +27,7 @@ import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet;
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao;
 import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
@@ -221,7 +222,7 @@ public class FhirResourceDaoValueSetR4 extends BaseHapiFhirResourceDao<ValueSet>
 	private void addFilterIfPresent(String theFilter, ConceptSetComponent include) {
 		if (ElementUtil.isEmpty(include.getConcept())) {
 			if (isNotBlank(theFilter)) {
-				include.addFilter().setProperty("display").setOp(FilterOperator.EQUAL).setValue(theFilter);
+				include.addFilter().setProperty(JpaConstants.VALUESET_FILTER_DISPLAY).setOp(FilterOperator.EQUAL).setValue(theFilter);
 			}
 		}
 	}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/tx/HapiTransactionService.java

@@ -94,7 +94,6 @@ public class HapiTransactionService {
 			}
 		}
 
-
 	}
 
 	/**

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java

@@ -40,7 +40,7 @@ import static org.apache.commons.lang3.StringUtils.length;
  * bork up migration tasks.
  */
 @Table(name = "TRM_VALUESET_CONCEPT", uniqueConstraints = {
-	@UniqueConstraint(name = "IDX_VS_CONCEPT_CS_CD", columnNames = {"VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL"}),
+	@UniqueConstraint(name = "IDX_VS_CONCEPT_CS_CODE", columnNames = {"VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL"}),
 	@UniqueConstraint(name = "IDX_VS_CONCEPT_ORDER", columnNames = {"VALUESET_PID", "VALUESET_ORDER"})
 })
 @Entity()

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java

@@ -217,10 +217,16 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 			}
 
 			boolean currentVersion = updateCurrentVersionFlagForAllPackagesBasedOnNewIncomingVersion(thePackageId, packageVersionId);
+			String packageDesc;
+			if (npmPackage.description().length() > NpmPackageVersionEntity.PACKAGE_DESC_LENGTH) {
+				packageDesc = npmPackage.description().substring(0, NpmPackageVersionEntity.PACKAGE_DESC_LENGTH - 4) + "...";
+			} else {
+				packageDesc = npmPackage.description();
+			}
 			if (currentVersion) {
 				getProcessingMessages(npmPackage).add("Marking package " + thePackageId + "#" + thePackageVersionId + " as current version");
 				pkg.setCurrentVersionId(packageVersionId);
-				pkg.setDescription(npmPackage.description());
+				pkg.setDescription(packageDesc);
 				myPackageDao.save(pkg);
 			} else {
 				getProcessingMessages(npmPackage).add("Package " + thePackageId + "#" + thePackageVersionId + " is not the newest version");
@@ -232,7 +238,7 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 			packageVersion.setPackage(pkg);
 			packageVersion.setPackageBinary(persistedPackage);
 			packageVersion.setSavedTime(new Date());
-			packageVersion.setDescription(npmPackage.description());
+			packageVersion.setDescription(packageDesc);
 			packageVersion.setFhirVersionId(npmPackage.fhirVersion());
 			packageVersion.setFhirVersion(fhirVersion);
 			packageVersion.setCurrentVersion(currentVersion);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java

@@ -37,8 +37,6 @@ import ca.uhn.fhir.jpa.entity.Search;
 import ca.uhn.fhir.jpa.entity.SearchInclude;
 import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
 import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
-import ca.uhn.fhir.model.api.IQueryParameterType;
-import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
 import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
 import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
 import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
@@ -47,6 +45,7 @@ import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.util.InterceptorUtil;
 import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster;
+import ca.uhn.fhir.model.api.IQueryParameterType;
 import ca.uhn.fhir.model.api.Include;
 import ca.uhn.fhir.rest.api.CacheControlDirective;
 import ca.uhn.fhir.rest.api.Constants;
@@ -468,7 +467,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 		TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
 		txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
 		return txTemplate.execute(t -> {
-				
+
 			// Load the results synchronously
 			final List<ResourcePersistentId> pids = new ArrayList<>();
 
@@ -668,6 +667,12 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 		myRequestPartitionHelperService = theRequestPartitionHelperService;
 	}
 
+	private boolean isWantCount(SearchParameterMap myParams, boolean wantOnlyCount) {
+		return wantOnlyCount ||
+			SearchTotalModeEnum.ACCURATE.equals(myParams.getSearchTotalMode()) ||
+			(myParams.getSearchTotalMode() == null && SearchTotalModeEnum.ACCURATE.equals(myDaoConfig.getDefaultTotalMode()));
+	}
+
 	/**
 	 * A search task is a Callable task that runs in
 	 * a thread pool to handle an individual search. One instance
@@ -691,6 +696,8 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 		private final ArrayList<ResourcePersistentId> myUnsyncedPids = new ArrayList<>();
 		private final RequestDetails myRequest;
 		private final RequestPartitionId myRequestPartitionId;
+		private final SearchRuntimeDetails mySearchRuntimeDetails;
+		private final Transaction myParentTransaction;
 		private Search mySearch;
 		private boolean myAbortRequested;
 		private int myCountSavedTotal = 0;
@@ -699,8 +706,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 		private boolean myAdditionalPrefetchThresholdsRemaining;
 		private List<ResourcePersistentId> myPreviouslyAddedResourcePids;
 		private Integer myMaxResultsToFetch;
-		private final SearchRuntimeDetails mySearchRuntimeDetails;
-		private final Transaction myParentTransaction;
 
 		/**
 		 * Constructor
@@ -1193,17 +1198,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 		}
 	}
 
-	private boolean isWantCount(SearchParameterMap myParams, boolean wantOnlyCount) {
-		return wantOnlyCount ||
-			SearchTotalModeEnum.ACCURATE.equals(myParams.getSearchTotalMode()) ||
-			(myParams.getSearchTotalMode() == null && SearchTotalModeEnum.ACCURATE.equals(myDaoConfig.getDefaultTotalMode()));
-	}
-
-	private static boolean isWantOnlyCount(SearchParameterMap myParams) {
-		return SummaryEnum.COUNT.equals(myParams.getSummaryMode())
-			| INTEGER_0.equals(myParams.getCount());
-	}
-
 	public class SearchContinuationTask extends SearchTask {
 
 		public SearchContinuationTask(Search theSearch, IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequest, RequestPartitionId theRequestPartitionId) {
@@ -1242,6 +1236,10 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 
 	}
 
+	private static boolean isWantOnlyCount(SearchParameterMap myParams) {
+		return SummaryEnum.COUNT.equals(myParams.getSummaryMode())
+			| INTEGER_0.equals(myParams.getCount());
+	}
 
 	public static void populateSearchEntity(SearchParameterMap theParams, String theResourceType, String theSearchUuid, String theQueryString, Search theSearch) {
 		theSearch.setDeleted(false);
@@ -1270,8 +1268,8 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 	 * Creates a {@link Pageable} using a start and end index
 	 */
 	@SuppressWarnings("WeakerAccess")
-	public static @Nullable
+	@Nullable
-	Pageable toPage(final int theFromIndex, int theToIndex) {
+	public static Pageable toPage(final int theFromIndex, int theToIndex) {
 		int pageSize = theToIndex - theFromIndex;
 		if (pageSize < 1) {
 			return null;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java

@@ -705,6 +705,8 @@ public class SearchBuilder implements ISearchBuilder {
 						List<Long> results = q.getResultList();
 						for (Long resourceLink : results) {
 							if (resourceLink == null) {
+								// This can happen if there are outgoing references which are canonical or point to
+								// other servers
 								continue;
 							}
 							if (theReverseMode) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/UriPredicateBuilder.java

@@ -77,6 +77,7 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
 	public Condition addPredicate(List<? extends IQueryParameterType> theUriOrParameterList, String theParamName, SearchFilterParser.CompareOperation theOperation, RequestDetails theRequestDetails) {
 
 		List<Condition> codePredicates = new ArrayList<>();
+		boolean predicateIsHash = false;
 		for (IQueryParameterType nextOr : theUriOrParameterList) {
 
 			if (nextOr instanceof UriParam) {
@@ -141,8 +142,8 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
 					Condition uriPredicate = null;
 					if (theOperation == null || theOperation == SearchFilterParser.CompareOperation.eq) {
 						long hashUri = ResourceIndexedSearchParamUri.calculateHashUri(getPartitionSettings(), getRequestPartitionId(), getResourceType(), theParamName, value);
-						Condition hashPredicate = BinaryCondition.equalTo(myColumnHashUri, generatePlaceholder(hashUri));
+						uriPredicate = BinaryCondition.equalTo(myColumnHashUri, generatePlaceholder(hashUri));
-						codePredicates.add(hashPredicate);
+						predicateIsHash = true;
 					} else if (theOperation == SearchFilterParser.CompareOperation.ne) {
 						uriPredicate = BinaryCondition.notEqualTo(myColumnUri, generatePlaceholder(value));
 					} else if (theOperation == SearchFilterParser.CompareOperation.co) {
@@ -164,11 +165,7 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
 							theOperation.toString()));
 					}
 
-					if (uriPredicate != null) {
+					codePredicates.add(uriPredicate);
-						long hashIdentity = BaseResourceIndexedSearchParam.calculateHashIdentity(getPartitionSettings(), getRequestPartitionId(), getResourceType(), theParamName);
-						BinaryCondition hashIdentityPredicate = BinaryCondition.equalTo(getColumnHashIdentity(), generatePlaceholder(hashIdentity));
-						codePredicates.add(ComboCondition.and(hashIdentityPredicate, uriPredicate));
-					}
 				}
 
 			} else {
@@ -186,8 +183,11 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
 		}
 
 		ComboCondition orPredicate = ComboCondition.or(codePredicates.toArray(new Condition[0]));
-		Condition outerPredicate = combineWithHashIdentityPredicate(getResourceType(), theParamName, orPredicate);
+		if (predicateIsHash) {
-		return outerPredicate;
+			return orPredicate;
+		} else {
+			return combineWithHashIdentityPredicate(getResourceType(), theParamName, orPredicate);
+		}
 
 	}
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java

@@ -66,6 +66,7 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.model.sched.HapiJob;
 import ca.uhn.fhir.jpa.model.sched.ISchedulerService;
 import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
 import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
 import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
@@ -81,14 +82,17 @@ import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
 import ca.uhn.fhir.util.CoverageIgnore;
+import ca.uhn.fhir.util.FhirVersionIndependentConcept;
+import ca.uhn.fhir.util.HapiExtensions;
 import ca.uhn.fhir.util.StopWatch;
 import ca.uhn.fhir.util.UrlUtil;
 import ca.uhn.fhir.util.ValidateUtil;
-import ca.uhn.fhir.util.FhirVersionIndependentConcept;
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Stopwatch;
+import com.google.common.collect.ArrayListMultimap;
+import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 import org.apache.commons.lang3.time.DateUtils;
@@ -137,6 +141,7 @@ import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.transaction.interceptor.NoRollbackRuleAttribute;
 import org.springframework.transaction.interceptor.RuleBasedTransactionAttribute;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
 import org.springframework.transaction.support.TransactionTemplate;
 
 import javax.annotation.Nonnull;
@@ -158,9 +163,9 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -169,8 +174,10 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
 import static org.apache.commons.lang3.StringUtils.defaultString;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isEmpty;
@@ -248,66 +255,64 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		return cs != null;
 	}
 
-	private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, TermConcept theConcept, boolean theAdd, AtomicInteger theCodeCounter, String theValueSetIncludeVersion) {
+	private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, TermConcept theConcept, boolean theAdd, String theValueSetIncludeVersion) {
 		String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri();
 		String code = theConcept.getCode();
 		String display = theConcept.getDisplay();
 		Collection<TermConceptDesignation> designations = theConcept.getDesignations();
 		if (StringUtils.isNotEmpty(theValueSetIncludeVersion)) {
-			addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, theCodeCounter, codeSystem + "|" + theValueSetIncludeVersion, code, display);
+			return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem + "|" + theValueSetIncludeVersion, code, display);
 		} else {
-			addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, theCodeCounter, codeSystem, code, display);
+			return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem, code, display);
 		}
 	}
 
-	private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, AtomicInteger theCodeCounter, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay) {
+	private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay) {
 		if (StringUtils.isNotEmpty(theCodeSystemVersion)) {
 			if (isNoneBlank(theCodeSystem, theCode)) {
 				if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
-					theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, theDesignations);
+					theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, null);
-					theCodeCounter.incrementAndGet();
 				}
 
 				if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
 					theValueSetCodeAccumulator.excludeConcept(theCodeSystem + "|" + theCodeSystemVersion, theCode);
-					theCodeCounter.decrementAndGet();
 				}
 			}
 		} else {
 			if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
-				theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations);
+				theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, null);
-				theCodeCounter.incrementAndGet();
 			}
 
 			if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
 				theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
-				theCodeCounter.decrementAndGet();
 			}
 		}
 	}
 
-	private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, AtomicInteger theCodeCounter, String theCodeSystem, String theCode, String theDisplay) {
+	private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, String theCodeSystem, String theCode, String theDisplay) {
 		if (isNoneBlank(theCodeSystem, theCode)) {
 			if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
 				theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations);
-				theCodeCounter.incrementAndGet();
+				return true;
 			}
 
 			if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
 				theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
-				theCodeCounter.decrementAndGet();
+				return true;
 			}
 		}
+
+		return false;
 	}
 
-	private void addConceptsToList(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, String theSystem, List<CodeSystem.ConceptDefinitionComponent> theConcept, boolean theAdd, FhirVersionIndependentConcept theWantConceptOrNull) {
+	private void addConceptsToList(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, String theSystem, List<CodeSystem.ConceptDefinitionComponent> theConcept, boolean theAdd, @Nonnull ExpansionFilter theExpansionFilter) {
 		for (CodeSystem.ConceptDefinitionComponent next : theConcept) {
 			if (isNoneBlank(theSystem, next.getCode())) {
-				if (theWantConceptOrNull == null || theWantConceptOrNull.getCode().equals(next.getCode())) {
+				if (!theExpansionFilter.hasCode() || theExpansionFilter.getCode().equals(next.getCode())) {
 					addOrRemoveCode(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, next.getCode(), next.getDisplay());
 				}
 			}
-			addConceptsToList(theValueSetCodeAccumulator, theAddedCodes, theSystem, next.getConcept(), theAdd, theWantConceptOrNull);
+			addConceptsToList(theValueSetCodeAccumulator, theAddedCodes, theSystem, next.getConcept(), theAdd, theExpansionFilter);
 		}
 	}
 
@@ -387,28 +392,14 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		deleteValueSetForResource(theResourceTable);
 	}
 
-	private ValueSet expandValueSetInMemory(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, FhirVersionIndependentConcept theWantConceptOrNull) {
-
-		int maxCapacity = myDaoConfig.getMaximumExpansionSize();
-		ValueSetExpansionComponentWithConceptAccumulator expansionComponent = new ValueSetExpansionComponentWithConceptAccumulator(myContext, maxCapacity);
-		expansionComponent.setIdentifier(UUID.randomUUID().toString());
-		expansionComponent.setTimestamp(new Date());
-
-		AtomicInteger codeCounter = new AtomicInteger(0);
-
-		expandValueSet(theExpansionOptions, theValueSetToExpand, expansionComponent, codeCounter, theWantConceptOrNull);
-
-		expansionComponent.setTotal(codeCounter.get());
-
-		ValueSet valueSet = new ValueSet();
-		valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
-		valueSet.setCompose(theValueSetToExpand.getCompose());
-		valueSet.setExpansion(expansionComponent);
-		return valueSet;
-	}
 
 	@Override
 	public List<FhirVersionIndependentConcept> expandValueSet(ValueSetExpansionOptions theExpansionOptions, String theValueSet) {
+		ExpansionFilter expansionFilter = ExpansionFilter.NO_FILTER;
+		return expandValueSet(theExpansionOptions, theValueSet, expansionFilter);
+	}
+
+	private List<FhirVersionIndependentConcept> expandValueSet(ValueSetExpansionOptions theExpansionOptions, String theValueSet, ExpansionFilter theExpansionFilter) {
 		// TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
 
 		ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(theValueSet);
@@ -416,7 +407,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			throwInvalidValueSet(theValueSet);
 		}
 
-		return expandValueSetAndReturnVersionIndependentConcepts(theExpansionOptions, valueSet, null);
+		return expandValueSetAndReturnVersionIndependentConcepts(theExpansionOptions, valueSet, theExpansionFilter);
 	}
 
 	@Override
@@ -424,6 +415,45 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 	public ValueSet expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand) {
 		ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null");
 
+		ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions);
+		int offset = expansionOptions.getOffset();
+		int count = expansionOptions.getCount();
+
+		ValueSetExpansionComponentWithConceptAccumulator accumulator = new ValueSetExpansionComponentWithConceptAccumulator(myContext, count);
+		accumulator.setHardExpansionMaximumSize(myDaoConfig.getMaximumExpansionSize());
+		accumulator.setSkipCountRemaining(offset);
+		accumulator.setIdentifier(UUID.randomUUID().toString());
+		accumulator.setTimestamp(new Date());
+		accumulator.setOffset(offset);
+
+		if (theExpansionOptions != null && isHibernateSearchEnabled()) {
+			accumulator.addParameter().setName("offset").setValue(new IntegerType(offset));
+			accumulator.addParameter().setName("count").setValue(new IntegerType(count));
+		}
+
+		ExpansionFilter filter = ExpansionFilter.NO_FILTER;
+
+		expandValueSetIntoAccumulator(theValueSetToExpand, theExpansionOptions, accumulator, filter, true);
+
+		if (accumulator.getTotalConcepts() != null) {
+			accumulator.setTotal(accumulator.getTotalConcepts());
+		}
+
+		ValueSet valueSet = new ValueSet();
+		valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
+		valueSet.setCompose(theValueSetToExpand.getCompose());
+		valueSet.setExpansion(accumulator);
+
+		for (String next : accumulator.getMessages()) {
+			valueSet.getMeta().addExtension()
+				.setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE)
+				.setValue(new StringType(next));
+		}
+
+		return valueSet;
+	}
+
+	private void expandValueSetIntoAccumulator(ValueSet theValueSetToExpand, ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theAccumulator, ExpansionFilter theFilter, boolean theAdd) {
 		Optional<TermValueSet> optionalTermValueSet;
 		if (theValueSetToExpand.hasUrl()) {
 			if (theValueSetToExpand.hasVersion()) {
@@ -440,85 +470,87 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			optionalTermValueSet = Optional.empty();
 		}
 
+		/*
+		 * ValueSet doesn't exist in pre-expansion database, so perform in-memory expansion
+		 */
 		if (!optionalTermValueSet.isPresent()) {
 			ourLog.debug("ValueSet is not present in terminology tables. Will perform in-memory expansion without parameters. {}", getValueSetInfo(theValueSetToExpand));
-			return expandValueSetInMemory(theExpansionOptions, theValueSetToExpand, null); // In-memory expansion.
+			expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter);
-		}
-
-		TermValueSet termValueSet = optionalTermValueSet.get();
-
-		if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
-			ourLog.warn("{} is present in terminology tables but not ready for persistence-backed invocation of operation $expand. Will perform in-memory expansion without parameters. Current status: {} | {}",
-				getValueSetInfo(theValueSetToExpand), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
-			return expandValueSetInMemory(theExpansionOptions, theValueSetToExpand, null); // In-memory expansion.
-		}
-
-		ValueSet.ValueSetExpansionComponent expansionComponent = new ValueSet.ValueSetExpansionComponent();
-		expansionComponent.setIdentifier(UUID.randomUUID().toString());
-		expansionComponent.setTimestamp(new Date());
-
-		ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions);
-		int offset = expansionOptions.getOffset();
-		int count = expansionOptions.getCount();
-		populateExpansionComponent(expansionComponent, termValueSet, offset, count);
-
-		ValueSet valueSet = new ValueSet();
-		valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
-		valueSet.setCompose(theValueSetToExpand.getCompose());
-		valueSet.setExpansion(expansionComponent);
-		return valueSet;
-	}
-
-	private void populateExpansionComponent(ValueSet.ValueSetExpansionComponent theExpansionComponent, TermValueSet theTermValueSet, int theOffset, int theCount) {
-		int total = theTermValueSet.getTotalConcepts().intValue();
-		theExpansionComponent.setTotal(total);
-		theExpansionComponent.setOffset(theOffset);
-		theExpansionComponent.addParameter().setName("offset").setValue(new IntegerType(theOffset));
-		theExpansionComponent.addParameter().setName("count").setValue(new IntegerType(theCount));
-
-		if (theCount == 0) {
 			return;
 		}
 
-		expandConcepts(theExpansionComponent, theTermValueSet, theOffset, theCount);
+		/*
+		 * ValueSet exists in pre-expansion database, but pre-expansion is not yet complete so perform in-memory expansion
+		 */
+		TermValueSet termValueSet = optionalTermValueSet.get();
+		if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
+			String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded", getValueSetInfo(theValueSetToExpand), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
+			theAccumulator.addMessage(msg);
+			expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter);
+			return;
+		}
+
+		/*
+		 * ValueSet is pre-expanded in database so let's use that
+		 */
+		expandConcepts(theAccumulator, termValueSet, theFilter, theAdd);
 	}
 
-	private void expandConcepts(ValueSet.ValueSetExpansionComponent theExpansionComponent, TermValueSet theTermValueSet, int theOffset, int theCount) {
+
+	private void expandConcepts(IValueSetConceptAccumulator theAccumulator, TermValueSet theTermValueSet, ExpansionFilter theFilter, boolean theAdd) {
+		Integer offset = theAccumulator.getSkipCountRemaining();
+		offset = ObjectUtils.defaultIfNull(offset, 0);
+		offset = Math.min(offset, theTermValueSet.getTotalConcepts().intValue());
+
+		Integer count = theAccumulator.getCapacityRemaining();
+		count = defaultIfNull(count, myDaoConfig.getMaximumExpansionSize());
+
 		int conceptsExpanded = 0;
 		int designationsExpanded = 0;
-		int toIndex = theOffset + theCount;
+		int toIndex = offset + count;
-		Collection<TermValueSetConceptView> conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theOffset, toIndex, theTermValueSet.getId());
+
+		Collection<TermValueSetConceptView> conceptViews;
+		boolean wasFilteredResult = false;
+		if (!theFilter.getFilters().isEmpty() && JpaConstants.VALUESET_FILTER_DISPLAY.equals(theFilter.getFilters().get(0).getProperty()) && theFilter.getFilters().get(0).getOp() == ValueSet.FilterOperator.EQUAL) {
+			String displayValue = theFilter.getFilters().get(0).getValue().replace("%", "[%]") + "%";
+			conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theTermValueSet.getId(), displayValue);
+			wasFilteredResult = true;
+		} else {
+			conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId());
+			theAccumulator.consumeSkipCount(offset);
+			if (theAdd) {
+				theAccumulator.incrementOrDecrementTotalConcepts(true, theTermValueSet.getTotalConcepts().intValue());
+			}
+		}
 
 		if (conceptViews.isEmpty()) {
 			logConceptsExpanded("No concepts to expand. ", theTermValueSet, conceptsExpanded);
 			return;
 		}
 
-		Map<Long, ValueSet.ValueSetExpansionContainsComponent> pidToConcept = new HashMap<>();
+		Map<Long, FhirVersionIndependentConcept> pidToConcept = new LinkedHashMap<>();
+		ArrayListMultimap<Long, TermConceptDesignation> pidToDesignations = ArrayListMultimap.create();
+
 		for (TermValueSetConceptView conceptView : conceptViews) {
 
 			Long conceptPid = conceptView.getConceptPid();
-			ValueSet.ValueSetExpansionContainsComponent containsComponent;
-
 			if (!pidToConcept.containsKey(conceptPid)) {
-				containsComponent = theExpansionComponent.addContains();
+				String system = conceptView.getConceptSystemUrl();
-				containsComponent.setSystem(conceptView.getConceptSystemUrl());
+				String code = conceptView.getConceptCode();
-				containsComponent.setCode(conceptView.getConceptCode());
+				String display = conceptView.getConceptDisplay();
-				containsComponent.setDisplay(conceptView.getConceptDisplay());
+				FhirVersionIndependentConcept concept = new FhirVersionIndependentConcept(system, code, display);
-				pidToConcept.put(conceptPid, containsComponent);
+				pidToConcept.put(conceptPid, concept);
-			} else {
-				containsComponent = pidToConcept.get(conceptPid);
 			}
 
 			// TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to designations optional.
 			if (conceptView.getDesignationPid() != null) {
-				ValueSet.ConceptReferenceDesignationComponent designationComponent = containsComponent.addDesignation();
+				TermConceptDesignation designation = new TermConceptDesignation();
-				designationComponent.setLanguage(conceptView.getDesignationLang());
+				designation.setUseSystem(conceptView.getDesignationUseSystem());
-				designationComponent.setUse(new Coding(
+				designation.setUseCode(conceptView.getDesignationUseCode());
-					conceptView.getDesignationUseSystem(),
+				designation.setUseDisplay(conceptView.getDesignationUseDisplay());
-					conceptView.getDesignationUseCode(),
+				designation.setValue(conceptView.getDesignationVal());
-					conceptView.getDesignationUseDisplay()));
+				designation.setLanguage(conceptView.getDesignationLang());
-				designationComponent.setValue(conceptView.getDesignationVal());
+				pidToDesignations.put(conceptPid, designation);
 
 				if (++designationsExpanded % 250 == 0) {
 					logDesignationsExpanded("Expansion of designations in progress. ", theTermValueSet, designationsExpanded);
@@ -530,6 +562,33 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			}
 		}
 
+		for (Long nextPid : pidToConcept.keySet()) {
+			FhirVersionIndependentConcept concept = pidToConcept.get(nextPid);
+			List<TermConceptDesignation> designations = pidToDesignations.get(nextPid);
+			String system = concept.getSystem();
+			String code = concept.getCode();
+			String display = concept.getDisplay();
+
+			if (theAdd) {
+				if (theAccumulator.getCapacityRemaining() != null) {
+					if (theAccumulator.getCapacityRemaining() == 0) {
+						break;
+					}
+				}
+
+				theAccumulator.includeConceptWithDesignations(system, code, display, designations);
+			} else {
+				boolean removed = theAccumulator.excludeConcept(system, code);
+				if (removed) {
+					theAccumulator.incrementOrDecrementTotalConcepts(false, 1);
+				}
+			}
+		}
+
+		if (wasFilteredResult && theAdd) {
+			theAccumulator.incrementOrDecrementTotalConcepts(true, pidToConcept.size());
+		}
+
 		logDesignationsExpanded("Finished expanding designations. ", theTermValueSet, designationsExpanded);
 		logConceptsExpanded("Finished expanding concepts. ", theTermValueSet, conceptsExpanded);
 	}
@@ -549,25 +608,34 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 	@Override
 	@Transactional(propagation = Propagation.REQUIRED)
 	public void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) {
-		expandValueSet(theExpansionOptions, theValueSetToExpand, theValueSetCodeAccumulator, new AtomicInteger(0), null);
+		expandValueSet(theExpansionOptions, theValueSetToExpand, theValueSetCodeAccumulator, ExpansionFilter.NO_FILTER);
 	}
 
 	@SuppressWarnings("ConstantConditions")
-	private void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator, AtomicInteger theCodeCounter, FhirVersionIndependentConcept theWantConceptOrNull) {
+	private void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator, @Nonnull ExpansionFilter theExpansionFilter) {
 		Set<String> addedCodes = new HashSet<>();
 
 		StopWatch sw = new StopWatch();
 		String valueSetInfo = getValueSetInfo(theValueSetToExpand);
 		ourLog.debug("Working with {}", valueSetInfo);
 
+		// Offset can't be combined with excludes
+		Integer skipCountRemaining = theValueSetCodeAccumulator.getSkipCountRemaining();
+		if (skipCountRemaining != null && skipCountRemaining > 0) {
+			if (theValueSetToExpand.getCompose().getExclude().size() > 0) {
+				String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded_OffsetNotAllowed", valueSetInfo);
+				throw new InvalidRequestException(msg);
+			}
+		}
+
 		// Handle includes
 		ourLog.debug("Handling includes");
 		for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getInclude()) {
 			for (int i = 0; ; i++) {
 				int queryIndex = i;
-				Boolean shouldContinue = myTxTemplate.execute(t -> {
+				Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> {
 					boolean add = true;
-					return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, include, add, theCodeCounter, queryIndex, theWantConceptOrNull);
+					return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, include, add, queryIndex, theExpansionFilter);
 				});
 				if (!shouldContinue) {
 					break;
@@ -575,20 +643,15 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			}
 		}
 
-		// If the accumulator filled up, abort
-		if (theValueSetCodeAccumulator.getCapacityRemaining() != null && theValueSetCodeAccumulator.getCapacityRemaining() <= 0) {
-			String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myDaoConfig.getMaximumExpansionSize());
-			throw new ExpansionTooCostlyException(msg);
-		}
-
 		// Handle excludes
 		ourLog.debug("Handling excludes");
 		for (ValueSet.ConceptSetComponent exclude : theValueSetToExpand.getCompose().getExclude()) {
 			for (int i = 0; ; i++) {
 				int queryIndex = i;
-				Boolean shouldContinue = myTxTemplate.execute(t -> {
+				Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> {
 					boolean add = false;
-					return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, exclude, add, theCodeCounter, queryIndex, null);
+					ExpansionFilter expansionFilter = ExpansionFilter.NO_FILTER;
+					return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, exclude, add, queryIndex, expansionFilter);
 				});
 				if (!shouldContinue) {
 					break;
@@ -603,47 +666,49 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		ourLog.debug("Done working with {} in {}ms", valueSetInfo, sw.getMillis());
 	}
 
+	/**
+	 * Execute in a new transaction only if we aren't already in one. We do this because in some cases
+	 * when performing a VS expansion we throw an {@link ExpansionTooCostlyException} and we don't want
+	 * this to cause the TX to be marked a rollback prematurely.
+	 */
+	private <T> T executeInNewTransactionIfNeeded(Supplier<T> theAction) {
+		if (TransactionSynchronizationManager.isSynchronizationActive()) {
+			return theAction.get();
+		}
+		return myTxTemplate.execute(t->theAction.get());
+	}
+
 	private String getValueSetInfo(ValueSet theValueSet) {
 		StringBuilder sb = new StringBuilder();
 		boolean isIdentified = false;
-		sb
-			.append("ValueSet:");
-		if (theValueSet.hasId()) {
-			isIdentified = true;
-			sb
-				.append(" ValueSet.id[")
-				.append(theValueSet.getId())
-				.append("]");
-		}
 		if (theValueSet.hasUrl()) {
 			isIdentified = true;
 			sb
-				.append(" ValueSet.url[")
+				.append("ValueSet.url[")
 				.append(theValueSet.getUrl())
 				.append("]");
-		}
+		} else if (theValueSet.hasId()) {
-		if (theValueSet.hasIdentifier()) {
 			isIdentified = true;
 			sb
-				.append(" ValueSet.identifier[")
+				.append("ValueSet.id[")
-				.append(theValueSet.getIdentifierFirstRep().getSystem())
+				.append(theValueSet.getId())
-				.append("|")
-				.append(theValueSet.getIdentifierFirstRep().getValue())
 				.append("]");
 		}
 
 		if (!isIdentified) {
-			sb.append(" None of ValueSet.id, ValueSet.url, and ValueSet.identifier are provided.");
+			sb.append("Unidentified ValueSet");
 		}
 
 		return sb.toString();
 	}
 
-	protected List<FhirVersionIndependentConcept> expandValueSetAndReturnVersionIndependentConcepts(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpandR4, FhirVersionIndependentConcept theWantConceptOrNull) {
+	protected List<FhirVersionIndependentConcept> expandValueSetAndReturnVersionIndependentConcepts(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpandR4, @Nonnull ExpansionFilter theExpansionFilter) {
-		org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent expandedR4 = expandValueSetInMemory(theExpansionOptions, theValueSetToExpandR4, theWantConceptOrNull).getExpansion();
+		int maxCapacity = myDaoConfig.getMaximumExpansionSize();
+		ValueSetExpansionComponentWithConceptAccumulator accumulator = new ValueSetExpansionComponentWithConceptAccumulator(myContext, maxCapacity);
+		expandValueSet(theExpansionOptions, theValueSetToExpandR4, accumulator, theExpansionFilter);
 
 		ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
-		for (org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent nextContains : expandedR4.getContains()) {
+		for (org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent nextContains : accumulator.getContains()) {
 			retVal.add(new FhirVersionIndependentConcept(nextContains.getSystem(), nextContains.getCode(), nextContains.getDisplay(), nextContains.getVersion()));
 		}
 		return retVal;
@@ -652,7 +717,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 	/**
 	 * @return Returns true if there are potentially more results to process.
 	 */
-	private Boolean expandValueSetHandleIncludeOrExclude(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, AtomicInteger theCodeCounter, int theQueryIndex, FhirVersionIndependentConcept theWantConceptOrNull) {
+	private Boolean expandValueSetHandleIncludeOrExclude(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter) {
 
 		String system = theIncludeOrExclude.getSystem();
 		boolean hasSystem = isNotBlank(system);
@@ -660,7 +725,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 
 		if (hasSystem) {
 
-			if (theWantConceptOrNull != null && theWantConceptOrNull.getSystem() != null && !system.equals(theWantConceptOrNull.getSystem())) {
+			if (theExpansionFilter.hasCode() && theExpansionFilter.getSystem() != null && !system.equals(theExpansionFilter.getSystem())) {
 				return false;
 			}
 
@@ -669,13 +734,13 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system);
 			if (cs != null) {
 
-				return expandValueSetHandleIncludeOrExcludeUsingDatabase(theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, theAdd, theCodeCounter, theQueryIndex, theWantConceptOrNull, system, cs);
+				return expandValueSetHandleIncludeOrExcludeUsingDatabase(theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, theAdd, theQueryIndex, theExpansionFilter, system, cs);
 
 			} else {
 
-				if (theIncludeOrExclude.getConcept().size() > 0 && theWantConceptOrNull != null) {
+				if (theIncludeOrExclude.getConcept().size() > 0 && theExpansionFilter.hasCode()) {
-					if (defaultString(theIncludeOrExclude.getSystem()).equals(theWantConceptOrNull.getSystem())) {
+					if (defaultString(theIncludeOrExclude.getSystem()).equals(theExpansionFilter.getSystem())) {
-						if (theIncludeOrExclude.getConcept().stream().noneMatch(t -> t.getCode().equals(theWantConceptOrNull.getCode()))) {
+						if (theIncludeOrExclude.getConcept().stream().noneMatch(t -> t.getCode().equals(theExpansionFilter.getCode()))) {
 							return false;
 						}
 					}
@@ -691,9 +756,9 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 					// if someone creates a valueset that includes UCUM codes, since we don't have a CodeSystem resource for those
 					// but CommonCodeSystemsTerminologyService can validate individual codes.
 					List<FhirVersionIndependentConcept> includedConcepts = null;
-					if (theWantConceptOrNull != null) {
+					if (theExpansionFilter.hasCode()) {
 						includedConcepts = new ArrayList<>();
-						includedConcepts.add(theWantConceptOrNull);
+						includedConcepts.add(theExpansionFilter.toFhirVersionIndependentConcept());
 					} else if (!theIncludeOrExclude.getConcept().isEmpty()) {
 						includedConcepts = theIncludeOrExclude
 							.getConcept()
@@ -737,7 +802,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 				if (!theIncludeOrExclude.getConcept().isEmpty()) {
 					for (ValueSet.ConceptReferenceComponent next : theIncludeOrExclude.getConcept()) {
 						String nextCode = next.getCode();
-						if (theWantConceptOrNull == null || theWantConceptOrNull.getCode().equals(nextCode)) {
+						if (!theExpansionFilter.hasCode() || theExpansionFilter.getCode().equals(nextCode)) {
 							if (isNoneBlank(system, nextCode) && !theAddedCodes.contains(system + "|" + nextCode)) {
 
 								CodeSystem.ConceptDefinitionComponent code = findCode(codeSystemFromContext.getConcept(), nextCode);
@@ -751,7 +816,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 					}
 				} else {
 					List<CodeSystem.ConceptDefinitionComponent> concept = codeSystemFromContext.getConcept();
-					addConceptsToList(theValueSetCodeAccumulator, theAddedCodes, system, concept, theAdd, theWantConceptOrNull);
+					addConceptsToList(theValueSetCodeAccumulator, theAddedCodes, system, concept, theAdd, theExpansionFilter);
 				}
 
 				return false;
@@ -760,44 +825,20 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		} else if (hasValueSet) {
 
 			for (CanonicalType nextValueSet : theIncludeOrExclude.getValueSet()) {
-				ourLog.debug("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), nextValueSet.getValueAsString());
+				String valueSetUrl = nextValueSet.getValueAsString();
+				ourLog.debug("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), valueSetUrl);
 
-				List<FhirVersionIndependentConcept> expanded = expandValueSet(theExpansionOptions, nextValueSet.getValueAsString());
+				ExpansionFilter subExpansionFilter = new ExpansionFilter(theExpansionFilter, theIncludeOrExclude.getFilter(), theValueSetCodeAccumulator.getCapacityRemaining());
-				Map<String, TermCodeSystem> uriToCodeSystem = new HashMap<>();
 
-				for (FhirVersionIndependentConcept nextConcept : expanded) {
+				// TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
-					if (theAdd) {
 
-						if (!uriToCodeSystem.containsKey(nextConcept.getSystem())) {
+				ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(valueSetUrl);
-							TermCodeSystem codeSystem = myCodeSystemDao.findByCodeSystemUri(nextConcept.getSystem());
+				if (valueSet == null) {
-							uriToCodeSystem.put(nextConcept.getSystem(), codeSystem);
+					throw new ResourceNotFoundException("Unknown ValueSet: " + UrlUtil.escapeUrlParam(valueSetUrl));
-						}
-
-						TermCodeSystem codeSystem = uriToCodeSystem.get(nextConcept.getSystem());
-						if (codeSystem != null) {
-							TermCodeSystemVersion termCodeSystemVersion;
-							if (nextConcept.getSystemVersion() != null) {
-								termCodeSystemVersion = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(codeSystem.getPid(), nextConcept.getSystemVersion());
-							} else {
-								termCodeSystemVersion = codeSystem.getCurrentVersion();
-							}
-							myConceptDao
-								.findByCodeSystemAndCode(termCodeSystemVersion, nextConcept.getCode())
-								.ifPresent(concept ->
-									addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, theCodeCounter, nextConcept.getSystemVersion())
-								);
-						} else {
-							// This will happen if we're expanding against a built-in (part of FHIR) ValueSet that
-							// isn't actually in the database anywhere
-							Collection<TermConceptDesignation> emptyCollection = Collections.emptyList();
-							addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, emptyCollection, theAdd, theCodeCounter, nextConcept.getSystem(), nextConcept.getSystemVersion(), nextConcept.getCode(), nextConcept.getDisplay());
-						}
-					}
-					if (isNoneBlank(nextConcept.getSystem(), nextConcept.getCode()) && !theAdd && theAddedCodes.remove(nextConcept.getSystem() + "|" + nextConcept.getCode())) {
-						theValueSetCodeAccumulator.excludeConcept(nextConcept.getSystem(), nextConcept.getCode());
-					}
 				}
 
+				expandValueSetIntoAccumulator(valueSet, theExpansionOptions, theValueSetCodeAccumulator, subExpansionFilter, theAdd);
+
 			}
 
 			return false;
@@ -809,8 +850,12 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 
 	}
 
+	private boolean isHibernateSearchEnabled() {
+		return myFulltextSearchSvc != null;
+	}
+
 	@Nonnull
-	private Boolean expandValueSetHandleIncludeOrExcludeUsingDatabase(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, AtomicInteger theCodeCounter, int theQueryIndex, FhirVersionIndependentConcept theWantConceptOrNull, String theSystem, TermCodeSystem theCs) {
+	private Boolean expandValueSetHandleIncludeOrExcludeUsingDatabase(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter, String theSystem, TermCodeSystem theCs) {
 		String includeOrExcludeVersion = theIncludeOrExclude.getVersion();
 		TermCodeSystemVersion csv;
 		if (isEmpty(includeOrExcludeVersion)) {
@@ -824,8 +869,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		 * If FullText searching is not enabled, we can handle only basic expansions
 		 * since we're going to do it without the database.
 		 */
-		if (myFulltextSearchSvc == null) {
+		if (!isHibernateSearchEnabled()) {
-			expandWithoutHibernateSearch(theValueSetCodeAccumulator, csv, theAddedCodes, theIncludeOrExclude, theSystem, theAdd, theCodeCounter);
+			expandWithoutHibernateSearch(theValueSetCodeAccumulator, csv, theAddedCodes, theIncludeOrExclude, theSystem, theAdd);
 			return false;
 		}
 
@@ -837,8 +882,8 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 
 		bool.must(qb.keyword().onField("myCodeSystemVersionPid").matching(csv.getPid()).createQuery());
 
-		if (theWantConceptOrNull != null) {
+		if (theExpansionFilter.hasCode()) {
-			bool.must(qb.keyword().onField("myCode").matching(theWantConceptOrNull.getCode()).createQuery());
+			bool.must(qb.keyword().onField("myCode").matching(theExpansionFilter.getCode()).createQuery());
 		}
 
 		/*
@@ -850,7 +895,12 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		} else {
 			codeSystemUrlAndVersion = theSystem;
 		}
-		handleFilters(bool, codeSystemUrlAndVersion, qb, theIncludeOrExclude);
+		for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) {
+			handleFilter(codeSystemUrlAndVersion, qb, bool, nextFilter);
+		}
+		for (ValueSet.ConceptSetFilterComponent nextFilter : theExpansionFilter.getFilters()) {
+			handleFilter(codeSystemUrlAndVersion, qb, bool, nextFilter);
+		}
 
 		Query luceneQuery = bool.createQuery();
 
@@ -917,21 +967,22 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		StopWatch swForBatch = new StopWatch();
 		AtomicInteger countForBatch = new AtomicInteger(0);
 
-		List resultList = jpaQuery.getResultList();
+		List<?> resultList = jpaQuery.getResultList();
 		int resultsInBatch = resultList.size();
 		int firstResult = jpaQuery.getFirstResult();
+		int delta = 0;
 		for (Object next : resultList) {
 			count.incrementAndGet();
 			countForBatch.incrementAndGet();
 			TermConcept concept = (TermConcept) next;
-			try {
+			boolean added = addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, includeOrExcludeVersion);
-				addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, theCodeCounter, includeOrExcludeVersion);
+			if (added) {
-			} catch (ExpansionTooCostlyException e) {
+				delta++;
-				return false;
 			}
 		}
 
 		ourLog.debug("Batch expansion for {} with starting index of {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), firstResult, countForBatch, swForBatch.getMillis());
+		theValueSetCodeAccumulator.incrementOrDecrementTotalConcepts(theAdd, delta);
 
 		if (resultsInBatch < maxResultsPerBatch) {
 			ourLog.debug("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis());
@@ -959,14 +1010,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		}
 	}
 
-	private void handleFilters(BooleanJunction<?> theBool, String theCodeSystemIdentifier, QueryBuilder theQb, ValueSet.ConceptSetComponent theIncludeOrExclude) {
-		if (theIncludeOrExclude.getFilter().size() > 0) {
-			for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) {
-				handleFilter(theCodeSystemIdentifier, theQb, theBool, nextFilter);
-			}
-		}
-	}
-
 	private void handleFilter(String theCodeSystemIdentifier, QueryBuilder theQb, BooleanJunction<?> theBool, ValueSet.ConceptSetFilterComponent theFilter) {
 		if (isBlank(theFilter.getValue()) && theFilter.getOp() == null && isBlank(theFilter.getProperty())) {
 			return;
@@ -1264,7 +1307,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		}
 	}
 
-	private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, TermCodeSystemVersion theVersion, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, AtomicInteger theCodeCounter) {
+	private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, TermCodeSystemVersion theVersion, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd) {
 		ourLog.trace("Hibernate search is not enabled");
 
 		if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) {
@@ -1277,7 +1320,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 
 		if (theInclude.getConcept().isEmpty()) {
 			for (TermConcept next : theVersion.getConcepts()) {
-				addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, null, theAdd, theCodeCounter, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay());
+				addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay());
 			}
 		}
 
@@ -1285,7 +1328,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			if (!theSystem.equals(theInclude.getSystem()) && isNotBlank(theSystem)) {
 				continue;
 			}
-			addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, null, theAdd, theCodeCounter, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay());
+			addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay());
 		}
 
 
@@ -1447,14 +1490,13 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 
 	@Nullable
 	private TermCodeSystemVersion getCurrentCodeSystemVersion(String theCodeSystemIdentifier) {
-		String myVersion = getVersionFromIdentifier(theCodeSystemIdentifier);
+		String version = getVersionFromIdentifier(theCodeSystemIdentifier);
-		String key = theCodeSystemIdentifier;
+		TermCodeSystemVersion retVal = myCodeSystemCurrentVersionCache.get(theCodeSystemIdentifier, t -> myTxTemplate.execute(tx -> {
-		TermCodeSystemVersion retVal = myCodeSystemCurrentVersionCache.get(key.toString(), t -> myTxTemplate.execute(tx -> {
 			TermCodeSystemVersion csv = null;
 			TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(getUrlFromIdentifier(theCodeSystemIdentifier));
 			if (cs != null) {
-				if (myVersion != null) {
+				if (version != null) {
-					csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), myVersion);
+					csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), version);
 				} else if (cs.getCurrentVersion() != null) {
 					csv = cs.getCurrentVersion();
 				}
@@ -1642,7 +1684,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		 */
 		String conceptMapUrl = termConceptMap.getUrl();
 		String conceptMapVersion = termConceptMap.getVersion();
-		Optional<TermConceptMap> optionalExistingTermConceptMapByUrl = null;
+		Optional<TermConceptMap> optionalExistingTermConceptMapByUrl;
 		if (isBlank(conceptMapVersion)) {
 			optionalExistingTermConceptMapByUrl = myConceptMapDao.findTermConceptMapByUrlAndNullVersion(conceptMapUrl);
 		} else {
@@ -1776,9 +1818,10 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			// We have a ValueSet to pre-expand.
 			try {
 				ValueSet valueSet = txTemplate.execute(t -> {
-					TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).get();
+					TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).orElseThrow(()->new IllegalStateException("Unknown VS ID: " + valueSetToExpand.getId()));
 					return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
 				});
+				assert valueSet != null;
 
 				ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator(valueSetToExpand, myValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao);
 				expandValueSet(null, valueSet, accumulator);
@@ -2604,6 +2647,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 		String systemVersion = theCodeSystemIdentifierType != null ? getVersionFromIdentifier(theCodeSystemIdentifierType.getValueAsString()): null;
 		if (theCodingType != null) {
 			Coding canonicalizedCoding = toCanonicalCoding(theCodingType);
+			assert canonicalizedCoding != null; // Shouldn't be null, since theCodingType isn't
 			code = canonicalizedCoding.getCode();
 			system = canonicalizedCoding.getSystem();
 			systemVersion = canonicalizedCoding.getVersion();
@@ -2675,7 +2719,6 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 
 
 		String code = theCode;
-		String version = theVersion;
 		String display = theDisplay;
 
 		if (haveCodeableConcept) {
@@ -2689,7 +2732,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 				}
 				code = nextCoding.getCode();
 				display = nextCoding.getDisplay();
-				CodeValidationResult nextValidation = codeSystemValidateCode(codeSystemUrl, version, code, display);
+				CodeValidationResult nextValidation = codeSystemValidateCode(codeSystemUrl, theVersion, code, display);
 				if (nextValidation.isOk() || i == codeableConcept.getCoding().size() - 1) {
 					return nextValidation;
 				}
@@ -2705,7 +2748,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
 			display = coding.getDisplay();
 		}
 
-		return codeSystemValidateCode(codeSystemUrl, version, code, display);
+		return codeSystemValidateCode(codeSystemUrl, theVersion, code, display);
 	}
 
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ExpansionFilter.java

@@ -0,0 +1,98 @@
+package ca.uhn.fhir.jpa.term;
+
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2020 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import ca.uhn.fhir.util.FhirVersionIndependentConcept;
+import org.apache.commons.lang3.Validate;
+import org.hl7.fhir.r4.model.ValueSet;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
+class ExpansionFilter {
+
+	public static final ExpansionFilter NO_FILTER = new ExpansionFilter(null, null);
+	private final String myCode;
+	private final String mySystem;
+	private final List<ValueSet.ConceptSetFilterComponent> myFilters;
+	private final Integer myMaxCount;
+	/**
+	 * Constructor
+	 */
+	ExpansionFilter(String theSystem, String theCode) {
+		this(theSystem, theCode, Collections.emptyList(), null);
+	}
+
+	/**
+	 * Constructor
+	 */
+	ExpansionFilter(ExpansionFilter theExpansionFilter, List<ValueSet.ConceptSetFilterComponent> theFilters, Integer theMaxCount) {
+		this(theExpansionFilter.getSystem(), theExpansionFilter.getCode(), theFilters, theMaxCount);
+	}
+
+	/**
+	 * Constructor
+	 */
+	ExpansionFilter(@Nullable String theSystem, @Nullable String theCode, @Nonnull List<ValueSet.ConceptSetFilterComponent> theFilters, Integer theMaxCount) {
+		Validate.isTrue(isNotBlank(theSystem) == isNotBlank(theCode));
+		Validate.notNull(theFilters);
+
+		mySystem = theSystem;
+		myCode = theCode;
+		myFilters = theFilters;
+		myMaxCount = theMaxCount;
+	}
+
+	public List<ValueSet.ConceptSetFilterComponent> getFilters() {
+		return myFilters;
+	}
+
+	boolean hasCode() {
+		return myCode != null;
+	}
+
+	String getCode() {
+		return myCode;
+	}
+
+	String getSystem() {
+		return mySystem;
+	}
+
+	/**
+	 * Converts the system/code in this filter to a FhirVersionIndependentConcept. This method
+	 * should not be called if {@link #hasCode()} returns <code>false</code>
+	 */
+	@Nonnull
+	public FhirVersionIndependentConcept toFhirVersionIndependentConcept() {
+		Validate.isTrue(hasCode());
+
+		return new FhirVersionIndependentConcept(mySystem, myCode);
+	}
+
+	public Integer getMaxCount() {
+		return myMaxCount;
+	}
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetConceptAccumulator.java

@@ -33,11 +33,36 @@ public interface IValueSetConceptAccumulator {
 
 	void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, @Nullable Collection<TermConceptDesignation> theDesignations);
 
-	void excludeConcept(String theSystem, String theCode);
+	/**
+	 * @return Returns <code>true</code> if the code was actually present and was removed
+	 */
+	boolean excludeConcept(String theSystem, String theCode);
 
 	@Nullable
 	default Integer getCapacityRemaining() {
 		return null;
 	}
 
+	@Nullable
+	default Integer getSkipCountRemaining() {
+		return null;
+	}
+
+	@Nullable
+	default void consumeSkipCount(int theSkipCountToConsume) {
+		// nothing
+	}
+
+	/**
+	 * Add or subtract from the total concept count (this is not necessarily the same thing as the number of concepts in
+	 * the accumulator, since the <code>offset</code> and <code>count</code> parameters applied to the expansion can cause
+	 * concepts to not actually be added.
+	 *
+	 * @param theAdd   If <code>true</code>, increment. If <code>false</code>, decrement.
+	 * @param theDelta The number of codes to add or subtract
+	 */
+	default void incrementOrDecrementTotalConcepts(boolean theAdd, int theDelta) {
+		// nothing
+	}
+
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImpl.java

@@ -55,6 +55,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 
 public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
 
@@ -157,7 +158,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
 		StopWatch stopwatch = new StopWatch();
 
 		int count = Math.min(1000, myDeferredConcepts.size());
-		ourLog.info("Saving {} deferred concepts...", count);
+		ourLog.debug("Saving {} deferred concepts...", count);
 		while (codeCount < count && myDeferredConcepts.size() > 0) {
 			TermConcept next = myDeferredConcepts.remove(0);
 			if(myCodeSystemVersionDao.findById(next.getCodeSystemVersion().getPid()).isPresent()) {
@@ -174,8 +175,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
 		}
 
 		if (codeCount > 0) {
-			ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)",
+			ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({} codes/sec)",
-				codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount));
+				codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.formatThroughput(codeCount, TimeUnit.SECONDS));
 		}
 
 		if (codeCount == 0) {
@@ -198,8 +199,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
 		}
 
 		if (relCount > 0) {
-			ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({}ms / entry)",
+			ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({} entries/sec)",
-				relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(relCount));
+				relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.formatThroughput(relCount, TimeUnit.SECONDS));
 		}
 
 		if ((myDeferredConcepts.size() + myConceptLinksToSaveLater.size()) == 0) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java

@@ -81,9 +81,9 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 	}
 
 	@Override
-	public void excludeConcept(String theSystem, String theCode) {
+	public boolean excludeConcept(String theSystem, String theCode) {
 		if (isAnyBlank(theSystem, theCode)) {
-			return;
+			return false;
 		}
 
 		// Get existing entity so it can be deleted.
@@ -114,6 +114,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 				ourLog.info("Have excluded {} concepts from ValueSet[{}]", myConceptsExcluded, myTermValueSet.getUrl());
 			}
 		}
+		return false;
 	}
 
 	private TermValueSetConcept saveConcept(String theSystem, String theCode, String theDisplay) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithConceptAccumulator.java

@@ -25,19 +25,25 @@ import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
 import ca.uhn.fhir.jpa.term.ex.ExpansionTooCostlyException;
 import ca.uhn.fhir.model.api.annotation.Block;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
-import ca.uhn.fhir.util.HapiExtensions;
 import org.apache.commons.lang3.StringUtils;
-import org.hl7.fhir.r4.model.StringType;
 import org.hl7.fhir.r4.model.ValueSet;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 
 @Block()
 public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.ValueSetExpansionComponent implements IValueSetConceptAccumulator {
 	private final int myMaxCapacity;
 	private final FhirContext myContext;
-	private int myConceptsCount;
+	private int mySkipCountRemaining;
+	private int myHardExpansionMaximumSize;
+	private List<String> myMessages;
+	private int myAddedConcepts;
+	private Integer myTotalConcepts;
 
 	/**
 	 * Constructor
@@ -46,27 +52,40 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
 	 *                       an {@link InternalErrorException}
 	 */
 	ValueSetExpansionComponentWithConceptAccumulator(FhirContext theContext, int theMaxCapacity) {
-		myContext = theContext;
 		myMaxCapacity = theMaxCapacity;
-		myConceptsCount = 0;
+		myContext = theContext;
 	}
 
-	@Nullable
+	@Nonnull
 	@Override
 	public Integer getCapacityRemaining() {
-		return myMaxCapacity - myConceptsCount;
+		return (myMaxCapacity - myAddedConcepts) + mySkipCountRemaining;
+	}
+
+	public List<String> getMessages() {
+		if (myMessages == null) {
+			return Collections.emptyList();
+		}
+		return Collections.unmodifiableList(myMessages);
 	}
 
 	@Override
 	public void addMessage(String theMessage) {
-		addExtension()
+		if (myMessages == null) {
-			.setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE)
+			myMessages = new ArrayList<>();
-			.setValue(new StringType(theMessage));
+		}
+		myMessages.add(theMessage);
 	}
 
 	@Override
 	public void includeConcept(String theSystem, String theCode, String theDisplay) {
+		if (mySkipCountRemaining > 0) {
+			mySkipCountRemaining--;
+			return;
+		}
+
 		incrementConceptsCount();
+
 		ValueSet.ValueSetExpansionContainsComponent contains = this.addContains();
 		setSystemAndVersion(theSystem, contains);
 		contains.setCode(theCode);
@@ -75,7 +94,13 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
 
 	@Override
 	public void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, Collection<TermConceptDesignation> theDesignations) {
+		if (mySkipCountRemaining > 0) {
+			mySkipCountRemaining--;
+			return;
+		}
+
 		incrementConceptsCount();
+
 		ValueSet.ValueSetExpansionContainsComponent contains = this.addContains();
 		setSystemAndVersion(theSystem, contains);
 		contains.setCode(theCode);
@@ -95,11 +120,22 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
 	}
 
 	@Override
-	public void excludeConcept(String theSystem, String theCode) {
+	public void consumeSkipCount(int theSkipCountToConsume) {
+		mySkipCountRemaining -= theSkipCountToConsume;
+	}
+
+	@Nullable
+	@Override
+	public Integer getSkipCountRemaining() {
+		return mySkipCountRemaining;
+	}
+
+	@Override
+	public boolean excludeConcept(String theSystem, String theCode) {
 		String excludeSystem;
 		String excludeSystemVersion;
 		int versionSeparator = theSystem.indexOf("|");
-		if(versionSeparator > -1) {
+		if (versionSeparator > -1) {
 			excludeSystemVersion = theSystem.substring(versionSeparator + 1);
 			excludeSystem = theSystem.substring(0, versionSeparator);
 		} else {
@@ -107,22 +143,47 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
 			excludeSystemVersion = null;
 		}
 		if (excludeSystemVersion != null) {
-			this.getContains().removeIf(t ->
+			return this.getContains().removeIf(t ->
 				excludeSystem.equals(t.getSystem()) &&
-				theCode.equals(t.getCode()) &&
+					theCode.equals(t.getCode()) &&
-				excludeSystemVersion.equals(t.getVersion()));
+					excludeSystemVersion.equals(t.getVersion()));
 		} else {
-			this.getContains().removeIf(t ->
+			return this.getContains().removeIf(t ->
-					theSystem.equals(t.getSystem()) &&
+				theSystem.equals(t.getSystem()) &&
-						theCode.equals(t.getCode()));
+					theCode.equals(t.getCode()));
 		}
 	}
 
 	private void incrementConceptsCount() {
-		if (++myConceptsCount > myMaxCapacity) {
+		Integer capacityRemaining = getCapacityRemaining();
+		if (capacityRemaining == 0) {
 			String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myMaxCapacity);
 			throw new ExpansionTooCostlyException(msg);
 		}
+
+		if (myHardExpansionMaximumSize > 0 && myAddedConcepts > myHardExpansionMaximumSize) {
+			String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myHardExpansionMaximumSize);
+			throw new ExpansionTooCostlyException(msg);
+		}
+
+		myAddedConcepts++;
+	}
+
+	public Integer getTotalConcepts() {
+		return myTotalConcepts;
+	}
+
+	@Override
+	public void incrementOrDecrementTotalConcepts(boolean theAdd, int theDelta) {
+		int delta = theDelta;
+		if (!theAdd) {
+			delta = -delta;
+		}
+		if (myTotalConcepts == null) {
+			myTotalConcepts = delta;
+		} else {
+			myTotalConcepts = myTotalConcepts + delta;
+		}
 	}
 
 	private void setSystemAndVersion(String theSystemAndVersion, ValueSet.ValueSetExpansionContainsComponent myComponent) {
@@ -130,11 +191,18 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
 			int versionSeparator = theSystemAndVersion.lastIndexOf('|');
 			if (versionSeparator != -1) {
 				myComponent.setVersion(theSystemAndVersion.substring(versionSeparator + 1));
-				myComponent.setSystem(theSystemAndVersion.substring(0,versionSeparator));
+				myComponent.setSystem(theSystemAndVersion.substring(0, versionSeparator));
 			} else {
 				myComponent.setSystem(theSystemAndVersion);
 			}
 		}
 	}
 
+	public void setSkipCountRemaining(int theSkipCountRemaining) {
+		mySkipCountRemaining = theSkipCountRemaining;
+	}
+
+	public void setHardExpansionMaximumSize(int theHardExpansionMaximumSize) {
+		myHardExpansionMaximumSize = theHardExpansionMaximumSize;
+	}
 }

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java

@@ -32,7 +32,9 @@ import org.springframework.test.util.AopTestUtils;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.stream.Collectors;
 
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.fail;
@@ -100,7 +102,7 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test {
 
 
 	@Test
-	public void testValidateQuestionnaireResponseWithValueSetIncludingCompleteCodeSystem() throws IOException {
+	public void testExpandLocalCodeSystemWithExplicitCodes() throws IOException {
 		CodeSystem cs = loadResourceFromClasspath(CodeSystem.class, "/dstu3/iar/CodeSystem-iar-citizenship-status.xml");
 		myCodeSystemDao.create(cs);
 
@@ -110,18 +112,9 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test {
 		ValueSet expansion = myValueSetDao.expandByIdentifier("http://ccim.on.ca/fhir/iar/ValueSet/iar-citizenship-status", null);
 		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
 
-//		Questionnaire q = loadResourceFromClasspath(Questionnaire.class,"/dstu3/iar/Questionnaire-iar-test.xml" );
+		assertThat(expansion.getExpansion().getContains().stream().map(t->t.getCode()).collect(Collectors.toList()), containsInAnyOrder(
-//		myQuestionnaireDao.create(q);
+			"CDN", "PR", "TR", "REF", "UNK", "ASKU"
-//
+		));
-//
-//
-//		Bundle bundleForValidation = loadResourceFromClasspath(Bundle.class, "/dstu3/iar/Bundle-for-validation.xml");
-//		try {
-//			MethodOutcome outcome = myBundleDao.validate(bundleForValidation, null, null, null, null, null, null);
-//			ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome.getOperationOutcome()));
-//		} catch (PreconditionFailedException e) {
-//			ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome()));
-//		}
 	}
 
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java

@@ -1015,6 +1015,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
 		myAllergyIntoleranceDao.create(ai3, mySrd).getId().toUnqualifiedVersionless().getValue();
 
 		SearchParameterMap params;
+
 		params = new SearchParameterMap();
 		params.add(AllergyIntolerance.SP_CLINICAL_STATUS, new TokenParam(null, "active"));
 		assertThat(toUnqualifiedVersionlessIdValues(myAllergyIntoleranceDao.search(params)), containsInAnyOrder(id1));

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/JpaPackageCacheTest.java

@@ -1,7 +1,11 @@
 package ca.uhn.fhir.jpa.packages;
 
+import ca.uhn.fhir.jpa.dao.data.INpmPackageDao;
+import ca.uhn.fhir.jpa.dao.data.INpmPackageVersionDao;
 import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
+import ca.uhn.fhir.util.JsonUtil;
 import org.hl7.fhir.utilities.npm.IPackageCacheManager;
 import org.hl7.fhir.utilities.npm.NpmPackage;
 import org.junit.jupiter.api.Test;
@@ -10,13 +14,21 @@ import org.springframework.beans.factory.annotation.Autowired;
 import java.io.IOException;
 import java.io.InputStream;
 
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.fail;
 
 public class JpaPackageCacheTest extends BaseJpaR4Test {
 
 	@Autowired
-	private IPackageCacheManager myPackageCacheManager;
+	private IHapiPackageCacheManager myPackageCacheManager;
+
+	@Autowired
+	private INpmPackageDao myPackageDao;
+	@Autowired
+	private INpmPackageVersionDao myPackageVersionDao;
+
 
 	@Test
 	public void testSavePackage() throws IOException {
@@ -41,6 +53,23 @@ public class JpaPackageCacheTest extends BaseJpaR4Test {
 	}
 
 
+	@Test
+	public void testSavePackageWithLongDescription() throws IOException {
+		try (InputStream stream = IgInstallerDstu3Test.class.getResourceAsStream("/packages/package-davinci-cdex-0.2.0.tgz")) {
+			myPackageCacheManager.addPackageToCache("hl7.fhir.us.davinci-cdex", "0.2.0", stream, "hl7.fhir.us.davinci-cdex");
+		}
+
+		NpmPackage pkg;
+
+		pkg = myPackageCacheManager.loadPackage("hl7.fhir.us.davinci-cdex", null);
+		assertEquals("0.2.0", pkg.version());
+
+		assertEquals("This IG provides detailed guidance that helps implementers use FHIR-based interactions and resources relevant to support specific exchanges of clinical information between provider and payers (or ...", myPackageDao.findByPackageId("hl7.fhir.us.davinci-cdex").get().getDescription());
+		assertEquals("This IG provides detailed guidance that helps implementers use FHIR-based interactions and resources relevant to support specific exchanges of clinical information between provider and payers (or ...", myPackageVersionDao.findByPackageIdAndVersion("hl7.fhir.us.davinci-cdex", "0.2.0").get().getDescription());
+
+	}
+
+
 	@Test
 	public void testSavePackageCorrectFhirVersion() {
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java

@@ -168,7 +168,7 @@ public abstract class BaseResourceProviderR4Test extends BaseJpaR4Test {
 			confProvider.setSearchParamRegistry(ourSearchParamRegistry);
 
 			myFhirCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
-			myFhirCtx.getRestfulClientFactory().setSocketTimeout(5000000);
+			myFhirCtx.getRestfulClientFactory().setSocketTimeout(20000);
 
 			PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
 			HttpClientBuilder builder = HttpClientBuilder.create();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/PatientEverythingR4Test.java

@@ -9,6 +9,7 @@ import java.util.*;
 
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.parser.StrictErrorHandler;
+import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.EncodingEnum;
 import com.google.common.base.Charsets;
 import org.apache.commons.io.IOUtils;
@@ -23,6 +24,7 @@ import org.hl7.fhir.r4.model.Observation;
 import org.hl7.fhir.r4.model.Observation.ObservationStatus;
 import org.hl7.fhir.r4.model.Organization;
 import org.hl7.fhir.r4.model.Patient;
+import org.hl7.fhir.r4.model.Reference;
 import org.junit.jupiter.api.*;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -52,6 +54,7 @@ public class PatientEverythingR4Test extends BaseResourceProviderR4Test {
 		myDaoConfig.setReuseCachedSearchResultsForMillis(new DaoConfig().getReuseCachedSearchResultsForMillis());
 		myDaoConfig.setEverythingIncludesFetchPageSize(new DaoConfig().getEverythingIncludesFetchPageSize());
 		myDaoConfig.setSearchPreFetchThresholds(new DaoConfig().getSearchPreFetchThresholds());
+		myDaoConfig.setAllowExternalReferences(new DaoConfig().isAllowExternalReferences());
 	}
 
 	@Override
@@ -104,6 +107,44 @@ public class PatientEverythingR4Test extends BaseResourceProviderR4Test {
 
 	}
 
+	@Test
+	public void testEverythingWithCanonicalReferences() throws Exception {
+		myDaoConfig.setAllowExternalReferences(true);
+
+		Patient p = new Patient();
+		p.setManagingOrganization(new Reference("http://example.com/Organization/123"));
+		String patientId = myPatientDao.create(p).getId().toUnqualifiedVersionless().getValue();
+
+		Observation obs = new Observation();
+		obs.getSubject().setReference(patientId);
+		obs.getEncounter().setReference("http://example.com/Encounter/999");
+		String observationId = myObservationDao.create(obs).getId().toUnqualifiedVersionless().getValue();
+
+		// Normal call
+		Bundle bundle = fetchBundle(ourServerBase + "/" + patientId + "/$everything?_format=json&_count=100", EncodingEnum.JSON);
+		assertNull(bundle.getLink("next"));
+		Set<String> actual = new TreeSet<>();
+		for (BundleEntryComponent nextEntry : bundle.getEntry()) {
+			actual.add(nextEntry.getResource().getIdElement().toUnqualifiedVersionless().getValue());
+		}
+		assertThat(actual, containsInAnyOrder(patientId, observationId));
+
+		// Synchronous call
+		HttpGet get = new HttpGet(ourServerBase + "/" + patientId + "/$everything?_format=json&_count=100");
+		get.addHeader(Constants.HEADER_CACHE_CONTROL, Constants.CACHE_CONTROL_NO_CACHE);
+		try (CloseableHttpResponse resp = ourHttpClient.execute(get)) {
+			assertEquals(EncodingEnum.JSON.getResourceContentTypeNonLegacy(), resp.getFirstHeader(ca.uhn.fhir.rest.api.Constants.HEADER_CONTENT_TYPE).getValue().replaceAll(";.*", ""));
+			bundle = EncodingEnum.JSON.newParser(myFhirCtx).parseResource(Bundle.class, IOUtils.toString(resp.getEntity().getContent(), Charsets.UTF_8));
+		}
+		assertNull(bundle.getLink("next"));
+		actual = new TreeSet<>();
+		for (BundleEntryComponent nextEntry : bundle.getEntry()) {
+			actual.add(nextEntry.getResource().getIdElement().toUnqualifiedVersionless().getValue());
+		}
+		assertThat(actual, containsInAnyOrder(patientId, observationId));
+	}
+
+
 	/**
 	 * See #674
 	 */
@@ -114,7 +155,7 @@ public class PatientEverythingR4Test extends BaseResourceProviderR4Test {
 		
 		assertNull(bundle.getLink("next"));
 		
-		Set<String> actual = new TreeSet<String>();
+		Set<String> actual = new TreeSet<>();
 		for (BundleEntryComponent nextEntry : bundle.getEntry()) {
 			actual.add(nextEntry.getResource().getIdElement().toUnqualifiedVersionless().getValue());
 		}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ValueSetLuceneDisabledTest.java

@@ -0,0 +1,251 @@
+package ca.uhn.fhir.jpa.provider.r4;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet;
+import ca.uhn.fhir.jpa.config.TestR4WithLuceneDisabledConfig;
+import ca.uhn.fhir.jpa.dao.BaseJpaTest;
+import ca.uhn.fhir.jpa.dao.dstu2.FhirResourceDaoDstu2SearchNoFtTest;
+import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
+import ca.uhn.fhir.jpa.subscription.match.config.WebsocketDispatcherConfig;
+import ca.uhn.fhir.parser.IParser;
+import ca.uhn.fhir.parser.StrictErrorHandler;
+import ca.uhn.fhir.rest.api.EncodingEnum;
+import ca.uhn.fhir.rest.client.api.IGenericClient;
+import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum;
+import ca.uhn.fhir.rest.server.RestfulServer;
+import ca.uhn.fhir.rest.server.interceptor.CorsInterceptor;
+import ca.uhn.fhir.rest.server.provider.ResourceProviderFactory;
+import ca.uhn.fhir.test.utilities.JettyUtil;
+import org.apache.commons.io.IOUtils;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.r4.model.CodeSystem;
+import org.hl7.fhir.r4.model.CodeableConcept;
+import org.hl7.fhir.r4.model.Coding;
+import org.hl7.fhir.r4.model.Parameters;
+import org.hl7.fhir.r4.model.ValueSet;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.ApplicationContext;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallbackWithoutResult;
+import org.springframework.transaction.support.TransactionTemplate;
+import org.springframework.web.context.ContextLoader;
+import org.springframework.web.context.WebApplicationContext;
+import org.springframework.web.context.support.AnnotationConfigWebApplicationContext;
+import org.springframework.web.context.support.GenericWebApplicationContext;
+import org.springframework.web.cors.CorsConfiguration;
+import org.springframework.web.servlet.DispatcherServlet;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.jupiter.api.Assertions.fail;
+
+@ExtendWith(SpringExtension.class)
+@ContextConfiguration(classes = {TestR4WithLuceneDisabledConfig.class})
+public class ResourceProviderR4ValueSetLuceneDisabledTest  extends BaseJpaTest {
+
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceProviderR4ValueSetLuceneDisabledTest.class);
+
+	private static RestfulServer ourRestServer;
+	private static String ourServerBase;
+	private static Server ourServer;
+	private static DatabaseBackedPagingProvider ourPagingProvider;
+
+	@Autowired
+	private FhirContext myFhirCtx;
+	@Autowired
+	private PlatformTransactionManager myTxManager;
+	@Autowired
+	@Qualifier("myCodeSystemDaoR4")
+	private IFhirResourceDaoCodeSystem<CodeSystem, Coding, CodeableConcept> myCodeSystemDao;
+	@Autowired
+	@Qualifier("myValueSetDaoR4")
+	private IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> myValueSetDao;
+	@Autowired
+	@Qualifier("myResourceProvidersR4")
+	private ResourceProviderFactory myResourceProviders;
+	@Autowired
+	private ApplicationContext myAppCtx;
+
+
+	private IIdType myExtensionalCsId;
+	private IIdType myExtensionalVsId;
+	private IGenericClient myClient;
+
+	private void loadAndPersistCodeSystemAndValueSet() throws IOException {
+		loadAndPersistCodeSystem();
+		loadAndPersistValueSet();
+	}
+
+	private <T extends IBaseResource> T loadResourceFromClasspath(Class<T> type, String resourceName) throws IOException {
+		InputStream stream = FhirResourceDaoDstu2SearchNoFtTest.class.getResourceAsStream(resourceName);
+		if (stream == null) {
+			fail("Unable to load resource: " + resourceName);
+		}
+		String string = IOUtils.toString(stream, StandardCharsets.UTF_8);
+		IParser newJsonParser = EncodingEnum.detectEncodingNoDefault(string).newParser(myFhirCtx);
+		return newJsonParser.parseResource(type, string);
+	}
+
+	private void loadAndPersistCodeSystem() throws IOException {
+		CodeSystem codeSystem = loadResourceFromClasspath(CodeSystem.class, "/extensional-case-3-cs.xml");
+		codeSystem.setId("CodeSystem/cs");
+		persistCodeSystem(codeSystem);
+	}
+
+	private void persistCodeSystem(CodeSystem theCodeSystem) {
+		new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
+			@Override
+			protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theStatus) {
+				myExtensionalCsId = myCodeSystemDao.create(theCodeSystem, mySrd).getId().toUnqualifiedVersionless();
+			}
+		});
+		myCodeSystemDao.readEntity(myExtensionalCsId, null).getId();
+	}
+
+	private void loadAndPersistValueSet() throws IOException {
+		ValueSet valueSet = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml");
+		valueSet.setId("ValueSet/vs");
+		persistValueSet(valueSet);
+	}
+
+	private void persistValueSet(ValueSet theValueSet) {
+		new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
+					@Override
+					protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theStatus) {
+				myExtensionalVsId = myValueSetDao.create(theValueSet, mySrd).getId().toUnqualifiedVersionless();
+			}
+				});
+		myValueSetDao.readEntity(myExtensionalVsId, null).getId();
+	}
+
+	@Override
+	protected FhirContext getContext() {
+		return myFhirCtx;
+	}
+
+	@Override
+	protected PlatformTransactionManager getTxManager() {
+		return myTxManager;
+	}
+
+
+	@Test
+	public void testExpandById() throws Exception {
+		loadAndPersistCodeSystemAndValueSet();
+
+		Parameters respParam = myClient
+			.operation()
+			.onInstance(myExtensionalVsId)
+			.named("expand")
+			.withNoParameters(Parameters.class)
+			.execute();
+		ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource();
+
+		String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded);
+		ourLog.info(resp);
+		assertThat(resp, containsString("<ValueSet xmlns=\"http://hl7.org/fhir\">"));
+		assertThat(resp, containsString("<expansion>"));
+		assertThat(resp, containsString("<contains>"));
+		assertThat(resp, containsString("<system value=\"http://acme.org\"/>"));
+		assertThat(resp, containsString("<code value=\"8450-9\"/>"));
+		assertThat(resp, containsString("<display value=\"Systolic blood pressure--expiration\"/>"));
+		assertThat(resp, containsString("</contains>"));
+		assertThat(resp, containsString("<contains>"));
+		assertThat(resp, containsString("<system value=\"http://acme.org\"/>"));
+		assertThat(resp, containsString("<code value=\"11378-7\"/>"));
+		assertThat(resp, containsString("<display value=\"Systolic blood pressure at First encounter\"/>"));
+		assertThat(resp, containsString("</contains>"));
+		assertThat(resp, containsString("</expansion>"));
+
+	}
+
+	@BeforeEach
+	public void before() throws Exception {
+		myFhirCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
+		myFhirCtx.getRestfulClientFactory().setSocketTimeout(1200 * 1000);
+		myFhirCtx.setParserErrorHandler(new StrictErrorHandler());
+
+		if (ourServer == null) {
+			ourRestServer = new RestfulServer(myFhirCtx);
+			ourRestServer.registerProviders(myResourceProviders.createProviders());
+			ourRestServer.setDefaultResponseEncoding(EncodingEnum.XML);
+
+			ourPagingProvider = myAppCtx.getBean(DatabaseBackedPagingProvider.class);
+
+			Server server = new Server(0);
+
+			ServletContextHandler proxyHandler = new ServletContextHandler();
+			proxyHandler.setContextPath("/");
+
+			ServletHolder servletHolder = new ServletHolder();
+			servletHolder.setServlet(ourRestServer);
+			proxyHandler.addServlet(servletHolder, "/fhir/context/*");
+
+			GenericWebApplicationContext webApplicationContext = new GenericWebApplicationContext();
+			webApplicationContext.setParent(myAppCtx);
+			webApplicationContext.refresh();
+			proxyHandler.getServletContext().setAttribute(WebApplicationContext.ROOT_WEB_APPLICATION_CONTEXT_ATTRIBUTE, webApplicationContext);
+
+			DispatcherServlet dispatcherServlet = new DispatcherServlet();
+			// dispatcherServlet.setApplicationContext(webApplicationContext);
+			dispatcherServlet.setContextClass(AnnotationConfigWebApplicationContext.class);
+			ServletHolder subsServletHolder = new ServletHolder();
+			subsServletHolder.setServlet(dispatcherServlet);
+			subsServletHolder.setInitParameter(
+				ContextLoader.CONFIG_LOCATION_PARAM,
+				WebsocketDispatcherConfig.class.getName());
+			proxyHandler.addServlet(subsServletHolder, "/*");
+
+			// Register a CORS filter
+			CorsConfiguration config = new CorsConfiguration();
+			CorsInterceptor corsInterceptor = new CorsInterceptor(config);
+			config.addAllowedHeader("x-fhir-starter");
+			config.addAllowedHeader("Origin");
+			config.addAllowedHeader("Accept");
+			config.addAllowedHeader("X-Requested-With");
+			config.addAllowedHeader("Content-Type");
+			config.addAllowedHeader("Access-Control-Request-Method");
+			config.addAllowedHeader("Access-Control-Request-Headers");
+			config.addAllowedOrigin("*");
+			config.addExposedHeader("Location");
+			config.addExposedHeader("Content-Location");
+			config.setAllowedMethods(Arrays.asList("GET", "POST", "PUT", "DELETE", "OPTIONS"));
+			ourRestServer.registerInterceptor(corsInterceptor);
+
+			server.setHandler(proxyHandler);
+			JettyUtil.startServer(server);
+			int port = JettyUtil.getPortForStartedServer(server);
+			ourServerBase = "http://localhost:" + port + "/fhir/context";
+
+			myFhirCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
+			myFhirCtx.getRestfulClientFactory().setSocketTimeout(20000);
+
+			ourServer = server;
+		}
+
+		ourRestServer.setPagingProvider(ourPagingProvider);
+
+		myClient = myFhirCtx.newRestfulGenericClient(ourServerBase);
+	}
+
+
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4Test.java

@@ -10,8 +10,11 @@ import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
 import ca.uhn.fhir.jpa.entity.TermValueSetConceptDesignation;
 import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
+import ca.uhn.fhir.jpa.util.SqlQuery;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.util.HapiExtensions;
 import com.google.common.collect.Lists;
 import org.hl7.fhir.instance.model.api.IIdType;
@@ -31,9 +34,12 @@ import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -87,6 +93,260 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
 		assertEquals(24, expandedValueSet.getExpansion().getContains().size());
 	}
 
+
+	@Test
+	public void testExpandInline_IncludeCodeSystem_FilterOnDisplay_NoFilter() throws Exception {
+		loadAndPersistCodeSystemWithDesignations(HttpVerb.PUT);
+
+		ValueSet input = new ValueSet();
+		input.getCompose()
+			.addInclude()
+			.setSystem("http://acme.org");
+		ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
+		ourLog.info("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
+
+		assertEquals(24, expandedValueSet.getExpansion().getTotal());
+	}
+
+	@Test
+	public void testExpandInline_IncludeCodeSystem_FilterOnDisplay_ExactFilter() throws Exception {
+		loadAndPersistCodeSystemWithDesignations(HttpVerb.PUT);
+
+		ValueSet input = new ValueSet();
+		input.getCompose()
+			.addInclude()
+			.setSystem("http://acme.org")
+			.addFilter()
+			.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
+			.setOp(ValueSet.FilterOperator.EQUAL)
+			.setValue("Systolic blood pressure--inspiration");
+
+		ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
+		ourLog.info("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
+
+		assertEquals(1, expandedValueSet.getExpansion().getTotal());
+		assertThat(expandedValueSet.getExpansion().getContains().stream().map(t -> t.getDisplay()).collect(Collectors.toList()), containsInAnyOrder(
+			"Systolic blood pressure--inspiration"
+		));
+	}
+
+	@Test
+	public void testExpandInline_IncludeCodeSystem_FilterOnDisplay_LeftMatchFilter() throws Exception {
+		loadAndPersistCodeSystemWithDesignations(HttpVerb.PUT);
+
+		ValueSet input = new ValueSet();
+		input.getCompose()
+			.addInclude()
+			.setSystem("http://acme.org")
+			.addFilter()
+			.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
+			.setOp(ValueSet.FilterOperator.EQUAL)
+			.setValue("Systolic blood pressure 1");
+
+		ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
+		ourLog.info("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
+
+		assertEquals(3, expandedValueSet.getExpansion().getTotal());
+		assertThat(expandedValueSet.getExpansion().getContains().stream().map(t -> t.getDisplay()).collect(Collectors.toList()), containsInAnyOrder(
+			"Systolic blood pressure 1 hour minimum",
+			"Systolic blood pressure 1 hour mean",
+			"Systolic blood pressure 1 hour maximum"
+		));
+	}
+
+	@Test
+	public void testExpandInline_IncludePreExpandedValueSetByUri_FilterOnDisplay_LeftMatch_SelectAll() {
+		myDaoConfig.setPreExpandValueSets(true);
+		create100ConceptsCodeSystemAndValueSet();
+
+		ValueSet input = new ValueSet();
+		input.getCompose()
+			.addInclude()
+			.addValueSet("http://foo/vs")
+			.addFilter()
+			.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
+			.setOp(ValueSet.FilterOperator.EQUAL)
+			.setValue("display value 9");
+
+		myCaptureQueriesListener.clear();
+		ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
+		ourLog.debug("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
+
+		assertThat(toCodes(expandedValueSet).toString(), toCodes(expandedValueSet), contains(
+			"code9", "code90", "code91", "code92", "code93", "code94", "code95", "code96", "code97", "code98", "code99"
+		));
+		assertEquals(11, expandedValueSet.getExpansion().getContains().size(), toCodes(expandedValueSet).toString());
+		assertEquals(11, expandedValueSet.getExpansion().getTotal());
+
+		// Make sure we used the pre-expanded version
+		List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
+		String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
+		assertThat(lastSelectQuery, containsString("concept_display like 'display value 9%'"));
+
+	}
+
+	@Test
+	public void testExpandInline_IncludePreExpandedValueSetByUri_FilterOnDisplay_LeftMatch_SelectRange() {
+		myDaoConfig.setPreExpandValueSets(true);
+		create100ConceptsCodeSystemAndValueSet();
+
+		ValueSet input = new ValueSet();
+		input.getCompose()
+			.addInclude()
+			.addValueSet("http://foo/vs")
+			.addFilter()
+			.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
+			.setOp(ValueSet.FilterOperator.EQUAL)
+			.setValue("display value 9");
+
+		myCaptureQueriesListener.clear();
+		ValueSetExpansionOptions expansionOptions = new ValueSetExpansionOptions()
+			.setOffset(3)
+			.setCount(4);
+		ValueSet expandedValueSet = myTermSvc.expandValueSet(expansionOptions, input);
+		ourLog.debug("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
+
+		assertThat(toCodes(expandedValueSet).toString(), toCodes(expandedValueSet), contains(
+			"code92", "code93", "code94", "code95"
+		));
+		assertEquals(4, expandedValueSet.getExpansion().getContains().size(), toCodes(expandedValueSet).toString());
+		assertEquals(11, expandedValueSet.getExpansion().getTotal());
+
+		// Make sure we used the pre-expanded version
+		List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
+		String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
+		assertThat(lastSelectQuery, containsString("concept_display like 'display value 9%'"));
+
+	}
+
+
+	@Test
+	public void testExpandInline_IncludePreExpandedValueSetByUri_ExcludeCodes_FilterOnDisplay_LeftMatch_SelectAll() {
+		myDaoConfig.setPreExpandValueSets(true);
+		create100ConceptsCodeSystemAndValueSet();
+
+		ValueSet input = new ValueSet();
+		input.getCompose()
+			.addInclude()
+			.addValueSet("http://foo/vs")
+			.addFilter()
+			.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
+			.setOp(ValueSet.FilterOperator.EQUAL)
+			.setValue("display value 9");
+		input.getCompose()
+			.addExclude()
+			.addValueSet("http://foo/vs")
+			.addFilter()
+			.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
+			.setOp(ValueSet.FilterOperator.EQUAL)
+			.setValue("display value 90");
+
+		myCaptureQueriesListener.clear();
+		ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
+		ourLog.debug("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
+
+		assertThat(toCodes(expandedValueSet).toString(), toCodes(expandedValueSet), contains(
+			"code9", "code91", "code92", "code93", "code94", "code95", "code96", "code97", "code98", "code99"
+		));
+		assertEquals(10, expandedValueSet.getExpansion().getContains().size(), toCodes(expandedValueSet).toString());
+		assertEquals(10, expandedValueSet.getExpansion().getTotal());
+
+		// Make sure we used the pre-expanded version
+		List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
+		String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
+		assertThat(lastSelectQuery, containsString("concept_display like 'display value 90%'"));
+
+	}
+
+
+	@Test
+	public void testExpandInline_IncludePreExpandedValueSetByUri_ExcludeCodes_FilterOnDisplay_LeftMatch_SelectRange() {
+		myDaoConfig.setPreExpandValueSets(true);
+		create100ConceptsCodeSystemAndValueSet();
+
+		ValueSet input = new ValueSet();
+		input.getCompose()
+			.addInclude()
+			.addValueSet("http://foo/vs")
+			.addFilter()
+			.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
+			.setOp(ValueSet.FilterOperator.EQUAL)
+			.setValue("display value 9");
+		input.getCompose()
+			.addExclude()
+			.addValueSet("http://foo/vs")
+			.addFilter()
+			.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
+			.setOp(ValueSet.FilterOperator.EQUAL)
+			.setValue("display value 90");
+
+		myCaptureQueriesListener.clear();
+		ValueSetExpansionOptions options = new ValueSetExpansionOptions();
+		options.setOffset(3);
+		options.setCount(4);
+		try {
+			myTermSvc.expandValueSet(options, input);
+			fail();
+		} catch (InvalidRequestException e) {
+			assertEquals("ValueSet expansion can not combine \"offset\" with \"ValueSet.compose.exclude\" unless the ValueSet has been pre-expanded. ValueSet \"Unidentified ValueSet\" must be pre-expanded for this operation to work.", e.getMessage());
+		}
+	}
+
+
+	public void create100ConceptsCodeSystemAndValueSet() {
+		CodeSystem cs = new CodeSystem();
+		cs.setUrl("http://foo/cs");
+		cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
+		myCodeSystemDao.create(cs);
+
+		CustomTerminologySet additions = new CustomTerminologySet();
+		for (int i = 0; i < 100; i++) {
+			additions.addRootConcept("code" + i, "display value " + i);
+		}
+		myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", additions);
+
+		ValueSet vs = new ValueSet();
+		vs.setUrl("http://foo/vs");
+		vs.getCompose().addInclude().setSystem("http://foo/cs");
+		myValueSetDao.create(vs);
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+	}
+
+	@Test
+	public void testExpandInline_IncludeNonPreExpandedValueSetByUri_FilterOnDisplay_LeftMatch() {
+		myDaoConfig.setPreExpandValueSets(true);
+		create100ConceptsCodeSystemAndValueSet();
+
+		ValueSet input = new ValueSet();
+		input.getCompose()
+			.addInclude()
+			.addValueSet("http://foo/vs")
+			.addFilter()
+			.setProperty(JpaConstants.VALUESET_FILTER_DISPLAY)
+			.setOp(ValueSet.FilterOperator.EQUAL)
+			.setValue("display value 9");
+
+		myCaptureQueriesListener.clear();
+		ValueSet expandedValueSet = myTermSvc.expandValueSet(new ValueSetExpansionOptions(), input);
+		ourLog.debug("Expanded ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expandedValueSet));
+
+		assertThat(toCodes(expandedValueSet).toString(), toCodes(expandedValueSet), contains(
+			"code9", "code90", "code91", "code92", "code93", "code94", "code95", "code96", "code97", "code98", "code99"
+		));
+		assertEquals(11, expandedValueSet.getExpansion().getContains().size(), toCodes(expandedValueSet).toString());
+		assertEquals(11, expandedValueSet.getExpansion().getTotal(), toCodes(expandedValueSet).toString());
+
+		// Make sure we used the pre-expanded version
+		List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
+		String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
+		assertThat(lastSelectQuery, containsString("concept_display like 'display value 9%'"));
+	}
+
+	@Nonnull
+	public List<String> toCodes(ValueSet theExpandedValueSet) {
+		return theExpandedValueSet.getExpansion().getContains().stream().map(t -> t.getCode()).collect(Collectors.toList());
+	}
+
 	@SuppressWarnings("SpellCheckingInspection")
 	@Test
 	public void testExpandTermValueSetAndChildren() throws Exception {
@@ -115,11 +375,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
 
 		assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getTotal());
 		assertEquals(myDaoConfig.getPreExpandValueSetsDefaultOffset(), expandedValueSet.getExpansion().getOffset());
-		assertEquals(2, expandedValueSet.getExpansion().getParameter().size());
+		assertEquals(0, expandedValueSet.getExpansion().getParameter().size());
-		assertEquals("offset", expandedValueSet.getExpansion().getParameter().get(0).getName());
-		assertEquals(0, expandedValueSet.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue());
-		assertEquals("count", expandedValueSet.getExpansion().getParameter().get(1).getName());
-		assertEquals(1000, expandedValueSet.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue());
 
 		assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getContains().size());
 
@@ -337,11 +593,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
 
 		assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getTotal());
 		assertEquals(myDaoConfig.getPreExpandValueSetsDefaultOffset(), expandedValueSet.getExpansion().getOffset());
-		assertEquals(2, expandedValueSet.getExpansion().getParameter().size());
+		assertEquals(0, expandedValueSet.getExpansion().getParameter().size());
-		assertEquals("offset", expandedValueSet.getExpansion().getParameter().get(0).getName());
-		assertEquals(0, expandedValueSet.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue());
-		assertEquals("count", expandedValueSet.getExpansion().getParameter().get(1).getName());
-		assertEquals(1000, expandedValueSet.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue());
 
 		assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getContains().size());
 
@@ -796,7 +1048,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
 		String encoded = myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome);
 		ourLog.info(encoded);
 
-		Extension extensionByUrl = outcome.getExpansion().getExtensionByUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE);
+		Extension extensionByUrl = outcome.getMeta().getExtensionByUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE);
 		assertEquals("Unknown CodeSystem URI \"http://unknown-system\" referenced from ValueSet", extensionByUrl.getValueAsPrimitive().getValueAsString());
 	}
 
@@ -877,8 +1129,8 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
 		ValueSet.ConceptSetComponent include = vs.getCompose().addInclude();
 		include.setSystem(CS_URL);
 		try {
-			myTermSvc.expandValueSet(null, vs);
+			ValueSet expansion = myTermSvc.expandValueSet(null, vs);
-			fail();
+			fail("Expanded to " + expansion.getExpansion().getContains().size() + " but max was " + myDaoConfig.getMaximumExpansionSize());
 		} catch (InternalErrorException e) {
 			assertEquals("Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage());
 		}

hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java

@@ -134,7 +134,7 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 		init510_20200706_to_20200714();
 
 		Builder.BuilderWithTableName empiLink = version.onTable("MPI_LINK");
-		empiLink.addColumn("20200715.1", "VERSION").nonNullable().type(ColumnTypeEnum.STRING, EmpiLink.VERSION_LENGTH);
+		empiLink.addColumn("20200715.1", "VERSION").nonNullable().type(ColumnTypeEnum.STRING, 16);
 		empiLink.addColumn("20200715.2", "EID_MATCH").nullable().type(ColumnTypeEnum.BOOLEAN);
 		empiLink.addColumn("20200715.3", "NEW_PERSON").nullable().type(ColumnTypeEnum.BOOLEAN);
 		empiLink.addColumn("20200715.4", "VECTOR").nullable().type(ColumnTypeEnum.LONG);
@@ -143,11 +143,11 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 		init510_20200725();
 
 		//EMPI Target Type
-		empiLink.addColumn("20200727.1","TARGET_TYPE").nullable().type(ColumnTypeEnum.STRING, EmpiLink.TARGET_TYPE_LENGTH);
+		empiLink.addColumn("20200727.1","TARGET_TYPE").nullable().type(ColumnTypeEnum.STRING, 40);
 
 		//ConceptMap add version for search
 		Builder.BuilderWithTableName trmConceptMap = version.onTable("TRM_CONCEPT_MAP");
-		trmConceptMap.addColumn("20200910.1", "VER").nullable().type(ColumnTypeEnum.STRING, TermConceptMap.MAX_VER_LENGTH);
+		trmConceptMap.addColumn("20200910.1", "VER").nullable().type(ColumnTypeEnum.STRING, 200);
 		trmConceptMap.dropIndex("20200910.2", "IDX_CONCEPT_MAP_URL");
 		trmConceptMap.addIndex("20200910.3", "IDX_CONCEPT_MAP_URL").unique(true).withColumns("URL", "VER");
 
@@ -155,15 +155,15 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 		Builder.BuilderWithTableName trmCodeSystemVer = version.onTable("TRM_CODESYSTEM_VER");
 		trmCodeSystemVer.addIndex("20200923.1", "IDX_CODESYSTEM_AND_VER").unique(true).withColumns("CODESYSTEM_PID", "CS_VERSION_ID");
 		Builder.BuilderWithTableName trmValueSet = version.onTable("TRM_VALUESET");
-		trmValueSet.addColumn("20200923.2", "VER").nullable().type(ColumnTypeEnum.STRING, TermValueSet.MAX_VER_LENGTH);
+		trmValueSet.addColumn("20200923.2", "VER").nullable().type(ColumnTypeEnum.STRING, 200);
 		trmValueSet.dropIndex("20200923.3", "IDX_VALUESET_URL");
 		trmValueSet.addIndex("20200923.4", "IDX_VALUESET_URL").unique(true).withColumns("URL", "VER");
 
 		//Term ValueSet Component add system version
 		Builder.BuilderWithTableName trmValueSetComp = version.onTable("TRM_VALUESET_CONCEPT");
-		trmValueSetComp.addColumn("20201028.1", "SYSTEM_VER").nullable().type(ColumnTypeEnum.STRING, TermCodeSystemVersion.MAX_VERSION_LENGTH);
+		trmValueSetComp.addColumn("20201028.1", "SYSTEM_VER").nullable().type(ColumnTypeEnum.STRING, 200);
 		trmValueSetComp.dropIndex("20201028.2", "IDX_VS_CONCEPT_CS_CD");
-		trmValueSetComp.addIndex("20201028.3", "IDX_VS_CONCEPT_CS_CD").unique(true).withColumns("VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL");
+		trmValueSetComp.addIndex("20201028.3", "IDX_VS_CONCEPT_CS_CODE").unique(true).withColumns("VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL");
 	}
 
 	protected void init510_20200725() {

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/NpmPackageVersionEntity.java

@@ -54,6 +54,7 @@ import java.util.List;
 public class NpmPackageVersionEntity {
 
 	public static final int VERSION_ID_LENGTH = 200;
+	public static final int PACKAGE_DESC_LENGTH = 200;
 	public static final int FHIR_VERSION_LENGTH = 10;
 
 	@SequenceGenerator(name = "SEQ_NPM_PACKVER", sequenceName = "SEQ_NPM_PACKVER")
@@ -74,9 +75,9 @@ public class NpmPackageVersionEntity {
 	@Temporal(TemporalType.TIMESTAMP)
 	@Column(name = "SAVED_TIME", nullable = false)
 	private Date mySavedTime;
-	@Column(name = "PKG_DESC", nullable = true, length = 200)
+	@Column(name = "PKG_DESC", nullable = true, length = PACKAGE_DESC_LENGTH)
 	private String myDescription;
-	@Column(name = "DESC_UPPER", nullable = true, length = 200)
+	@Column(name = "DESC_UPPER", nullable = true, length = PACKAGE_DESC_LENGTH)
 	private String myDescriptionUpper;
 	@Column(name = "CURRENT_VERSION", nullable = false)
 	private boolean myCurrentVersion;

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java

@@ -202,6 +202,7 @@ public class JpaConstants {
 	 * URL for extension on a Phonetic String SearchParameter indicating that text values should be phonetically indexed with the named encoder
 	 */
 	public static final String EXT_SEARCHPARAM_PHONETIC_ENCODER = "http://hapifhir.io/fhir/StructureDefinition/searchparameter-phonetic-encoder";
+	public static final String VALUESET_FILTER_DISPLAY = "display";
 
 	/**
 	 * Non-instantiable