Merge remote-tracking branch 'remotes/origin/master' into ja_20190822_1440_infinispan_query_cache

# Conflicts:
#	hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java
#	src/changes/changes.xml

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/MetaUtil.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.util;
 
+/*-
+ * #%L
+ * HAPI FHIR - Core Library
+ * %%
+ * Copyright (C) 2014 - 2019 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
 import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
 import ca.uhn.fhir.context.FhirContext;

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/UrlUtil.java

@@ -304,13 +304,23 @@ public class UrlUtil {
 			retVal.setVersionId(id.getVersionIdPart());
 			return retVal;
 		}
+
+		int parsingStart = 0;
+		if (url.length() > 2) {
+			if (url.charAt(0) == '/') {
+				if (Character.isLetter(url.charAt(1))) {
+					parsingStart = 1;
+				}
+			}
+		}
+
 		if (url.matches("/[a-zA-Z]+\\?.*")) {
 			url = url.substring(1);
 		}
 		int nextStart = 0;
 		boolean nextIsHistory = false;
 
-		for (int idx = 0; idx < url.length(); idx++) {
+		for (int idx = parsingStart; idx < url.length(); idx++) {
 			char nextChar = url.charAt(idx);
 			boolean atEnd = (idx + 1) == url.length();
 			if (nextChar == '?' || nextChar == '/' || atEnd) {

hapi-fhir-base/src/main/resources/ca/uhn/fhir/i18n/hapi-messages.properties

@@ -127,11 +127,5 @@ ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl.cannotCreateDuplicateCodeSystemU
 ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl.cannotCreateDuplicateConceptMapUrl=Can not create multiple ConceptMap resources with ConceptMap.url "{0}", already have one with resource ID: {1}
 ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl.cannotCreateDuplicateValueSetUrl=Can not create multiple ValueSet resources with ValueSet.url "{0}", already have one with resource ID: {1}
 ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl.expansionTooLarge=Expansion of ValueSet produced too many codes (maximum {0}) - Operation aborted!
-ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl.valueSetNotReadyForExpand=ValueSet is not ready for operation $expand; current status: {0} | {1}
 
 ca.uhn.fhir.jpa.util.jsonpatch.JsonPatchUtils.failedToApplyPatch=Failed to apply JSON patch to {0}: {1}
-
-ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum.notExpanded=The ValueSet is waiting to be picked up and pre-expanded by a scheduled task.
-ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum.expansionInProgress=The ValueSet has been picked up by a scheduled task and pre-expansion is in progress.
-ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum.expanded=The ValueSet has been picked up by a scheduled task and pre-expansion is complete.
-ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum.failedToExpand=The ValueSet has been picked up by a scheduled task and pre-expansion has failed.

hapi-fhir-base/src/test/java/ca/uhn/fhir/util/UrlUtilTest.java

@@ -56,6 +56,8 @@ public class UrlUtilTest {
 		assertEquals("a=b", UrlUtil.parseUrl("http://hl7.org/fhir/ConceptMap/ussgfht-loincde?a=b").getParams());
 
 		assertEquals("a=b", UrlUtil.parseUrl("ConceptMap/ussgfht-loincde?a=b").getParams());
+		assertEquals("a=b", UrlUtil.parseUrl("/ConceptMap?a=b").getParams());
+		assertEquals("a=b", UrlUtil.parseUrl("/ConceptMap/ussgfht-loincde?a=b").getParams());
 
 	}
 

hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/ToggleSearchParametersCommand.java

@@ -0,0 +1,57 @@
+package ca.uhn.fhir.cli;
+
+/*-
+ * #%L
+ * HAPI FHIR - Command Line Client - API
+ * %%
+ * Copyright (C) 2014 - 2019 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import ca.uhn.fhir.jpa.term.IHapiTerminologyLoaderSvc;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.util.concurrent.ExecutionException;
+
+public class ToggleSearchParametersCommand extends BaseCommand {
+
+	@Override
+	public String getCommandDescription() {
+		return null;
+	}
+
+	@Override
+	public String getCommandName() {
+		return null;
+	}
+
+	@Override
+	public Options getOptions() {
+		Options options = new Options();
+		addFhirVersionOption(options);
+		addBaseUrlOption(options);
+		addRequiredOption(options, "u", "url", true, "The code system URL associated with this upload (e.g. " + IHapiTerminologyLoaderSvc.SCT_URI + ")");
+		addBasicAuthOption(options);
+		return options;
+	}
+
+	@Override
+	public void run(CommandLine theCommandLine) throws ParseException, ExecutionException {
+
+	}
+
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java

@@ -16,6 +16,7 @@ import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
 import ca.uhn.fhir.jpa.model.entity.*;
 import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
 import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc;
 import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider;
 import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc;
@@ -27,7 +28,6 @@ import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
 import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
 import ca.uhn.fhir.jpa.util.AddRemoveCount;
-import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster;
 import ca.uhn.fhir.model.api.IResource;
 import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
@@ -37,7 +37,6 @@ import ca.uhn.fhir.model.base.composite.BaseCodingDt;
 import ca.uhn.fhir.model.base.composite.BaseResourceReferenceDt;
 import ca.uhn.fhir.model.primitive.IdDt;
 import ca.uhn.fhir.model.primitive.InstantDt;
-import ca.uhn.fhir.model.primitive.StringDt;
 import ca.uhn.fhir.model.valueset.BundleEntryTransactionMethodEnum;
 import ca.uhn.fhir.parser.DataFormatException;
 import ca.uhn.fhir.parser.IParser;
@@ -149,8 +148,13 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 	@Autowired
 	protected ISearchParamRegistry mySearchParamRegistry;
 	@Autowired
+	protected DeleteConflictService myDeleteConflictService;
+	@Autowired
+	protected IInterceptorBroadcaster myInterceptorBroadcaster;
+	@Autowired
+	ExpungeService myExpungeService;
+	@Autowired
 	private DaoConfig myConfig;
-
 	@Autowired
 	private PlatformTransactionManager myPlatformTransactionManager;
 	@Autowired
@@ -167,20 +171,8 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 	private DaoSearchParamSynchronizer myDaoSearchParamSynchronizer;
 	@Autowired
 	private SearchBuilderFactory mySearchBuilderFactory;
-	@Autowired
-	ExpungeService myExpungeService;
-	@Autowired
-	protected DeleteConflictService myDeleteConflictService;
-
 	private FhirContext myContext;
 	private ApplicationContext myApplicationContext;
-	@Autowired
-	protected IInterceptorBroadcaster myInterceptorBroadcaster;
-
-	@Autowired
-	public void setContext(FhirContext theContext) {
-		myContext = theContext;
-	}
 
 	@Override
 	public void setApplicationContext(ApplicationContext theApplicationContext) throws BeansException {
@@ -348,6 +340,11 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 		return myContext;
 	}
 
+	@Autowired
+	public void setContext(FhirContext theContext) {
+		myContext = theContext;
+	}
+
 	public FhirContext getContext(FhirVersionEnum theVersion) {
 		Validate.notNull(theVersion, "theVersion must not be null");
 		synchronized (ourRetrievalContexts) {
@@ -879,7 +876,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 			ResourceTable resource = (ResourceTable) theEntity;
 			version = theEntity.getVersion();
 			ResourceHistoryTable history = myResourceHistoryTableDao.findForIdAndVersionAndFetchProvenance(theEntity.getId(), version);
-			((ResourceTable)theEntity).setCurrentVersionEntity(history);
+			((ResourceTable) theEntity).setCurrentVersionEntity(history);
 
 			while (history == null) {
 				if (version > 1L) {
@@ -1457,6 +1454,26 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 		return mySearchParamRegistry;
 	}
 
+	@SuppressWarnings("unchecked")
+	public static String parseContentTextIntoWords(FhirContext theContext, IBaseResource theResource) {
+
+		Class<IPrimitiveType<String>> stringType = (Class<IPrimitiveType<String>>) theContext.getElementDefinition("string").getImplementingClass();
+
+		StringBuilder retVal = new StringBuilder();
+		List<IPrimitiveType<String>> childElements = theContext.newTerser().getAllPopulatedChildElementsOfType(theResource, stringType);
+		for (@SuppressWarnings("rawtypes")
+			IPrimitiveType<String> nextType : childElements) {
+			if (stringType.equals(nextType.getClass())) {
+				String nextValue = nextType.getValueAsString();
+				if (isNotBlank(nextValue)) {
+					retVal.append(nextValue.replace("\n", " ").replace("\r", " "));
+					retVal.append("\n");
+				}
+			}
+		}
+		return retVal.toString();
+	}
+
 	public static void clearRequestAsProcessingSubRequest(ServletRequestDetails theRequestDetails) {
 		if (theRequestDetails != null) {
 			theRequestDetails.getUserData().remove(PROCESSING_SUB_REQUEST);
@@ -1469,23 +1486,6 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 		}
 	}
 
-	public static String parseContentTextIntoWords(FhirContext theContext, IBaseResource theResource) {
-		StringBuilder retVal = new StringBuilder();
-		@SuppressWarnings("rawtypes")
-		List<IPrimitiveType> childElements = theContext.newTerser().getAllPopulatedChildElementsOfType(theResource, IPrimitiveType.class);
-		for (@SuppressWarnings("rawtypes")
-			IPrimitiveType nextType : childElements) {
-			if (nextType instanceof StringDt || nextType.getClass().getSimpleName().equals("StringType")) {
-				String nextValue = nextType.getValueAsString();
-				if (isNotBlank(nextValue)) {
-					retVal.append(nextValue.replace("\n", " ").replace("\r", " "));
-					retVal.append("\n");
-				}
-			}
-		}
-		return retVal.toString();
-	}
-
 	public static void populateFullTextFields(final FhirContext theContext, final IBaseResource theResource, ResourceTable theEntity) {
 		if (theEntity.getDeleted() != null) {
 			theEntity.setNarrativeTextParsedIntoWords(null);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java

@@ -151,7 +151,7 @@ public class DaoConfig {
 	 */
 	private boolean myPreExpandValueSetsExperimental = false;
 	private boolean myFilterParameterEnabled = false;
-	private StoreMetaSourceInformation myStoreMetaSourceInformation = StoreMetaSourceInformation.SOURCE_URI_AND_REQUEST_ID;
+	private StoreMetaSourceInformationEnum myStoreMetaSourceInformation = StoreMetaSourceInformationEnum.SOURCE_URI_AND_REQUEST_ID;
 	/**
 	 * EXPERIMENTAL - Do not use in production! Do not change default of {@code 0}!
 	 */
@@ -1676,10 +1676,10 @@ public class DaoConfig {
 	 * each resource. This adds extra table and index space so it should be disabled if it is not being
 	 * used.
 	 * <p>
-	 * Default is {@link StoreMetaSourceInformation#SOURCE_URI_AND_REQUEST_ID}
+	 * Default is {@link StoreMetaSourceInformationEnum#SOURCE_URI_AND_REQUEST_ID}
 	 * </p>
 	 */
-	public StoreMetaSourceInformation getStoreMetaSourceInformation() {
+	public StoreMetaSourceInformationEnum getStoreMetaSourceInformation() {
 		return myStoreMetaSourceInformation;
 	}
 
@@ -1688,15 +1688,15 @@ public class DaoConfig {
 	 * each resource. This adds extra table and index space so it should be disabled if it is not being
 	 * used.
 	 * <p>
-	 * Default is {@link StoreMetaSourceInformation#SOURCE_URI_AND_REQUEST_ID}
+	 * Default is {@link StoreMetaSourceInformationEnum#SOURCE_URI_AND_REQUEST_ID}
 	 * </p>
 	 */
-	public void setStoreMetaSourceInformation(StoreMetaSourceInformation theStoreMetaSourceInformation) {
+	public void setStoreMetaSourceInformation(StoreMetaSourceInformationEnum theStoreMetaSourceInformation) {
 		Validate.notNull(theStoreMetaSourceInformation, "theStoreMetaSourceInformation must not be null");
 		myStoreMetaSourceInformation = theStoreMetaSourceInformation;
 	}
 
-	public enum StoreMetaSourceInformation {
+	public enum StoreMetaSourceInformationEnum {
 		NONE(false, false),
 		SOURCE_URI(true, false),
 		REQUEST_ID(false, true),
@@ -1705,7 +1705,7 @@ public class DaoConfig {
 		private final boolean myStoreSourceUri;
 		private final boolean myStoreRequestId;
 
-		StoreMetaSourceInformation(boolean theStoreSourceUri, boolean theStoreRequestId) {
+		StoreMetaSourceInformationEnum(boolean theStoreSourceUri, boolean theStoreRequestId) {
 			myStoreSourceUri = theStoreSourceUri;
 			myStoreRequestId = theStoreRequestId;
 		}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoRegistry.java

@@ -49,9 +49,9 @@ public class DaoRegistry implements ApplicationContextAware, IDaoRegistry {
 		super();
 	}
 
-
 	private volatile Map<String, IFhirResourceDao<?>> myResourceNameToResourceDao;
 	private volatile IFhirSystemDao<?, ?> mySystemDao;
+
 	private Set<String> mySupportedResourceTypes;
 
 	public void setSupportedResourceTypes(Collection<String> theSupportedResourceTypes) {
@@ -156,7 +156,9 @@ public class DaoRegistry implements ApplicationContextAware, IDaoRegistry {
 		myResourceNameToResourceDao = new HashMap<>();
 
 		for (IFhirResourceDao nextResourceDao : theResourceDaos) {
-			RuntimeResourceDefinition nextResourceDef = myContext.getResourceDefinition(nextResourceDao.getResourceType());
+			Class resourceType = nextResourceDao.getResourceType();
+			assert resourceType != null;
+			RuntimeResourceDefinition nextResourceDef = myContext.getResourceDefinition(resourceType);
 			if (mySupportedResourceTypes == null || mySupportedResourceTypes.contains(nextResourceDef.getName())) {
 				myResourceNameToResourceDao.put(nextResourceDef.getName(), nextResourceDao);
 			}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IDao.java

@@ -30,6 +30,11 @@ import java.util.Collection;
  * #L%
  */
 
+/**
+ * Note that this interface is not considered a stable interface. While it is possible to build applications
+ * that use it directly, please be aware that we may modify methods, add methods, or even remove methods from
+ * time to time, even within minor point releases.
+ */
 public interface IDao {
 
 	MetadataKeyResourcePid RESOURCE_PID = new MetadataKeyResourcePid("RESOURCE_PID");

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFhirResourceDao.java

@@ -44,8 +44,16 @@ import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
 
 import javax.servlet.http.HttpServletResponse;
-import java.util.*;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
+/**
+ * Note that this interface is not considered a stable interface. While it is possible to build applications
+ * that use it directly, please be aware that we may modify methods, add methods, or even remove methods from
+ * time to time, even within minor point releases.
+ */
 public interface IFhirResourceDao<T extends IBaseResource> extends IDao {
 
 	void addTag(IIdType theId, TagTypeEnum theTagType, String theScheme, String theTerm, String theLabel, RequestDetails theRequest);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFhirSystemDao.java

@@ -32,6 +32,10 @@ import java.util.Date;
 import java.util.Map;
 
 /**
+ * Note that this interface is not considered a stable interface. While it is possible to build applications
+ * that use it directly, please be aware that we may modify methods, add methods, or even remove methods from
+ * time to time, even within minor point releases.
+ *
  * @param <T>  The bundle type
  * @param <MT> The Meta datatype type
  */

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java

@@ -825,7 +825,7 @@ public class SearchBuilder implements ISearchBuilder {
 
 
 	private Predicate addPredicateSource(List<? extends IQueryParameterType> theList, SearchFilterParser.CompareOperation theOperation, RequestDetails theRequest) {
-		if (myDaoConfig.getStoreMetaSourceInformation() == DaoConfig.StoreMetaSourceInformation.NONE) {
+		if (myDaoConfig.getStoreMetaSourceInformation() == DaoConfig.StoreMetaSourceInformationEnum.NONE) {
 			String msg = myContext.getLocalizer().getMessage(SearchBuilder.class, "sourceParamDisabled");
 			throw new InvalidRequestException(msg);
 		}
@@ -2700,8 +2700,7 @@ public class SearchBuilder implements ISearchBuilder {
 	private Predicate processFilterParameter(SearchFilterParser.FilterParameter theFilter,
 														  String theResourceName, RequestDetails theRequest) {
 
-		RuntimeSearchParam searchParam = mySearchParamRegistry.getActiveSearchParam(theResourceName,
+		RuntimeSearchParam searchParam = mySearchParamRegistry.getActiveSearchParam(theResourceName,	theFilter.getParamPath().getName());
-			theFilter.getParamPath().getName());
 
 		if (searchParam.getName().equals(IAnyResource.SP_RES_ID)) {
 			if (searchParam.getParamType() == RestSearchParameterTypeEnum.TOKEN) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptDao.java

@@ -40,15 +40,15 @@ public interface ITermValueSetConceptDao extends JpaRepository<TermValueSetConce
 	@Modifying
 	void deleteByTermValueSetId(@Param("pid") Long theValueSetId);
 
-	@Query("SELECT vsc from TermValueSetConcept vsc WHERE vsc.myValueSet.myId = :pid")
+	@Query("SELECT vsc from TermValueSetConcept vsc LEFT OUTER JOIN FETCH vsc.myDesignations WHERE vsc.myValueSet.myId = :pid")
-	Slice<TermValueSetConcept> findByTermValueSetId(Pageable thePage, @Param("pid") Long theValueSetId);
+	Slice<TermValueSetConcept> findByTermValueSetIdAndPreFetchDesignations(Pageable thePage, @Param("pid") Long theValueSetId);
 
 	@Query("SELECT vsc FROM TermValueSetConcept vsc WHERE vsc.myValueSet.myId = :pid AND vsc.mySystem = :system_url AND vsc.myCode = :codeval")
 	Optional<TermValueSetConcept> findByTermValueSetIdSystemAndCode(@Param("pid") Long theValueSetId, @Param("system_url") String theSystem, @Param("codeval") String theCode);
 
 	@Query("SELECT vsc FROM TermValueSetConcept vsc WHERE vsc.myValueSet.myResourcePid = :resource_pid AND vsc.myCode = :codeval")
-	List<TermValueSetConcept> findOneByValueSetIdAndCode(@Param("resource_pid") Long theValueSetId, @Param("codeval") String theCode);
+	List<TermValueSetConcept> findByValueSetResourcePidAndCode(@Param("resource_pid") Long theValueSetId, @Param("codeval") String theCode);
 
 	@Query("SELECT vsc FROM TermValueSetConcept vsc WHERE vsc.myValueSet.myResourcePid = :resource_pid AND vsc.mySystem = :system_url AND vsc.myCode = :codeval")
-	List<TermValueSetConcept> findOneByValueSetIdSystemAndCode(@Param("resource_pid") Long theValueSetId, @Param("system_url") String theSystem, @Param("codeval") String theCode);
+	Optional<TermValueSetConcept> findByValueSetResourcePidSystemAndCode(@Param("resource_pid") Long theValueSetId, @Param("system_url") String theSystem, @Param("codeval") String theCode);
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java

@@ -331,7 +331,7 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
 
 		if (vs != null) {
 			ValidateCodeResult result;
-			if (myDaoConfig.isPreExpandValueSetsExperimental()) {
+			if (myDaoConfig.isPreExpandValueSetsExperimental() && myTerminologySvc.isValueSetPreExpandedForCodeValidation(vs)) {
 				result = myTerminologySvc.validateCodeIsInPreExpandedValueSet(vs, toStringOrNull(theSystem), toStringOrNull(theCode), toStringOrNull(theDisplay), theCoding, theCodeableConcept);
 			} else {
 				ValueSet expansion = doExpand(vs);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java

@@ -327,7 +327,7 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4<ValueSet> imple
 
 		if (vs != null) {
 			ValidateCodeResult result;
-			if (myDaoConfig.isPreExpandValueSetsExperimental()) {
+			if (myDaoConfig.isPreExpandValueSetsExperimental() && myTerminologySvc.isValueSetPreExpandedForCodeValidation(vs)) {
 				result = myTerminologySvc.validateCodeIsInPreExpandedValueSet(vs, toStringOrNull(theSystem), toStringOrNull(theCode), toStringOrNull(theDisplay), theCoding, theCodeableConcept);
 			} else {
 				ValueSet expansion = doExpand(vs);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoValueSetR5.java

@@ -333,7 +333,7 @@ public class FhirResourceDaoValueSetR5 extends FhirResourceDaoR5<ValueSet> imple
 
 		if (vs != null) {
 			ValidateCodeResult result;
-			if (myDaoConfig.isPreExpandValueSetsExperimental()) {
+			if (myDaoConfig.isPreExpandValueSetsExperimental() && myTerminologySvc.isValueSetPreExpandedForCodeValidation(vs)) {
 				result = myTerminologySvc.validateCodeIsInPreExpandedValueSet(vs, toStringOrNull(theSystem), toStringOrNull(theCode), toStringOrNull(theDisplay), theCoding, theCodeableConcept);
 			} else {
 				ValueSet expansion = doExpand(vs);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java

@@ -26,6 +26,7 @@ import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
+import org.hibernate.annotations.ColumnDefault;
 
 import javax.annotation.Nonnull;
 import javax.persistence.*;
@@ -69,6 +70,14 @@ public class TermValueSet implements Serializable {
 	@OneToMany(mappedBy = "myValueSet")
 	private List<TermValueSetConcept> myConcepts;
 
+	@Column(name = "TOTAL_CONCEPTS", nullable = false)
+	@ColumnDefault("0")
+	private Long myTotalConcepts;
+
+	@Column(name = "TOTAL_CONCEPT_DESIGNATIONS", nullable = false)
+	@ColumnDefault("0")
+	private Long myTotalConceptDesignations;
+
 	@Enumerated(EnumType.STRING)
 	@Column(name = "EXPANSION_STATUS", nullable = false, length = MAX_EXPANSION_STATUS_LENGTH)
 	private TermValueSetPreExpansionStatusEnum myExpansionStatus;
@@ -76,6 +85,8 @@ public class TermValueSet implements Serializable {
 	public TermValueSet() {
 		super();
 		myExpansionStatus = TermValueSetPreExpansionStatusEnum.NOT_EXPANDED;
+		myTotalConcepts = 0L;
+		myTotalConceptDesignations = 0L;
 	}
 
 	public Long getId() {
@@ -120,6 +131,48 @@ public class TermValueSet implements Serializable {
 		return myConcepts;
 	}
 
+	public Long getTotalConcepts() {
+		return myTotalConcepts;
+	}
+
+	public TermValueSet setTotalConcepts(Long theTotalConcepts) {
+		myTotalConcepts = theTotalConcepts;
+		return this;
+	}
+
+	public TermValueSet decrementTotalConcepts() {
+		if (myTotalConcepts > 0) {
+			myTotalConcepts--;
+		}
+		return this;
+	}
+
+	public TermValueSet incrementTotalConcepts() {
+		myTotalConcepts++;
+		return this;
+	}
+
+	public Long getTotalConceptDesignations() {
+		return myTotalConceptDesignations;
+	}
+
+	public TermValueSet setTotalConceptDesignations(Long theTotalConceptDesignations) {
+		myTotalConceptDesignations = theTotalConceptDesignations;
+		return this;
+	}
+
+	public TermValueSet decrementTotalConceptDesignations() {
+		if (myTotalConceptDesignations > 0) {
+			myTotalConceptDesignations--;
+		}
+		return this;
+	}
+
+	public TermValueSet incrementTotalConceptDesignations() {
+		myTotalConceptDesignations++;
+		return this;
+	}
+
 	public TermValueSetPreExpansionStatusEnum getExpansionStatus() {
 		return myExpansionStatus;
 	}
@@ -157,6 +210,8 @@ public class TermValueSet implements Serializable {
 			.append("myResourcePid", myResourcePid)
 			.append("myName", myName)
 			.append(myConcepts != null ? ("myConcepts - size=" + myConcepts.size()) : ("myConcepts=(null)"))
+			.append("myTotalConcepts", myTotalConcepts)
+			.append("myTotalConceptDesignations", myTotalConceptDesignations)
 			.append("myExpansionStatus", myExpansionStatus)
 			.toString();
 	}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetPreExpansionStatusEnum.java

@@ -29,26 +29,32 @@ import java.util.Map;
  * an expanded ValueSet has its included concepts stored in the terminology tables as well.
  */
 public enum TermValueSetPreExpansionStatusEnum {
-	/**
+	/*
 	 * Sorting agnostic.
 	 */
 
-	NOT_EXPANDED("notExpanded"),
+	NOT_EXPANDED("notExpanded", "The ValueSet is waiting to be picked up and pre-expanded by a scheduled task."),
-	EXPANSION_IN_PROGRESS("expansionInProgress"),
+	EXPANSION_IN_PROGRESS("expansionInProgress", "The ValueSet has been picked up by a scheduled task and pre-expansion is in progress."),
-	EXPANDED("expanded"),
+	EXPANDED("expanded", "The ValueSet has been picked up by a scheduled task and pre-expansion is complete."),
-	FAILED_TO_EXPAND("failedToExpand");
+	FAILED_TO_EXPAND("failedToExpand", "The ValueSet has been picked up by a scheduled task and pre-expansion has failed.");
 
 	private static Map<String, TermValueSetPreExpansionStatusEnum> ourValues;
 	private String myCode;
+	private String myDescription;
 
-	TermValueSetPreExpansionStatusEnum(String theCode) {
+	TermValueSetPreExpansionStatusEnum(String theCode, String theDescription) {
 		myCode = theCode;
+		myDescription = theDescription;
 	}
 
 	public String getCode() {
 		return myCode;
 	}
 
+	public String getDescription() {
+		return myDescription;
+	}
+
 	public static TermValueSetPreExpansionStatusEnum fromCode(String theCode) {
 		if (ourValues == null) {
 			HashMap<String, TermValueSetPreExpansionStatusEnum> values = new HashMap<String, TermValueSetPreExpansionStatusEnum>();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaProvider.java

@@ -44,13 +44,19 @@ import java.util.TreeSet;
 public class BaseJpaProvider {
 	public static final String REMOTE_ADDR = "req.remoteAddr";
 	public static final String REMOTE_UA = "req.userAgent";
-
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseJpaProvider.class);
 	@Autowired
 	protected DaoConfig myDaoConfig;
-
-	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseJpaProvider.class);
 	private FhirContext myContext;
 
+	public BaseJpaProvider() {
+		super();
+	}
+
+	public void setDaoConfigForUnitTest(DaoConfig theDaoConfig) {
+		myDaoConfig = theDaoConfig;
+	}
+
 	protected ExpungeOptions createExpungeOptions(IPrimitiveType<? extends Integer> theLimit, IPrimitiveType<? extends Boolean> theExpungeDeletedResources, IPrimitiveType<? extends Boolean> theExpungeOldVersions, IPrimitiveType<? extends Boolean> theExpungeEverything) {
 		ExpungeOptions options = new ExpungeOptions();
 		if (theLimit != null && theLimit.getValue() != null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java

@@ -28,6 +28,7 @@ import ca.uhn.fhir.jpa.dao.data.*;
 import ca.uhn.fhir.jpa.entity.*;
 import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
+import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl;
 import ca.uhn.fhir.jpa.util.ScrollableResultsIterator;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
@@ -55,6 +56,7 @@ import org.hibernate.search.query.dsl.BooleanJunction;
 import org.hibernate.search.query.dsl.QueryBuilder;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.instance.model.api.IBaseCoding;
+import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
 import org.hl7.fhir.r4.model.*;
@@ -145,6 +147,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 	@Autowired
 	private PlatformTransactionManager myTransactionMgr;
 	private IFhirResourceDaoCodeSystem<?, ?, ?> myCodeSystemResourceDao;
+	private IFhirResourceDaoValueSet<?, ?, ?> myValueSetResourceDao;
 	private Cache<TranslationQuery, List<TermConceptMapGroupElementTarget>> myTranslationCache;
 	private Cache<TranslationQuery, List<TermConceptMapGroupElement>> myTranslationWithReverseCache;
 	private int myFetchSize = DEFAULT_FETCH_SIZE;
@@ -370,13 +373,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 
 			myConceptMapDao.deleteTermConceptMapById(existingTermConceptMap.getId());
 			ourLog.info("Done deleting existing TermConceptMap[{}] and its children.", existingTermConceptMap.getId());
-
-			ourLog.info("Flushing...");
-			myConceptMapGroupElementTargetDao.flush();
-			myConceptMapGroupElementDao.flush();
-			myConceptMapGroupDao.flush();
-			myConceptMapDao.flush();
-			ourLog.info("Done flushing.");
 		}
 	}
 
@@ -398,12 +394,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 			myValueSetConceptDao.deleteByTermValueSetId(existingTermValueSet.getId());
 			myValueSetDao.deleteByTermValueSetId(existingTermValueSet.getId());
 			ourLog.info("Done deleting existing TermValueSet[{}] and its children.", existingTermValueSet.getId());
-
-			ourLog.info("Flushing...");
-			myValueSetConceptDesignationDao.flush();
-			myValueSetConceptDao.flush();
-			myValueSetDao.flush();
-			ourLog.info("Done flushing.");
 		}
 	}
 
@@ -486,7 +476,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 
 		Optional<TermValueSet> optionalTermValueSet;
 		if (theValueSetToExpand.hasId()) {
-			optionalTermValueSet = myValueSetDao.findByResourcePid(theValueSetToExpand.getIdElement().getIdPartAsLong());
+			Long valueSetResourcePid = getValueSetResourcePid(theValueSetToExpand.getIdElement());
+			optionalTermValueSet = myValueSetDao.findByResourcePid(valueSetResourcePid);
 		} else if (theValueSetToExpand.hasUrl()) {
 			optionalTermValueSet = myValueSetDao.findByUrl(theValueSetToExpand.getUrl());
 		} else {
@@ -494,12 +485,18 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 		}
 
 		if (!optionalTermValueSet.isPresent()) {
-			throw new InvalidRequestException("ValueSet is not present in terminology tables: " + theValueSetToExpand.getUrl());
+			ourLog.warn("ValueSet is not present in terminology tables. Will perform in-memory expansion without parameters. Will schedule this ValueSet for pre-expansion. {}", getValueSetInfo(theValueSetToExpand));
+			myDeferredValueSets.add(theValueSetToExpand);
+			return expandValueSet(theValueSetToExpand); // In-memory expansion.
 		}
 
 		TermValueSet termValueSet = optionalTermValueSet.get();
 
-		validatePreExpansionStatusOfValueSetOrThrowException(termValueSet.getExpansionStatus());
+		if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
+			ourLog.warn("{} is present in terminology tables but not ready for persistence-backed invocation of operation $expand. Will perform in-memory expansion without parameters. Current status: {} | {}",
+				getValueSetInfo(theValueSetToExpand), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
+			return expandValueSet(theValueSetToExpand); // In-memory expansion.
+		}
 
 		ValueSet.ValueSetExpansionComponent expansionComponent = new ValueSet.ValueSetExpansionComponent();
 		expansionComponent.setIdentifier(UUID.randomUUID().toString());
@@ -514,22 +511,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 		return valueSet;
 	}
 
-	private void validatePreExpansionStatusOfValueSetOrThrowException(TermValueSetPreExpansionStatusEnum thePreExpansionStatus) {
-		if (TermValueSetPreExpansionStatusEnum.EXPANDED != thePreExpansionStatus) {
-			String statusMsg = myContext.getLocalizer().getMessage(
-				TermValueSetPreExpansionStatusEnum.class,
-				thePreExpansionStatus.getCode());
-			String msg = myContext.getLocalizer().getMessage(
-				BaseHapiTerminologySvcImpl.class,
-				"valueSetNotReadyForExpand",
-				thePreExpansionStatus.name(),
-				statusMsg);
-			throw new UnprocessableEntityException(msg);
-		}
-	}
-
 	private void populateExpansionComponent(ValueSet.ValueSetExpansionComponent theExpansionComponent, TermValueSet theTermValueSet, int theOffset, int theCount) {
-		int total = myValueSetConceptDao.countByTermValueSetId(theTermValueSet.getId());
+		int total = theTermValueSet.getTotalConcepts().intValue();
 		theExpansionComponent.setTotal(total);
 		theExpansionComponent.setOffset(theOffset);
 		theExpansionComponent.addParameter().setName("offset").setValue(new IntegerType(theOffset));
@@ -544,72 +527,58 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 
 	private void expandConcepts(ValueSet.ValueSetExpansionComponent theExpansionComponent, TermValueSet theTermValueSet, int theOffset, int theCount) {
 		int conceptsExpanded = 0;
-		for (int i = theOffset; i < (theOffset + theCount); i++) {
+		int toIndex = theOffset + theCount;
-			final int page = i;
+		Slice<TermValueSetConcept> slice = myValueSetConceptDao.findByTermValueSetIdAndPreFetchDesignations(SearchCoordinatorSvcImpl.toPage(theOffset, toIndex), theTermValueSet.getId());
-			Supplier<Slice<TermValueSetConcept>> loader = () -> myValueSetConceptDao.findByTermValueSetId(PageRequest.of(page, 1), theTermValueSet.getId());
+		if (!slice.hasContent()) {
+			logConceptsExpanded(theTermValueSet, conceptsExpanded);
+			return;
+		}
 
-			Slice<TermValueSetConcept> slice = loader.get();
+		for (TermValueSetConcept concept : slice.getContent()) {
-			if (!slice.hasContent()) {
+			ValueSet.ValueSetExpansionContainsComponent containsComponent = theExpansionComponent.addContains();
-				break;
+			containsComponent.setSystem(concept.getSystem());
-			}
+			containsComponent.setCode(concept.getCode());
+			containsComponent.setDisplay(concept.getDisplay());
 
-			for (TermValueSetConcept concept : slice.getContent()) {
+			// TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to make this optional.
-				ValueSet.ValueSetExpansionContainsComponent containsComponent = theExpansionComponent.addContains();
+			expandDesignations(theTermValueSet, concept, containsComponent);
-				containsComponent.setSystem(concept.getSystem());
-				containsComponent.setCode(concept.getCode());
-				containsComponent.setDisplay(concept.getDisplay());
 
-				// TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to make this optional.
+			if (++conceptsExpanded % 250 == 0) {
-				expandDesignations(theTermValueSet, concept, containsComponent);
+				logConceptsExpanded(theTermValueSet, conceptsExpanded);
-
-				if (++conceptsExpanded % 250 == 0) {
-					ourLog.info("Have expanded {} concepts in ValueSet[{}]", conceptsExpanded, theTermValueSet.getUrl());
-				}
-			}
-
-			if (!slice.hasNext()) {
-				break;
 			}
 		}
 
-		if (conceptsExpanded > 0) {
+		logConceptsExpanded(theTermValueSet, conceptsExpanded);
-			ourLog.info("Have expanded {} concepts in ValueSet[{}]", conceptsExpanded, theTermValueSet.getUrl());
+	}
+
+	private void logConceptsExpanded(TermValueSet theTermValueSet, int theConceptsExpanded) {
+		if (theConceptsExpanded > 0) {
+			ourLog.info("Have expanded {} concepts in ValueSet[{}]", theConceptsExpanded, theTermValueSet.getUrl());
 		}
 	}
 
 	private void expandDesignations(TermValueSet theValueSet, TermValueSetConcept theConcept, ValueSet.ValueSetExpansionContainsComponent theContainsComponent) {
 		int designationsExpanded = 0;
-		int index = 0;
+		for (TermValueSetConceptDesignation designation : theConcept.getDesignations()) {
-		while (true) {
+			ValueSet.ConceptReferenceDesignationComponent designationComponent = theContainsComponent.addDesignation();
-			final int page = index++;
+			designationComponent.setLanguage(designation.getLanguage());
-			Supplier<Slice<TermValueSetConceptDesignation>> loader = () -> myValueSetConceptDesignationDao.findByTermValueSetConceptId(PageRequest.of(page, 1000), theConcept.getId());
+			designationComponent.setUse(new Coding(
+				designation.getUseSystem(),
+				designation.getUseCode(),
+				designation.getUseDisplay()));
+			designationComponent.setValue(designation.getValue());
 
-			Slice<TermValueSetConceptDesignation> slice = loader.get();
+			if (++designationsExpanded % 250 == 0) {
-			if (!slice.hasContent()) {
+				logDesignationsExpanded(theValueSet, theConcept, designationsExpanded);
-				break;
-			}
-
-			for (TermValueSetConceptDesignation designation : slice.getContent()) {
-				ValueSet.ConceptReferenceDesignationComponent designationComponent = theContainsComponent.addDesignation();
-				designationComponent.setLanguage(designation.getLanguage());
-				designationComponent.setUse(new Coding(
-					designation.getUseSystem(),
-					designation.getUseCode(),
-					designation.getUseDisplay()));
-				designationComponent.setValue(designation.getValue());
-
-				if (++designationsExpanded % 250 == 0) {
-					ourLog.info("Have expanded {} designations for Concept[{}|{}] in ValueSet[{}]", designationsExpanded, theConcept.getSystem(), theConcept.getCode(), theValueSet.getUrl());
-				}
-			}
-
-			if (!slice.hasNext()) {
-				break;
 			}
 		}
 
-		if (designationsExpanded > 0) {
+		logDesignationsExpanded(theValueSet, theConcept, designationsExpanded);
-			ourLog.info("Have expanded {} designations for Concept[{}|{}] in ValueSet[{}]", designationsExpanded, theConcept.getSystem(), theConcept.getCode(), theValueSet.getUrl());
+	}
+
+	private void logDesignationsExpanded(TermValueSet theValueSet, TermValueSetConcept theConcept, int theDesignationsExpanded) {
+		if (theDesignationsExpanded > 0) {
+			ourLog.info("Have expanded {} designations for Concept[{}|{}] in ValueSet[{}]", theDesignationsExpanded, theConcept.getSystem(), theConcept.getCode(), theValueSet.getUrl());
 		}
 	}
 
@@ -960,28 +929,49 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 		}
 	}
 
+	@Override
+	public boolean isValueSetPreExpandedForCodeValidation(ValueSet theValueSet) {
+		Long valueSetResourcePid = getValueSetResourcePid(theValueSet.getIdElement());
+		Optional<TermValueSet> optionalTermValueSet = myValueSetDao.findByResourcePid(valueSetResourcePid);
+
+		if (!optionalTermValueSet.isPresent()) {
+			ourLog.warn("ValueSet is not present in terminology tables. Will perform in-memory code validation. Will schedule this ValueSet for pre-expansion. {}", getValueSetInfo(theValueSet));
+			myDeferredValueSets.add(theValueSet);
+			return false;
+		}
+
+		TermValueSet termValueSet = optionalTermValueSet.get();
+
+		if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
+			ourLog.warn("{} is present in terminology tables but not ready for persistence-backed invocation of operation $validation-code. Will perform in-memory code validation. Current status: {} | {}",
+				getValueSetInfo(theValueSet), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
+			return false;
+		}
+
+		return true;
+	}
+
 	protected ValidateCodeResult validateCodeIsInPreExpandedValueSet(
 		ValueSet theValueSet, String theSystem, String theCode, String theDisplay, Coding theCoding, CodeableConcept theCodeableConcept) {
 
 		ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet.hasId(), "ValueSet.id is required");
-
+		Long valueSetResourcePid = getValueSetResourcePid(theValueSet.getIdElement());
-		Long valueSetId = theValueSet.getIdElement().toUnqualifiedVersionless().getIdPartAsLong();
 
 		List<TermValueSetConcept> concepts = new ArrayList<>();
 		if (isNotBlank(theCode)) {
 			if (isNotBlank(theSystem)) {
-				concepts = myValueSetConceptDao.findOneByValueSetIdSystemAndCode(valueSetId, theSystem, theCode);
+				concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theSystem, theCode));
 			} else {
-				concepts = myValueSetConceptDao.findOneByValueSetIdAndCode(valueSetId, theCode);
+				concepts.addAll(myValueSetConceptDao.findByValueSetResourcePidAndCode(valueSetResourcePid, theCode));
 			}
 		} else if (theCoding != null) {
 			if (theCoding.hasSystem() && theCoding.hasCode()) {
-				concepts = myValueSetConceptDao.findOneByValueSetIdSystemAndCode(valueSetId, theCoding.getSystem(), theCoding.getCode());
+				concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theCoding.getSystem(), theCoding.getCode()));
 			}
 		} else if (theCodeableConcept != null){
 			for (Coding coding : theCodeableConcept.getCoding()) {
 				if (coding.hasSystem() && coding.hasCode()) {
-					concepts = myValueSetConceptDao.findOneByValueSetIdSystemAndCode(valueSetId, coding.getSystem(), coding.getCode());
+					concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, coding.getSystem(), coding.getCode()));
 					if (!concepts.isEmpty()) {
 						break;
 					}
@@ -1002,6 +992,15 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 		return null;
 	}
 
+	private List<TermValueSetConcept> findByValueSetResourcePidSystemAndCode(Long theResourcePid, String theSystem, String theCode) {
+		List<TermValueSetConcept> retVal = new ArrayList<>();
+		Optional<TermValueSetConcept> optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCode(theResourcePid, theSystem, theCode);
+		if (optionalTermValueSetConcept.isPresent()) {
+			retVal.add(optionalTermValueSetConcept.get());
+		}
+		return retVal;
+	}
+
 	private void fetchChildren(TermConcept theConcept, Set<TermConcept> theSetToPopulate) {
 		for (TermConceptParentChildLink nextChildLink : theConcept.getChildren()) {
 			TermConcept nextChild = nextChildLink.getChild();
@@ -1134,6 +1133,27 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 
 	protected abstract CodeSystem getCodeSystemFromContext(String theSystem);
 
+	private Long getCodeSystemResourcePid(IIdType theIdType) {
+		return getCodeSystemResourcePid(theIdType, null);
+	}
+
+	private Long getCodeSystemResourcePid(IIdType theIdType, RequestDetails theRequestDetails) {
+		return getResourcePid(myCodeSystemResourceDao, theIdType, theRequestDetails);
+	}
+
+	private Long getValueSetResourcePid(IIdType theIdType) {
+		return getValueSetResourcePid(theIdType, null);
+	}
+
+	private Long getValueSetResourcePid(IIdType theIdType, RequestDetails theRequestDetails) {
+		return getResourcePid(myValueSetResourceDao, theIdType, theRequestDetails);
+	}
+
+	private Long getResourcePid(IFhirResourceDao<? extends IBaseResource> theResourceDao, IIdType theIdType, RequestDetails theRequestDetails) {
+		ResourceTable resourceTable = (ResourceTable) theResourceDao.readEntity(theIdType, theRequestDetails);
+		return resourceTable.getId();
+	}
+
 	private void persistChildren(TermConcept theConcept, TermCodeSystemVersion theCodeSystem, IdentityHashMap<TermConcept, Object> theConceptsStack, int theTotalConcepts) {
 		if (theConceptsStack.put(theConcept, PLACEHOLDER_OBJECT) != null) {
 			return;
@@ -1467,6 +1487,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 	@PostConstruct
 	public void start() {
 		myCodeSystemResourceDao = myApplicationContext.getBean(IFhirResourceDaoCodeSystem.class);
+		myValueSetResourceDao = myApplicationContext.getBean(IFhirResourceDaoValueSet.class);
 		myTxTemplate = new TransactionTemplate(myTransactionManager);
 	}
 
@@ -1600,7 +1621,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 			if (theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.COMPLETE || theCodeSystem.getContent() == null || theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) {
 				ourLog.info("CodeSystem {} has a status of {}, going to store concepts in terminology tables", theResourceEntity.getIdDt().getValue(), theCodeSystem.getContentElement().getValueAsString());
 
-				Long codeSystemResourcePid = IDao.RESOURCE_PID.get(theCodeSystem);
+				Long codeSystemResourcePid = getCodeSystemResourcePid(theCodeSystem.getIdElement());
 				TermCodeSystemVersion persCs = myCodeSystemVersionDao.findCurrentVersionForCodeSystemResourcePid(codeSystemResourcePid);
 				if (persCs != null) {
 					ourLog.info("Code system version already exists in database");
@@ -1799,7 +1820,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 
 	@Scheduled(fixedDelay = 600000) // 10 minutes.
 	@Override
-	public synchronized void preExpandValueSetToTerminologyTables() {
+	public synchronized void preExpandDeferredValueSetsToTerminologyTables() {
 		if (isNotSafeToPreExpandValueSets()) {
 			ourLog.info("Skipping scheduled pre-expansion of ValueSets while deferred entities are being loaded.");
 			return;
@@ -1827,7 +1848,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
 					TermValueSet refreshedValueSetToExpand = myValueSetDao.findById(valueSetToExpand.getId()).get();
 					return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
 				});
-				expandValueSet(valueSet, new ValueSetConceptAccumulator(valueSetToExpand, myValueSetConceptDao, myValueSetConceptDesignationDao));
+				expandValueSet(valueSet, new ValueSetConceptAccumulator(valueSetToExpand, myValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao));
 
 				// We are done with this ValueSet.
 				txTemplate.execute(t -> {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java

@@ -26,7 +26,9 @@ import org.hl7.fhir.instance.hapi.validation.IValidationSupport;
 import org.hl7.fhir.instance.model.api.IBaseDatatype;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
-import org.hl7.fhir.r4.model.*;
+import org.hl7.fhir.r4.model.CodeSystem;
+import org.hl7.fhir.r4.model.ConceptMap;
+import org.hl7.fhir.r4.model.ValueSet;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.util.ArrayList;
@@ -153,4 +155,9 @@ public class HapiTerminologySvcDstu2 extends BaseHapiTerminologySvcImpl {
 	public ValidateCodeResult validateCodeIsInPreExpandedValueSet(IBaseResource theValueSet, String theSystem, String theCode, String theDisplay, IBaseDatatype theCoding, IBaseDatatype theCodeableConcept) {
 		throw new UnsupportedOperationException();
 	}
+
+	@Override
+	public boolean isValueSetPreExpandedForCodeValidation(IBaseResource theValueSet) {
+		throw new UnsupportedOperationException();
+	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java

@@ -9,6 +9,7 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.util.CoverageIgnore;
 import ca.uhn.fhir.util.UrlUtil;
+import ca.uhn.fhir.util.ValidateUtil;
 import org.hl7.fhir.convertors.VersionConvertor_30_40;
 import org.hl7.fhir.dstu3.hapi.ctx.IValidationSupport;
 import org.hl7.fhir.dstu3.model.*;
@@ -364,24 +365,33 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
 
 	@Override
 	public ValidateCodeResult validateCodeIsInPreExpandedValueSet(IBaseResource theValueSet, String theSystem, String theCode, String theDisplay, IBaseDatatype theCoding, IBaseDatatype theCodeableConcept) {
+		ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null");
 		ValueSet valueSet = (ValueSet) theValueSet;
+		org.hl7.fhir.r4.model.ValueSet valueSetR4 = VersionConvertor_30_40.convertValueSet(valueSet);
+
 		Coding coding = (Coding) theCoding;
+		org.hl7.fhir.r4.model.Coding codingR4 = null;
+		if (coding != null) {
+			codingR4 = new org.hl7.fhir.r4.model.Coding(coding.getSystem(), coding.getCode(), coding.getDisplay());
+		}
+
 		CodeableConcept codeableConcept = (CodeableConcept) theCodeableConcept;
-
+		org.hl7.fhir.r4.model.CodeableConcept codeableConceptR4 = null;
-		try {
+		if (codeableConcept != null) {
-			org.hl7.fhir.r4.model.ValueSet valueSetR4;
+			codeableConceptR4 = new org.hl7.fhir.r4.model.CodeableConcept();
-			valueSetR4 = VersionConvertor_30_40.convertValueSet(valueSet);
-
-			org.hl7.fhir.r4.model.Coding codingR4 = new org.hl7.fhir.r4.model.Coding(coding.getSystem(), coding.getCode(), coding.getDisplay());
-
-			org.hl7.fhir.r4.model.CodeableConcept codeableConceptR4 = new org.hl7.fhir.r4.model.CodeableConcept();
 			for (Coding nestedCoding : codeableConcept.getCoding()) {
 				codeableConceptR4.addCoding(new org.hl7.fhir.r4.model.Coding(nestedCoding.getSystem(), nestedCoding.getCode(), nestedCoding.getDisplay()));
 			}
-
-			return super.validateCodeIsInPreExpandedValueSet(valueSetR4, theSystem, theCode, theDisplay, codingR4, codeableConceptR4);
-		} catch (FHIRException e) {
-			throw new InternalErrorException(e);
 		}
+
+		return super.validateCodeIsInPreExpandedValueSet(valueSetR4, theSystem, theCode, theDisplay, codingR4, codeableConceptR4);
+	}
+
+	@Override
+	public boolean isValueSetPreExpandedForCodeValidation(IBaseResource theValueSet) {
+		ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null");
+		ValueSet valueSet = (ValueSet) theValueSet;
+		org.hl7.fhir.r4.model.ValueSet valueSetR4 = VersionConvertor_30_40.convertValueSet(valueSet);
+		return super.isValueSetPreExpandedForCodeValidation(valueSetR4);
 	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java

@@ -284,4 +284,10 @@ public class HapiTerminologySvcR4 extends BaseHapiTerminologySvcImpl implements
 		CodeableConcept codeableConcept = (CodeableConcept) theCodeableConcept;
 		return super.validateCodeIsInPreExpandedValueSet(valueSet, theSystem, theCode, theDisplay, coding, codeableConcept);
 	}
+
+	@Override
+	public boolean isValueSetPreExpandedForCodeValidation(IBaseResource theValueSet) {
+		ValueSet valueSet = (ValueSet) theValueSet;
+		return super.isValueSetPreExpandedForCodeValidation(valueSet);
+	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR5.java

@@ -7,11 +7,12 @@ import ca.uhn.fhir.jpa.entity.TermConcept;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.util.CoverageIgnore;
 import ca.uhn.fhir.util.UrlUtil;
+import ca.uhn.fhir.util.ValidateUtil;
 import org.hl7.fhir.instance.model.api.IBaseDatatype;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
-import org.hl7.fhir.r5.model.*;
 import org.hl7.fhir.r5.hapi.ctx.IValidationSupport;
+import org.hl7.fhir.r5.model.*;
 import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
 import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
 import org.hl7.fhir.r5.terminologies.ValueSetExpander;
@@ -291,17 +292,33 @@ public class HapiTerminologySvcR5 extends BaseHapiTerminologySvcImpl implements
 
 	@Override
 	public ValidateCodeResult validateCodeIsInPreExpandedValueSet(IBaseResource theValueSet, String theSystem, String theCode, String theDisplay, IBaseDatatype theCoding, IBaseDatatype theCodeableConcept) {
-		org.hl7.fhir.r4.model.ValueSet valueSetR4 = org.hl7.fhir.convertors.conv40_50.ValueSet.convertValueSet((ValueSet) theValueSet);
+		ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null");
+		ValueSet valueSet = (ValueSet) theValueSet;
+		org.hl7.fhir.r4.model.ValueSet valueSetR4 = org.hl7.fhir.convertors.conv40_50.ValueSet.convertValueSet(valueSet);
 
 		Coding coding = (Coding) theCoding;
-		org.hl7.fhir.r4.model.Coding codingR4 = new org.hl7.fhir.r4.model.Coding(coding.getSystem(), coding.getCode(), coding.getDisplay());
+		org.hl7.fhir.r4.model.Coding codingR4 = null;
+		if (coding != null) {
+			codingR4 = new org.hl7.fhir.r4.model.Coding(coding.getSystem(), coding.getCode(), coding.getDisplay());
+		}
 
 		CodeableConcept codeableConcept = (CodeableConcept) theCodeableConcept;
-		org.hl7.fhir.r4.model.CodeableConcept codeableConceptR4 = new org.hl7.fhir.r4.model.CodeableConcept();
+		org.hl7.fhir.r4.model.CodeableConcept codeableConceptR4 = null;
-		for (Coding nestedCoding : codeableConcept.getCoding()) {
+		if (codeableConcept != null) {
-			codeableConceptR4.addCoding(new org.hl7.fhir.r4.model.Coding(nestedCoding.getSystem(), nestedCoding.getCode(), nestedCoding.getDisplay()));
+			codeableConceptR4 = new org.hl7.fhir.r4.model.CodeableConcept();
+			for (Coding nestedCoding : codeableConcept.getCoding()) {
+				codeableConceptR4.addCoding(new org.hl7.fhir.r4.model.Coding(nestedCoding.getSystem(), nestedCoding.getCode(), nestedCoding.getDisplay()));
+			}
 		}
 
 		return super.validateCodeIsInPreExpandedValueSet(valueSetR4, theSystem, theCode, theDisplay, codingR4, codeableConceptR4);
 	}
+
+	@Override
+	public boolean isValueSetPreExpandedForCodeValidation(IBaseResource theValueSet) {
+		ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null");
+		ValueSet valueSet = (ValueSet) theValueSet;
+		org.hl7.fhir.r4.model.ValueSet valueSetR4 = org.hl7.fhir.convertors.conv40_50.ValueSet.convertValueSet(valueSet);
+		return super.isValueSetPreExpandedForCodeValidation(valueSetR4);
+	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java

@@ -6,7 +6,9 @@ import ca.uhn.fhir.jpa.entity.*;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import org.hl7.fhir.instance.model.api.*;
-import org.hl7.fhir.r4.model.*;
+import org.hl7.fhir.r4.model.CodeSystem;
+import org.hl7.fhir.r4.model.ConceptMap;
+import org.hl7.fhir.r4.model.ValueSet;
 
 import javax.annotation.Nullable;
 import java.util.List;
@@ -111,10 +113,17 @@ public interface IHapiTerminologySvc {
 
 	AtomicInteger applyDeltaCodesystemsRemove(String theSystem, CodeSystem theDelta);
 
-	void preExpandValueSetToTerminologyTables();
+	void preExpandDeferredValueSetsToTerminologyTables();
 
 	/**
 	 * Version independent
 	 */
 	ValidateCodeResult validateCodeIsInPreExpandedValueSet(IBaseResource theValueSet, String theSystem, String theCode, String theDisplay, IBaseDatatype theCoding, IBaseDatatype theCodeableConcept);
+
+	boolean isValueSetPreExpandedForCodeValidation(ValueSet theValueSet);
+
+	/**
+	 * Version independent
+	 */
+	boolean isValueSetPreExpandedForCodeValidation(IBaseResource theValueSet);
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java

@@ -22,6 +22,7 @@ package ca.uhn.fhir.jpa.term;
 
 import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao;
 import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDesignationDao;
+import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao;
 import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
 import ca.uhn.fhir.jpa.entity.TermValueSet;
 import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
@@ -38,13 +39,15 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ValueSetConceptAccumulator.class);
 
 	private TermValueSet myTermValueSet;
+	private ITermValueSetDao myValueSetDao;
 	private ITermValueSetConceptDao myValueSetConceptDao;
 	private ITermValueSetConceptDesignationDao myValueSetConceptDesignationDao;
 	private int myConceptsSaved;
 	private int myDesignationsSaved;
 
-	public ValueSetConceptAccumulator(@Nonnull TermValueSet theTermValueSet, @Nonnull ITermValueSetConceptDao theValueSetConceptDao, @Nonnull ITermValueSetConceptDesignationDao theValueSetConceptDesignationDao) {
+	public ValueSetConceptAccumulator(@Nonnull TermValueSet theTermValueSet, @Nonnull ITermValueSetDao theValueSetDao, @Nonnull ITermValueSetConceptDao theValueSetConceptDao, @Nonnull ITermValueSetConceptDesignationDao theValueSetConceptDesignationDao) {
 		myTermValueSet = theTermValueSet;
+		myValueSetDao = theValueSetDao;
 		myValueSetConceptDao = theValueSetConceptDao;
 		myValueSetConceptDesignationDao = theValueSetConceptDesignationDao;
 		myConceptsSaved = 0;
@@ -79,14 +82,12 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 			ourLog.info("Excluding [{}|{}] from ValueSet[{}]", concept.getSystem(), concept.getCode(), myTermValueSet.getUrl());
 			for (TermValueSetConceptDesignation designation : concept.getDesignations()) {
 				myValueSetConceptDesignationDao.deleteById(designation.getId());
+				myTermValueSet.decrementTotalConceptDesignations();
 			}
 			myValueSetConceptDao.deleteById(concept.getId());
+			myTermValueSet.decrementTotalConcepts();
+			myValueSetDao.save(myTermValueSet);
 			ourLog.info("Done excluding [{}|{}] from ValueSet[{}]", concept.getSystem(), concept.getCode(), myTermValueSet.getUrl());
-
-			ourLog.info("Flushing...");
-			myValueSetConceptDesignationDao.flush();
-			myValueSetConceptDao.flush();
-			ourLog.info("Done flushing.");
 		}
 	}
 
@@ -102,6 +103,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 			concept.setDisplay(theDisplay);
 		}
 		myValueSetConceptDao.save(concept);
+		myValueSetDao.save(myTermValueSet.incrementTotalConcepts());
 
 		if (myConceptsSaved++ % 250 == 0) { // TODO: DM 2019-08-23 - This message never appears in the log. Fix it!
 			ourLog.info("Have pre-expanded {} concepts in ValueSet[{}]", myConceptsSaved, myTermValueSet.getUrl());
@@ -126,6 +128,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 		}
 		designation.setValue(theDesignation.getValue());
 		myValueSetConceptDesignationDao.save(designation);
+		myValueSetDao.save(myTermValueSet.incrementTotalConceptDesignations());
 
 		if (myDesignationsSaved++ % 250 == 0) { // TODO: DM 2019-08-23 - This message never appears in the log. Fix it!
 			ourLog.info("Have pre-expanded {} designations for Concept[{}|{}] in ValueSet[{}]", myDesignationsSaved, theConcept.getSystem(), theConcept.getCode(), myTermValueSet.getUrl());

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SourceTest.java

@@ -37,7 +37,7 @@ public class FhirResourceDaoDstu3SourceTest extends BaseJpaDstu3Test {
 
 	@Before
 	public void before() {
-		myDaoConfig.setStoreMetaSourceInformation(DaoConfig.StoreMetaSourceInformation.SOURCE_URI_AND_REQUEST_ID);
+		myDaoConfig.setStoreMetaSourceInformation(DaoConfig.StoreMetaSourceInformationEnum.SOURCE_URI_AND_REQUEST_ID);
 	}
 
 	@Test
@@ -184,7 +184,7 @@ public class FhirResourceDaoDstu3SourceTest extends BaseJpaDstu3Test {
 
 	@Test
 	public void testSourceDisabled() {
-		myDaoConfig.setStoreMetaSourceInformation(DaoConfig.StoreMetaSourceInformation.NONE);
+		myDaoConfig.setStoreMetaSourceInformation(DaoConfig.StoreMetaSourceInformationEnum.NONE);
 		when(mySrd.getRequestId()).thenReturn("0000000000000000");
 
 		Patient pt0 = new Patient();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SourceTest.java

@@ -18,8 +18,6 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.UUID;
-
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.matchesPattern;
 import static org.junit.Assert.assertEquals;
@@ -37,7 +35,7 @@ public class FhirResourceDaoR4SourceTest extends BaseJpaR4Test {
 
 	@Before
 	public void before() {
-		myDaoConfig.setStoreMetaSourceInformation(DaoConfig.StoreMetaSourceInformation.SOURCE_URI_AND_REQUEST_ID);
+		myDaoConfig.setStoreMetaSourceInformation(DaoConfig.StoreMetaSourceInformationEnum.SOURCE_URI_AND_REQUEST_ID);
 	}
 
 	@Test
@@ -184,7 +182,7 @@ public class FhirResourceDaoR4SourceTest extends BaseJpaR4Test {
 
 	@Test
 	public void testSourceDisabled() {
-		myDaoConfig.setStoreMetaSourceInformation(DaoConfig.StoreMetaSourceInformation.NONE);
+		myDaoConfig.setStoreMetaSourceInformation(DaoConfig.StoreMetaSourceInformationEnum.NONE);
 		when(mySrd.getRequestId()).thenReturn("0000000000000000");
 
 		Patient pt0 = new Patient();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValueSetTest.java

@@ -1,14 +1,12 @@
 package ca.uhn.fhir.jpa.dao.r4;
 
+import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet.ValidateCodeResult;
 import ca.uhn.fhir.util.TestUtil;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
 import org.hl7.fhir.r4.model.*;
-import org.junit.AfterClass;
+import org.junit.*;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.io.IOException;
@@ -22,6 +20,10 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
 
 	private IIdType myExtensionalVsId;
 
+	@After
+	public void after() {
+		myDaoConfig.setPreExpandValueSetsExperimental(new DaoConfig().isPreExpandValueSetsExperimental());
+	}
 
 	@AfterClass
 	public static void afterClassClearContext() {
@@ -124,6 +126,33 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
 		assertEquals("Systolic blood pressure at First encounter", result.getDisplay());
 	}
 
+	@Test
+	public void testValidateCodeOperationByResourceIdAndCodeableConceptWithExistingValueSetAndPreExpansionEnabled() {
+		myDaoConfig.setPreExpandValueSetsExperimental(true);
+
+		UriType valueSetIdentifier = null;
+		IIdType id = myExtensionalVsId;
+		CodeType code = null;
+		UriType system = null;
+		StringType display = null;
+		Coding coding = null;
+		CodeableConcept codeableConcept = new CodeableConcept();
+		codeableConcept.addCoding().setSystem("http://acme.org").setCode("11378-7");
+		ValidateCodeResult result = myValueSetDao.validateCode(valueSetIdentifier, id, code, system, display, coding, codeableConcept, mySrd);
+		assertTrue(result.isResult());
+		assertEquals("Systolic blood pressure at First encounter", result.getDisplay());
+
+		myTermSvc.saveDeferred();
+		result = myValueSetDao.validateCode(valueSetIdentifier, id, code, system, display, coding, codeableConcept, mySrd);
+		assertTrue(result.isResult());
+		assertEquals("Systolic blood pressure at First encounter", result.getDisplay());
+
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+		result = myValueSetDao.validateCode(valueSetIdentifier, id, code, system, display, coding, codeableConcept, mySrd);
+		assertTrue(result.isResult());
+		assertEquals("Systolic blood pressure at First encounter", result.getDisplay());
+	}
+
 	@Test
 	public void testValidateCodeOperationByResourceIdAndCodeAndSystem() {
 		UriType valueSetIdentifier = null;
@@ -138,6 +167,32 @@ public class FhirResourceDaoR4ValueSetTest extends BaseJpaR4Test {
 		assertEquals("Systolic blood pressure at First encounter", result.getDisplay());
 	}
 
+	@Test
+	public void testValidateCodeOperationByResourceIdAndCodeAndSystemWithExistingValueSetAndPreExpansionEnabled() {
+		myDaoConfig.setPreExpandValueSetsExperimental(true);
+
+		UriType valueSetIdentifier = null;
+		IIdType id = myExtensionalVsId;
+		CodeType code = new CodeType("11378-7");
+		UriType system = new UriType("http://acme.org");
+		StringType display = null;
+		Coding coding = null;
+		CodeableConcept codeableConcept = null;
+		ValidateCodeResult result = myValueSetDao.validateCode(valueSetIdentifier, id, code, system, display, coding, codeableConcept, mySrd);
+		assertTrue(result.isResult());
+		assertEquals("Systolic blood pressure at First encounter", result.getDisplay());
+
+		myTermSvc.saveDeferred();
+		result = myValueSetDao.validateCode(valueSetIdentifier, id, code, system, display, coding, codeableConcept, mySrd);
+		assertTrue(result.isResult());
+		assertEquals("Systolic blood pressure at First encounter", result.getDisplay());
+
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+		result = myValueSetDao.validateCode(valueSetIdentifier, id, code, system, display, coding, codeableConcept, mySrd);
+		assertTrue(result.isResult());
+		assertEquals("Systolic blood pressure at First encounter", result.getDisplay());
+	}
+
 	@Test
 	public void testExpandById() throws IOException {
 		String resp;

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/entity/TermValueSetPreExpansionStatusEnumTest.java

@@ -1,21 +0,0 @@
-package ca.uhn.fhir.jpa.entity;
-
-import ca.uhn.fhir.i18n.HapiLocalizer;
-import org.junit.Test;
-
-import static org.junit.Assert.fail;
-
-public class TermValueSetPreExpansionStatusEnumTest {
-	@Test
-	public void testHaveDescriptions() {
-		HapiLocalizer localizer = new HapiLocalizer();
-
-		for (TermValueSetPreExpansionStatusEnum next : TermValueSetPreExpansionStatusEnum.values()) {
-			String key = "ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum." + next.getCode();
-			String msg = localizer.getMessage(key);
-			if (msg.equals(HapiLocalizer.UNKNOWN_I18N_KEY_MESSAGE)) {
-				fail("No value for key: " + key);
-			}
-		}
-	}
-}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java

@@ -319,6 +319,7 @@ public class SearchCoordinatorSvcImplTest {
 	public void testGetPage() {
 		Pageable page = SearchCoordinatorSvcImpl.toPage(50, 73);
 		assertEquals(50, page.getOffset());
+		assertEquals(23, page.getPageSize());
 	}
 
 	@Test

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java

@@ -600,7 +600,9 @@ public class TerminologySvcImplDstu3Test extends BaseJpaDstu3Test {
 		new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
 			@Override
 			protected void doInTransactionWithoutResult(TransactionStatus theStatus) {
-				TermCodeSystem codeSystem = myTermCodeSystemDao.findByResourcePid(codeSystemId.getIdPartAsLong());
+				ResourceTable resourceTable = (ResourceTable) myCodeSystemDao.readEntity(codeSystemResource.getIdElement(), null);
+				Long codeSystemResourcePid = resourceTable.getId();
+				TermCodeSystem codeSystem = myTermCodeSystemDao.findByResourcePid(codeSystemResourcePid);
 				assertEquals(CS_URL, codeSystem.getCodeSystemUri());
 				assertEquals("SYSTEM NAME", codeSystem.getName());
 

hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java

@@ -90,6 +90,12 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 		// Drop HFJ_SEARCH_RESULT foreign keys
 		version.onTable("HFJ_SEARCH_RESULT").dropForeignKey("FK_SEARCHRES_RES");
 		version.onTable("HFJ_SEARCH_RESULT").dropForeignKey("FK_SEARCHRES_SEARCH");
+
+		// TermValueSet
+		version.startSectionWithMessage("Processing table: TRM_VALUESET");
+		Builder.BuilderWithTableName termValueSetTable = version.onTable("TRM_VALUESET");
+		termValueSetTable.addColumn("TOTAL_CONCEPTS").nonNullable().type(BaseTableColumnTypeTask.ColumnTypeEnum.LONG);
+		termValueSetTable.addColumn("TOTAL_CONCEPT_DESIGNATIONS").nonNullable().type(BaseTableColumnTypeTask.ColumnTypeEnum.LONG);
 	}
 
 	protected void init400() {

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceHistoryProvenanceEntity.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.model.entity;
 
+/*-
+ * #%L
+ * HAPI FHIR Model
+ * %%
+ * Copyright (C) 2014 - 2019 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.rest.api.Constants;
 
 import javax.persistence.*;

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/ISearchParamExtractor.java

@@ -29,20 +29,20 @@ import java.util.Set;
 
 public interface ISearchParamExtractor {
 
-	public abstract Set<ResourceIndexedSearchParamCoords> extractSearchParamCoords(ResourceTable theEntity, IBaseResource theResource);
+	Set<ResourceIndexedSearchParamCoords> extractSearchParamCoords(ResourceTable theEntity, IBaseResource theResource);
 
-	public abstract Set<ResourceIndexedSearchParamDate> extractSearchParamDates(ResourceTable theEntity, IBaseResource theResource);
+	Set<ResourceIndexedSearchParamDate> extractSearchParamDates(ResourceTable theEntity, IBaseResource theResource);
 
-	public abstract Set<ResourceIndexedSearchParamNumber> extractSearchParamNumber(ResourceTable theEntity, IBaseResource theResource);
+	Set<ResourceIndexedSearchParamNumber> extractSearchParamNumber(ResourceTable theEntity, IBaseResource theResource);
 
-	public abstract Set<ResourceIndexedSearchParamQuantity> extractSearchParamQuantity(ResourceTable theEntity, IBaseResource theResource);
+	Set<ResourceIndexedSearchParamQuantity> extractSearchParamQuantity(ResourceTable theEntity, IBaseResource theResource);
 
-	public abstract Set<ResourceIndexedSearchParamString> extractSearchParamStrings(ResourceTable theEntity, IBaseResource theResource);
+	Set<ResourceIndexedSearchParamString> extractSearchParamStrings(ResourceTable theEntity, IBaseResource theResource);
 
-	public abstract Set<BaseResourceIndexedSearchParam> extractSearchParamTokens(ResourceTable theEntity, IBaseResource theResource);
+	Set<BaseResourceIndexedSearchParam> extractSearchParamTokens(ResourceTable theEntity, IBaseResource theResource);
 
-	public abstract Set<ResourceIndexedSearchParamUri> extractSearchParamUri(ResourceTable theEntity, IBaseResource theResource);
+	Set<ResourceIndexedSearchParamUri> extractSearchParamUri(ResourceTable theEntity, IBaseResource theResource);
 
-	public abstract List<PathAndRef> extractResourceLinks(IBaseResource theResource, RuntimeSearchParam theNextSpDef);
+	List<PathAndRef> extractResourceLinks(IBaseResource theResource, RuntimeSearchParam theNextSpDef);
 
 }

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/SearchParamExtractorR4.java

@@ -39,6 +39,7 @@ import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
 import org.hl7.fhir.r4.context.IWorkerContext;
+import org.hl7.fhir.r4.hapi.ctx.HapiWorkerContext;
 import org.hl7.fhir.r4.hapi.ctx.IValidationSupport;
 import org.hl7.fhir.r4.model.*;
 import org.hl7.fhir.r4.model.Enumeration;
@@ -48,6 +49,7 @@ import org.hl7.fhir.r4.model.Patient.PatientCommunicationComponent;
 import org.hl7.fhir.r4.utils.FHIRPathEngine;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import javax.annotation.PostConstruct;
 import javax.measure.unit.NonSI;
 import javax.measure.unit.Unit;
 import java.math.BigDecimal;
@@ -79,6 +81,8 @@ public class SearchParamExtractorR4 extends BaseSearchParamExtractor implements
 	@Autowired
 	private org.hl7.fhir.r4.hapi.ctx.IValidationSupport myValidationSupport;
 
+	private FHIRPathEngine myFhirPathEngine;
+
 	/**
 	 * Constructor
 	 */
@@ -91,6 +95,14 @@ public class SearchParamExtractorR4 extends BaseSearchParamExtractor implements
 	public SearchParamExtractorR4(ModelConfig theModelConfig, FhirContext theCtx, IValidationSupport theValidationSupport, ISearchParamRegistry theSearchParamRegistry) {
 		super(theCtx, theSearchParamRegistry);
 		myValidationSupport = theValidationSupport;
+		initFhirPath();
+	}
+
+	@PostConstruct
+	public void initFhirPath() {
+		IWorkerContext worker = new HapiWorkerContext(getContext(), myValidationSupport);
+		myFhirPathEngine = new FHIRPathEngine(worker);
+		myFhirPathEngine.setHostServices(new SearchParamExtractorR4HostServices());
 	}
 
 	private void addQuantity(ResourceTable theEntity, HashSet<ResourceIndexedSearchParamQuantity> retVal, String resourceName, Quantity nextValue) {
@@ -751,16 +763,12 @@ public class SearchParamExtractorR4 extends BaseSearchParamExtractor implements
 	 */
 	@Override
 	protected List<Object> extractValues(String thePaths, IBaseResource theResource) {
-		IWorkerContext worker = new org.hl7.fhir.r4.hapi.ctx.HapiWorkerContext(getContext(), myValidationSupport);
-		FHIRPathEngine fp = new FHIRPathEngine(worker);
-		fp.setHostServices(new SearchParamExtractorR4HostServices());
-
 		List<Object> values = new ArrayList<>();
 		String[] nextPathsSplit = SPLIT_R4.split(thePaths);
 		for (String nextPath : nextPathsSplit) {
 			List<Base> allValues;
 			try {
-				allValues = fp.evaluate((Base) theResource, nextPath);
+				allValues = myFhirPathEngine.evaluate((Base) theResource, nextPath);
 			} catch (FHIRException e) {
 				String msg = getContext().getLocalizer().getMessage(BaseSearchParamExtractor.class, "failedToExtractPaths", nextPath, e.toString());
 				throw new InternalErrorException(msg, e);

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/SearchParamExtractorR5.java

@@ -21,12 +21,10 @@ package ca.uhn.fhir.jpa.searchparam.extractor;
  */
 
 import ca.uhn.fhir.context.ConfigurationException;
-import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeSearchParam;
 import ca.uhn.fhir.jpa.model.entity.*;
 import ca.uhn.fhir.jpa.model.util.StringNormalizer;
 import ca.uhn.fhir.jpa.searchparam.SearchParamConstants;
-import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import com.google.common.annotations.VisibleForTesting;
@@ -39,6 +37,7 @@ import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
 import org.hl7.fhir.r5.context.IWorkerContext;
+import org.hl7.fhir.r5.hapi.ctx.HapiWorkerContext;
 import org.hl7.fhir.r5.hapi.ctx.IValidationSupport;
 import org.hl7.fhir.r5.model.*;
 import org.hl7.fhir.r5.model.Enumeration;
@@ -48,6 +47,7 @@ import org.hl7.fhir.r5.model.Patient.PatientCommunicationComponent;
 import org.hl7.fhir.r5.utils.FHIRPathEngine;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import javax.annotation.PostConstruct;
 import javax.measure.unit.NonSI;
 import javax.measure.unit.Unit;
 import java.math.BigDecimal;
@@ -78,6 +78,7 @@ public class SearchParamExtractorR5 extends BaseSearchParamExtractor implements
 
 	@Autowired
 	private IValidationSupport myValidationSupport;
+	private FHIRPathEngine myFhirPathEngine;
 
 	/**
 	 * Constructor
@@ -86,11 +87,11 @@ public class SearchParamExtractorR5 extends BaseSearchParamExtractor implements
 		super();
 	}
 
-	// This constructor is used by tests
+	@PostConstruct
-	@VisibleForTesting
+	public void initFhirPath() {
-	public SearchParamExtractorR5(ModelConfig theModelConfig, FhirContext theCtx, IValidationSupport theValidationSupport, ISearchParamRegistry theSearchParamRegistry) {
+		IWorkerContext worker = new HapiWorkerContext(getContext(), myValidationSupport);
-		super(theCtx, theSearchParamRegistry);
+		myFhirPathEngine = new FHIRPathEngine(worker);
-		myValidationSupport = theValidationSupport;
+		myFhirPathEngine.setHostServices(new SearchParamExtractorR5HostServices());
 	}
 
 	private void addQuantity(ResourceTable theEntity, HashSet<ResourceIndexedSearchParamQuantity> retVal, String resourceName, Quantity nextValue) {

hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/module/subscriber/SubscriptionMatchingSubscriber.java

@@ -163,14 +163,7 @@ public class SubscriptionMatchingSubscriber implements MessageHandler {
 				return;
 			}
 
-			ResourceDeliveryJsonMessage wrappedMsg = new ResourceDeliveryJsonMessage(deliveryMsg);
+			resourceMatched |= sendToDeliveryChannel(nextActiveSubscription, deliveryMsg);
-			MessageChannel deliveryChannel = nextActiveSubscription.getSubscribableChannel();
-			if (deliveryChannel != null) {
-				resourceMatched = true;
-				deliveryChannel.send(wrappedMsg);
-			} else {
-				ourLog.warn("Do not have delivery channel for subscription {}", nextActiveSubscription.getIdElement(myFhirContext));
-			}
 		}
 
 		if (!resourceMatched) {
@@ -181,6 +174,31 @@ public class SubscriptionMatchingSubscriber implements MessageHandler {
 		}
 	}
 
+	private boolean sendToDeliveryChannel(ActiveSubscription nextActiveSubscription, ResourceDeliveryMessage theDeliveryMsg) {
+		boolean retval = false;
+		ResourceDeliveryJsonMessage wrappedMsg = new ResourceDeliveryJsonMessage(theDeliveryMsg);
+		MessageChannel deliveryChannel = nextActiveSubscription.getSubscribableChannel();
+		if (deliveryChannel != null) {
+			retval = true;
+			trySendToDeliveryChannel(wrappedMsg, deliveryChannel);
+		} else {
+			ourLog.warn("Do not have delivery channel for subscription {}", nextActiveSubscription.getIdElement(myFhirContext));
+		}
+		return retval;
+	}
+
+	private void trySendToDeliveryChannel(ResourceDeliveryJsonMessage theWrappedMsg, MessageChannel theDeliveryChannel) {
+		try {
+			boolean success = theDeliveryChannel.send(theWrappedMsg);
+			if (!success) {
+				ourLog.warn("Failed to send message to Delivery Channel.");
+			}
+		} catch (RuntimeException e) {
+			ourLog.error("Failed to send message to Delivery Channel", e);
+			throw new RuntimeException("Failed to send message to Delivery Channel", e);
+		}
+	}
+
 	private String getId(ActiveSubscription theActiveSubscription) {
 		return theActiveSubscription.getIdElement(myFhirContext).toUnqualifiedVersionless().getValue();
 	}

pom.xml

@@ -547,6 +547,10 @@
 			<id>nickrobison-usds</id>
 			<name>Nick Robison</name>
 		</developer>
+		<developer>
+			<id>fitzoh</id>
+			<name>Andrew Fitzgerald</name>
+		</developer>
 	</developers>
 
 	<licenses>

src/changes/changes.xml

@@ -77,6 +77,15 @@
 				Two foreign keys have been dropped from the HFJ_SEARCH_RESULT table used by the FHIR search query cache. These
 				constraints did not add value and caused unneccessary contention when used under high load.
 			</action>
+			<action type="change">
+				An inefficient regex expression in UrlUtil was replaced with a much more efficient hand-written
+				checker. This regex was causing a noticable performance drop when feeding large numbers of transactions
+				into the JPA server at the same time (i.e. when loading Synthea data).
+			</action>
+			<action type="fix">
+				The FHIRPath engine used to parse search parameters in the JPA R4/R5 server is now reused across
+				requests, as it is somewhat expensive to create and is thread safe.
+			</action>
 			<action type="add">
 				It is now possible to submit a PATCH request as a part of a FHIR transaction in DSTU3 (previously this
 				was only supported in R4+). This is not officially part of the DSTU3 spec, but it can now be performed by
@@ -97,6 +106,10 @@
 				identifies the LOINC CodeSystem in <![CDATA[<code>ValueSet.compose.include.system</code>]]>. This ValueSet
 				includes all LOINC codes.
 			</action>
+			<action type="add" issue="1461">
+				A note has been added to the downloads page explaning the removal of the hapi-fhir-utilities
+				module. Thanks to Andrew Fitzgerald for the PR!
+			</action>
 		</release>
 		<release version="4.0.0" date="2019-08-14" description="Igloo">
 			<action type="add">