Merge remote-tracking branch 'remotes/origin/master' into ks-flyway

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/MetaUtil.java

@@ -26,18 +26,47 @@ import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.FhirVersionEnum;
 import ca.uhn.fhir.rest.api.Constants;
 import org.hl7.fhir.instance.model.api.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.List;
 
+
 public class MetaUtil {
+	private static final Logger ourLog = LoggerFactory.getLogger(MetaUtil.class);
 
 	private MetaUtil() {
 		// non-instantiable
 	}
 
 	public static String getSource(FhirContext theContext, IBaseMetaType theMeta) {
-		BaseRuntimeElementCompositeDefinition<?> elementDef = (BaseRuntimeElementCompositeDefinition<?>) theContext.getElementDefinition(theMeta.getClass());
+		if (theContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.R4)) {
+			return getSourceR4Plus(theContext, theMeta);
+		} else if (theContext.getVersion().getVersion().equals(FhirVersionEnum.DSTU3)) {
+			return getSourceDstu3((IBaseHasExtensions) theMeta);
+		} else {
+			throw new UnsupportedOperationException(MetaUtil.class.getSimpleName() + ".getSource() not supported on FHIR Version " + theContext.getVersion().getVersion());
+		}
+	}
+
+	private static String getSourceDstu3(IBaseHasExtensions theMeta) {
+		IBaseHasExtensions metaWithExtensions = theMeta;
+		List<? extends IBaseExtension<?, ?>> extensions = metaWithExtensions.getExtension();
+		for (IBaseExtension extension : extensions) {
+			if (Constants.EXT_META_SOURCE.equals(extension.getUrl())) {
+				IPrimitiveType<String> value = (IPrimitiveType<String>) extension.getValue();
+				return value.getValueAsString();
+			}
+		}
+		return null;
+	}
+
+	private static String getSourceR4Plus(FhirContext theFhirContext, IBaseMetaType theMeta) {
+		BaseRuntimeElementCompositeDefinition<?> elementDef = (BaseRuntimeElementCompositeDefinition<?>) theFhirContext.getElementDefinition(theMeta.getClass());
 		BaseRuntimeChildDefinition sourceChild = elementDef.getChildByName("source");
+		if (sourceChild == null) {
+			return null;
+		}
 		List<IBase> sourceValues = sourceChild.getAccessor().getValues(theMeta);
 		String retVal = null;
 		if (sourceValues.size() > 0) {
@@ -66,6 +95,8 @@ public class MetaUtil {
 			IPrimitiveType<String> value = (IPrimitiveType<String>) theContext.getElementDefinition("uri").newInstance();
 			value.setValue(theValue);
 			sourceExtension.setValue(value);
+		} else {
+			ourLog.error(MetaUtil.class.getSimpleName() + ".setSource() not supported on FHIR Version " + theContext.getVersion().getVersion());
 		}
 	}
 

hapi-fhir-base/src/main/resources/ca/uhn/fhir/i18n/hapi-messages.properties

@@ -1,4 +1,5 @@
 
+ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl.expansionRefersToUnknownCs=Unknown CodeSystem URI "{0}" referenced from ValueSet
 
 # Core Library Messages
 ca.uhn.fhir.context.FhirContext.unknownResourceName=Unknown resource name "{0}" (this name is not known in FHIR version "{1}")

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r4/JpaConformanceProviderR4.java

@@ -19,24 +19,25 @@ package ca.uhn.fhir.jpa.provider.r4;
  * limitations under the License.
  * #L%
  */
-import java.util.*;
-
-import javax.servlet.http.HttpServletRequest;
-
-import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
-import ca.uhn.fhir.rest.api.Constants;
-import ca.uhn.fhir.rest.api.server.RequestDetails;
-import org.hl7.fhir.r4.model.*;
-import org.hl7.fhir.r4.model.CapabilityStatement.*;
-import org.hl7.fhir.r4.model.Enumerations.SearchParamType;
 
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
 import ca.uhn.fhir.context.RuntimeSearchParam;
 import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.dao.IFhirSystemDao;
+import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
+import ca.uhn.fhir.rest.api.Constants;
+import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.server.RestfulServer;
 import ca.uhn.fhir.util.CoverageIgnore;
 import ca.uhn.fhir.util.ExtensionConstants;
+import org.hl7.fhir.r4.model.*;
+import org.hl7.fhir.r4.model.CapabilityStatement.*;
+import org.hl7.fhir.r4.model.Enumerations.SearchParamType;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
 
 import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
@@ -50,17 +51,17 @@ public class JpaConformanceProviderR4 extends org.hl7.fhir.r4.hapi.rest.server.S
 	private boolean myIncludeResourceCounts;
 	private RestfulServer myRestfulServer;
 	private IFhirSystemDao<Bundle, Meta> mySystemDao;
-	
+
 	/**
 	 * Constructor
 	 */
 	@CoverageIgnore
-	public JpaConformanceProviderR4(){
+	public JpaConformanceProviderR4() {
 		super();
 		super.setCache(false);
 		setIncludeResourceCounts(true);
 	}
-	
+
 	/**
 	 * Constructor
 	 */
@@ -94,7 +95,7 @@ public class JpaConformanceProviderR4 extends org.hl7.fhir.r4.hapi.rest.server.S
 			for (CapabilityStatementRestResourceComponent nextResource : nextRest.getResource()) {
 
 				nextResource.setVersioning(ResourceVersionPolicy.VERSIONEDUPDATE);
-				
+
 				ConditionalDeleteStatus conditionalDelete = nextResource.getConditionalDelete();
 				if (conditionalDelete == ConditionalDeleteStatus.MULTIPLE && myDaoConfig.isAllowMultipleDelete() == false) {
 					nextResource.setConditionalDelete(ConditionalDeleteStatus.SINGLE);
@@ -109,7 +110,7 @@ public class JpaConformanceProviderR4 extends org.hl7.fhir.r4.hapi.rest.server.S
 				nextResource.getSearchParam().clear();
 				String resourceName = nextResource.getType();
 				RuntimeResourceDefinition resourceDef = myRestfulServer.getFhirContext().getResourceDefinition(resourceName);
-				Collection<RuntimeSearchParam> searchParams =  mySearchParamRegistry.getSearchParamsByResourceType(resourceDef);
+				Collection<RuntimeSearchParam> searchParams = mySearchParamRegistry.getSearchParamsByResourceType(resourceDef);
 				for (RuntimeSearchParam runtimeSp : searchParams) {
 					CapabilityStatementRestResourceSearchParamComponent confSp = nextResource.addSearchParam();
 
@@ -117,37 +118,40 @@ public class JpaConformanceProviderR4 extends org.hl7.fhir.r4.hapi.rest.server.S
 					confSp.setDocumentation(runtimeSp.getDescription());
 					confSp.setDefinition(runtimeSp.getUri());
 					switch (runtimeSp.getParamType()) {
-					case COMPOSITE:
+						case COMPOSITE:
-						confSp.setType(SearchParamType.COMPOSITE);
+							confSp.setType(SearchParamType.COMPOSITE);
-						break;
+							break;
-					case DATE:
+						case DATE:
-						confSp.setType(SearchParamType.DATE);
+							confSp.setType(SearchParamType.DATE);
-						break;
+							break;
-					case NUMBER:
+						case NUMBER:
-						confSp.setType(SearchParamType.NUMBER);
+							confSp.setType(SearchParamType.NUMBER);
-						break;
+							break;
-					case QUANTITY:
+						case QUANTITY:
-						confSp.setType(SearchParamType.QUANTITY);
+							confSp.setType(SearchParamType.QUANTITY);
-						break;
+							break;
-					case REFERENCE:
+						case REFERENCE:
-						confSp.setType(SearchParamType.REFERENCE);
+							confSp.setType(SearchParamType.REFERENCE);
-						break;
+							break;
-					case STRING:
+						case STRING:
-						confSp.setType(SearchParamType.STRING);
+							confSp.setType(SearchParamType.STRING);
-						break;
+							break;
-					case TOKEN:
+						case TOKEN:
-						confSp.setType(SearchParamType.TOKEN);
+							confSp.setType(SearchParamType.TOKEN);
-						break;
+							break;
-					case URI:
+						case URI:
-						confSp.setType(SearchParamType.URI);
+							confSp.setType(SearchParamType.URI);
-						break;
+							break;
-					case HAS:
+						case SPECIAL:
-						// Shouldn't happen
+							confSp.setType(SearchParamType.SPECIAL);
-						break;
+							break;
+						case HAS:
+							// Shouldn't happen
+							break;
 					}
-					
+
 				}
-				
+
 			}
 		}
 
@@ -161,7 +165,7 @@ public class JpaConformanceProviderR4 extends org.hl7.fhir.r4.hapi.rest.server.S
 		}
 
 		massage(retVal);
-		
+
 		retVal.getImplementation().setDescription(myImplementationDescription);
 		myCachedValue = retVal;
 		return retVal;
@@ -170,7 +174,11 @@ public class JpaConformanceProviderR4 extends org.hl7.fhir.r4.hapi.rest.server.S
 	public boolean isIncludeResourceCounts() {
 		return myIncludeResourceCounts;
 	}
-	
+
+	public void setIncludeResourceCounts(boolean theIncludeResourceCounts) {
+		myIncludeResourceCounts = theIncludeResourceCounts;
+	}
+
 	/**
 	 * Subclasses may override
 	 */
@@ -187,10 +195,6 @@ public class JpaConformanceProviderR4 extends org.hl7.fhir.r4.hapi.rest.server.S
 		myImplementationDescription = theImplDesc;
 	}
 
-	public void setIncludeResourceCounts(boolean theIncludeResourceCounts) {
-		myIncludeResourceCounts = theIncludeResourceCounts;
-	}
-
 	@Override
 	public void setRestfulServer(RestfulServer theRestfulServer) {
 		this.myRestfulServer = theRestfulServer;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/r5/JpaConformanceProviderR5.java

@@ -51,16 +51,16 @@ public class JpaConformanceProviderR5 extends org.hl7.fhir.r5.hapi.rest.server.S
 	private boolean myIncludeResourceCounts;
 	private RestfulServer myRestfulServer;
 	private IFhirSystemDao<Bundle, Meta> mySystemDao;
-	
+
 	/**
 	 * Constructor
 	 */
 	@CoverageIgnore
-	public JpaConformanceProviderR5(){
+	public JpaConformanceProviderR5() {
 		super();
 		setIncludeResourceCounts(true);
 	}
-	
+
 	/**
 	 * Constructor
 	 */
@@ -93,7 +93,7 @@ public class JpaConformanceProviderR5 extends org.hl7.fhir.r5.hapi.rest.server.S
 			for (CapabilityStatementRestResourceComponent nextResource : nextRest.getResource()) {
 
 				nextResource.setVersioning(ResourceVersionPolicy.VERSIONEDUPDATE);
-				
+
 				ConditionalDeleteStatus conditionalDelete = nextResource.getConditionalDelete();
 				if (conditionalDelete == ConditionalDeleteStatus.MULTIPLE && myDaoConfig.isAllowMultipleDelete() == false) {
 					nextResource.setConditionalDelete(ConditionalDeleteStatus.SINGLE);
@@ -108,7 +108,7 @@ public class JpaConformanceProviderR5 extends org.hl7.fhir.r5.hapi.rest.server.S
 				nextResource.getSearchParam().clear();
 				String resourceName = nextResource.getType();
 				RuntimeResourceDefinition resourceDef = myRestfulServer.getFhirContext().getResourceDefinition(resourceName);
-				Collection<RuntimeSearchParam> searchParams =  mySearchParamRegistry.getSearchParamsByResourceType(resourceDef);
+				Collection<RuntimeSearchParam> searchParams = mySearchParamRegistry.getSearchParamsByResourceType(resourceDef);
 				for (RuntimeSearchParam runtimeSp : searchParams) {
 					CapabilityStatementRestResourceSearchParamComponent confSp = nextResource.addSearchParam();
 
@@ -116,37 +116,40 @@ public class JpaConformanceProviderR5 extends org.hl7.fhir.r5.hapi.rest.server.S
 					confSp.setDocumentation(runtimeSp.getDescription());
 					confSp.setDefinition(runtimeSp.getUri());
 					switch (runtimeSp.getParamType()) {
-					case COMPOSITE:
+						case COMPOSITE:
-						confSp.setType(SearchParamType.COMPOSITE);
+							confSp.setType(SearchParamType.COMPOSITE);
-						break;
+							break;
-					case DATE:
+						case DATE:
-						confSp.setType(SearchParamType.DATE);
+							confSp.setType(SearchParamType.DATE);
-						break;
+							break;
-					case NUMBER:
+						case NUMBER:
-						confSp.setType(SearchParamType.NUMBER);
+							confSp.setType(SearchParamType.NUMBER);
-						break;
+							break;
-					case QUANTITY:
+						case QUANTITY:
-						confSp.setType(SearchParamType.QUANTITY);
+							confSp.setType(SearchParamType.QUANTITY);
-						break;
+							break;
-					case REFERENCE:
+						case REFERENCE:
-						confSp.setType(SearchParamType.REFERENCE);
+							confSp.setType(SearchParamType.REFERENCE);
-						break;
+							break;
-					case STRING:
+						case STRING:
-						confSp.setType(SearchParamType.STRING);
+							confSp.setType(SearchParamType.STRING);
-						break;
+							break;
-					case TOKEN:
+						case TOKEN:
-						confSp.setType(SearchParamType.TOKEN);
+							confSp.setType(SearchParamType.TOKEN);
-						break;
+							break;
-					case URI:
+						case URI:
-						confSp.setType(SearchParamType.URI);
+							confSp.setType(SearchParamType.URI);
-						break;
+							break;
-					case HAS:
+						case SPECIAL:
-						// Shouldn't happen
+							confSp.setType(SearchParamType.SPECIAL);
-						break;
+							break;
+						case HAS:
+							// Shouldn't happen
+							break;
 					}
-					
+
 				}
-				
+
 			}
 		}
 
@@ -160,7 +163,7 @@ public class JpaConformanceProviderR5 extends org.hl7.fhir.r5.hapi.rest.server.S
 		}
 
 		massage(retVal);
-		
+
 		retVal.getImplementation().setDescription(myImplementationDescription);
 		myCachedValue = retVal;
 		return retVal;
@@ -169,7 +172,11 @@ public class JpaConformanceProviderR5 extends org.hl7.fhir.r5.hapi.rest.server.S
 	public boolean isIncludeResourceCounts() {
 		return myIncludeResourceCounts;
 	}
-	
+
+	public void setIncludeResourceCounts(boolean theIncludeResourceCounts) {
+		myIncludeResourceCounts = theIncludeResourceCounts;
+	}
+
 	/**
 	 * Subclasses may override
 	 */
@@ -186,10 +193,6 @@ public class JpaConformanceProviderR5 extends org.hl7.fhir.r5.hapi.rest.server.S
 		myImplementationDescription = theImplDesc;
 	}
 
-	public void setIncludeResourceCounts(boolean theIncludeResourceCounts) {
-		myIncludeResourceCounts = theIncludeResourceCounts;
-	}
-
 	@Override
 	public void setRestfulServer(RestfulServer theRestfulServer) {
 		this.myRestfulServer = theRestfulServer;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/reindex/ResourceReindexingSvcImpl.java

@@ -195,6 +195,11 @@ public class ResourceReindexingSvcImpl implements IResourceReindexingSvc {
 		mySchedulerService.scheduleFixedDelay(10 * DateUtils.MILLIS_PER_SECOND, true, jobDetail);
 	}
 
+	@VisibleForTesting
+	ReentrantLock getIndexingLockForUnitTest() {
+		return myIndexingLock;
+	}
+
 	@Override
 	@Transactional(Transactional.TxType.NEVER)
 	public Integer runReindexingPass() {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java

@@ -424,13 +424,13 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 
 	private void logConceptsExpanded(String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theConceptsExpanded) {
 		if (theConceptsExpanded > 0) {
-			ourLog.info("{}Have expanded {} concepts in ValueSet[{}]", theLogDescriptionPrefix, theConceptsExpanded, theTermValueSet.getUrl());
+			ourLog.debug("{}Have expanded {} concepts in ValueSet[{}]", theLogDescriptionPrefix, theConceptsExpanded, theTermValueSet.getUrl());
 		}
 	}
 
 	private void logDesignationsExpanded(String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theDesignationsExpanded) {
 		if (theDesignationsExpanded > 0) {
-			ourLog.info("{}Have expanded {} designations in ValueSet[{}]", theLogDescriptionPrefix, theDesignationsExpanded, theTermValueSet.getUrl());
+			ourLog.debug("{}Have expanded {} designations in ValueSet[{}]", theLogDescriptionPrefix, theDesignationsExpanded, theTermValueSet.getUrl());
 		}
 	}
 
@@ -446,7 +446,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 
 		StopWatch sw = new StopWatch();
 		String valueSetInfo = getValueSetInfo(theValueSetToExpand);
-		ourLog.info("Working with {}", valueSetInfo);
+		ourLog.debug("Working with {}", valueSetInfo);
 
 		// Handle includes
 		ourLog.debug("Handling includes");
@@ -488,7 +488,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 			myTxTemplate.execute(t -> ((ValueSetConceptAccumulator) theValueSetCodeAccumulator).removeGapsFromConceptOrder());
 		}
 
-		ourLog.info("Done working with {} in {}ms", valueSetInfo, sw.getMillis());
+		ourLog.debug("Done working with {} in {}ms", valueSetInfo, sw.getMillis());
 	}
 
 	private String getValueSetInfo(ValueSet theValueSet) {
@@ -553,7 +553,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 				return false;
 			}
 
-			ourLog.info("Starting {} expansion around CodeSystem: {}", (theAdd ? "inclusion" : "exclusion"), system);
+			ourLog.debug("Starting {} expansion around CodeSystem: {}", (theAdd ? "inclusion" : "exclusion"), system);
 
 			TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system);
 			if (cs != null) {
@@ -566,7 +566,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 				 * since we're going to do it without the database.
 				 */
 				if (myFulltextSearchSvc == null) {
-					expandWithoutHibernateSearch(theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, system, theAdd, theCodeCounter);
+					expandWithoutHibernateSearch(theValueSetCodeAccumulator, csv, theAddedCodes, theIncludeOrExclude, system, theAdd, theCodeCounter);
 					return false;
 				}
 
@@ -642,7 +642,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 				jpaQuery.setMaxResults(maxResultsPerBatch);
 				jpaQuery.setFirstResult(theQueryIndex * maxResultsPerBatch);
 
-				ourLog.info("Beginning batch expansion for {} with max results per batch: {}", (theAdd ? "inclusion" : "exclusion"), maxResultsPerBatch);
+				ourLog.debug("Beginning batch expansion for {} with max results per batch: {}", (theAdd ? "inclusion" : "exclusion"), maxResultsPerBatch);
 
 				StopWatch swForBatch = new StopWatch();
 				AtomicInteger countForBatch = new AtomicInteger(0);
@@ -661,10 +661,10 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 					}
 				}
 
-				ourLog.info("Batch expansion for {} with starting index of {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), firstResult, countForBatch, swForBatch.getMillis());
+				ourLog.debug("Batch expansion for {} with starting index of {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), firstResult, countForBatch, swForBatch.getMillis());
 
 				if (resultsInBatch < maxResultsPerBatch) {
-					ourLog.info("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis());
+					ourLog.debug("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis());
 					return false;
 				} else {
 					return true;
@@ -675,7 +675,10 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 
 				CodeSystem codeSystemFromContext = getCodeSystemFromContext(system);
 				if (codeSystemFromContext == null) {
-					throw new InvalidRequestException("Unknown code system: " + system);
+					String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionRefersToUnknownCs", system);
+					ourLog.warn(msg);
+					theValueSetCodeAccumulator.addMessage(msg);
+					return false;
 				}
 
 				if (!theIncludeOrExclude.getConcept().isEmpty()) {
@@ -709,7 +712,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 		} else if (hasValueSet) {
 
 			for (CanonicalType nextValueSet : theIncludeOrExclude.getValueSet()) {
-				ourLog.info("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), nextValueSet.getValueAsString());
+				ourLog.debug("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), nextValueSet.getValueAsString());
 
 				List<VersionIndependentConcept> expanded = expandValueSet(nextValueSet.getValueAsString());
 				Map<String, TermCodeSystem> uriToCodeSystem = new HashMap<>();
@@ -845,7 +848,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 			.orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + theSystem + "}" + theFilter.getValue()));
 
 		if (theFilter.getOp() == ValueSet.FilterOperator.ISA) {
-			ourLog.info(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
+			ourLog.debug(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
 			theBool.must(theQb.keyword().onField("myParentPids").matching("" + code.getId()).createQuery());
 		} else {
 			throw new InvalidRequestException("Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
@@ -1012,7 +1015,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 	}
 
 	private void logFilteringValueOnProperty(String theValue, String theProperty) {
-		ourLog.info(" * Filtering with value={} on property {}", theValue, theProperty);
+		ourLog.debug(" * Filtering with value={} on property {}", theValue, theProperty);
 	}
 
 	private void throwInvalidRequestForOpOnProperty(ValueSet.FilterOperator theOp, String theProperty) {
@@ -1057,7 +1060,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 		}
 	}
 
-	private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, AtomicInteger theCodeCounter) {
+	private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, TermCodeSystemVersion theVersion, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, AtomicInteger theCodeCounter) {
 		ourLog.trace("Hibernate search is not enabled");
 		if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) {
 			Validate.isTrue(((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator).getParameter().isEmpty(), "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server.");
@@ -1065,12 +1068,21 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 		Validate.isTrue(theInclude.getFilter().isEmpty(), "Can not expand ValueSet with filters - Hibernate Search is not enabled on this server.");
 		Validate.isTrue(isNotBlank(theSystem), "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server.");
 
+
+		if (theInclude.getConcept().isEmpty()) {
+			for (TermConcept next : theVersion.getConcepts()) {
+				addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, null, theAdd, theCodeCounter, theSystem, next.getCode(), next.getDisplay());
+			}
+		}
+
 		for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) {
 			if (!theSystem.equals(theInclude.getSystem())) {
 				continue;
 			}
 			addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, null, theAdd, theCodeCounter, theSystem, next.getCode(), next.getDisplay());
 		}
+
+
 	}
 
 	@Override
@@ -1212,7 +1224,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 
 		fetchParents(concept.get(), retVal);
 
-		ourLog.info("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis());
+		ourLog.debug("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis());
 		return retVal;
 	}
 
@@ -1243,7 +1255,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 
 		fetchChildren(concept.get(), retVal);
 
-		ourLog.info("Fetched {} codes below code {} in {}ms", retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS));
+		ourLog.debug("Fetched {} codes below code {} in {}ms", retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS));
 		return retVal;
 	}
 
@@ -1292,7 +1304,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 	@Override
 	@Transactional
 	public void storeTermConceptMapAndChildren(ResourceTable theResourceTable, ConceptMap theConceptMap) {
-		ourLog.info("Storing TermConceptMap for {}", theConceptMap.getIdElement().toVersionless().getValueAsString());
+		ourLog.debug("Storing TermConceptMap for {}", theConceptMap.getIdElement().toVersionless().getValueAsString());
 
 		ValidateUtil.isTrueOrThrowInvalidRequest(theResourceTable != null, "No resource supplied");
 		ValidateUtil.isNotBlankOrThrowUnprocessableEntity(theConceptMap.getUrl(), "ConceptMap has no value for ConceptMap.url");

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetConceptAccumulator.java

@@ -27,6 +27,8 @@ import java.util.Collection;
 
 public interface IValueSetConceptAccumulator {
 
+	void addMessage(String theMessage);
+
 	void includeConcept(String theSystem, String theCode, String theDisplay);
 
 	void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, Collection<TermConceptDesignation> theDesignations);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcImpl.java

@@ -335,7 +335,7 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 				populateCodeSystemVersionProperties(persCs, theCodeSystem, theResourceEntity);
 
 				persCs.getConcepts().addAll(BaseTermReadSvcImpl.toPersistedConcepts(theCodeSystem.getConcept(), persCs));
-				ourLog.info("Code system has {} concepts", persCs.getConcepts().size());
+				ourLog.debug("Code system has {} concepts", persCs.getConcepts().size());
 				storeNewCodeSystemVersion(codeSystemResourcePid, codeSystemUrl, theCodeSystem.getName(), theCodeSystem.getVersion(), persCs, theResourceEntity);
 			}
 
@@ -366,7 +366,7 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 	@Override
 	@Transactional(propagation = Propagation.REQUIRED)
 	public void storeNewCodeSystemVersion(Long theCodeSystemResourcePid, String theSystemUri, String theSystemName, String theSystemVersionId, TermCodeSystemVersion theCodeSystemVersion, ResourceTable theCodeSystemResourceTable) {
-		ourLog.info("Storing code system");
+		ourLog.debug("Storing code system");
 
 		ValidateUtil.isTrueOrThrowInvalidRequest(theCodeSystemVersion.getResource() != null, "No resource supplied");
 		ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystemUri, "No system URI supplied");
@@ -378,15 +378,15 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 		 * For now we always delete old versions. At some point it would be nice to allow configuration to keep old versions.
 		 */
 
-		ourLog.info("Deleting old code system versions");
 		for (TermCodeSystemVersion next : existing) {
+			ourLog.info("Deleting old code system version {}", next.getPid());
 			Long codeSystemVersionPid = next.getPid();
 			deleteCodeSystemVersion(codeSystemVersionPid);
 		}
 
-		ourLog.info("Flushing...");
+		ourLog.debug("Flushing...");
 		myConceptDao.flush();
-		ourLog.info("Done flushing");
+		ourLog.debug("Done flushing");
 
 		/*
 		 * Do the upload
@@ -399,7 +399,7 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 		theCodeSystemVersion.setCodeSystemDisplayName(theSystemName);
 		theCodeSystemVersion.setCodeSystemVersionId(theSystemVersionId);
 
-		ourLog.info("Validating all codes in CodeSystem for storage (this can take some time for large sets)");
+		ourLog.debug("Validating all codes in CodeSystem for storage (this can take some time for large sets)");
 
 		// Validate the code system
 		ArrayList<String> conceptsStack = new ArrayList<>();
@@ -409,35 +409,33 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 			totalCodeCount += validateConceptForStorage(next, theCodeSystemVersion, conceptsStack, allConcepts);
 		}
 
-		ourLog.info("Saving version containing {} concepts", totalCodeCount);
+		ourLog.debug("Saving version containing {} concepts", totalCodeCount);
 
 		TermCodeSystemVersion codeSystemVersion = myCodeSystemVersionDao.saveAndFlush(theCodeSystemVersion);
 
-		ourLog.info("Saving code system");
+		ourLog.debug("Saving code system");
 
 		codeSystem.setCurrentVersion(theCodeSystemVersion);
 		codeSystem = myCodeSystemDao.saveAndFlush(codeSystem);
 
-		ourLog.info("Setting CodeSystemVersion[{}] on {} concepts...", codeSystem.getPid(), totalCodeCount);
+		ourLog.debug("Setting CodeSystemVersion[{}] on {} concepts...", codeSystem.getPid(), totalCodeCount);
 
 		for (TermConcept next : theCodeSystemVersion.getConcepts()) {
 			populateVersion(next, codeSystemVersion);
 		}
 
-		ourLog.info("Saving {} concepts...", totalCodeCount);
+		ourLog.debug("Saving {} concepts...", totalCodeCount);
 
 		IdentityHashMap<TermConcept, Object> conceptsStack2 = new IdentityHashMap<>();
 		for (TermConcept next : theCodeSystemVersion.getConcepts()) {
 			persistChildren(next, codeSystemVersion, conceptsStack2, totalCodeCount);
 		}
 
-		ourLog.info("Done saving concepts, flushing to database");
+		ourLog.debug("Done saving concepts, flushing to database");
 
 		myConceptDao.flush();
 		myConceptParentChildLinkDao.flush();
 
-		ourLog.info("Done deleting old code system versions");
-
 		if (myDeferredStorageSvc.isStorageQueueEmpty() == false) {
 			ourLog.info("Note that some concept saving has been deferred");
 		}
@@ -572,7 +570,7 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 		// case for concepts being added to an existing child concept, but won't be the case when
 		// we're recursively adding children)
 		for (TermConcept nextParentConcept : parentConcepts) {
-			if (nextParentConcept.getChildren().stream().noneMatch(t->t.getChild().getCode().equals(nextCodeToAdd))) {
+			if (nextParentConcept.getChildren().stream().noneMatch(t -> t.getChild().getCode().equals(nextCodeToAdd))) {
 				TermConceptParentChildLink parentLink = new TermConceptParentChildLink();
 				parentLink.setParent(nextParentConcept);
 				parentLink.setChild(nextConceptToAdd);
@@ -612,7 +610,7 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 			return;
 		}
 
-		if (theConceptsStack.size() == 1 || theConceptsStack.size() % 10000 == 0) {
+		if ((theConceptsStack.size() + 1) % 10000 == 0) {
 			float pct = (float) theConceptsStack.size() / (float) theTotalConcepts;
 			ourLog.info("Have processed {}/{} concepts ({}%)", theConceptsStack.size(), theTotalConcepts, (int) (pct * 100.0f));
 		}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java

@@ -57,6 +57,12 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 		myConceptsExcluded = 0;
 	}
 
+	@Override
+	public void addMessage(String theMessage) {
+		// ignore for now
+
+	}
+
 	@Override
 	public void includeConcept(String theSystem, String theCode, String theDisplay) {
 		saveConcept(theSystem, theCode, theDisplay);
@@ -82,7 +88,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 		if (optionalConcept.isPresent()) {
 			TermValueSetConcept concept = optionalConcept.get();
 
-			ourLog.info("Excluding [{}|{}] from ValueSet[{}]", concept.getSystem(), concept.getCode(), myTermValueSet.getUrl());
+			ourLog.debug("Excluding [{}|{}] from ValueSet[{}]", concept.getSystem(), concept.getCode(), myTermValueSet.getUrl());
 			for (TermValueSetConceptDesignation designation : concept.getDesignations()) {
 				myValueSetConceptDesignationDao.deleteById(designation.getId());
 				myTermValueSet.decrementTotalConceptDesignations();
@@ -90,7 +96,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 			myValueSetConceptDao.deleteById(concept.getId());
 			myTermValueSet.decrementTotalConcepts();
 			myValueSetDao.save(myTermValueSet);
-			ourLog.info("Done excluding [{}|{}] from ValueSet[{}]", concept.getSystem(), concept.getCode(), myTermValueSet.getUrl());
+			ourLog.debug("Done excluding [{}|{}] from ValueSet[{}]", concept.getSystem(), concept.getCode(), myTermValueSet.getUrl());
 
 			if (++myConceptsExcluded % 250 == 0) {
 				ourLog.info("Have excluded {} concepts from ValueSet[{}]", myConceptsExcluded, myTermValueSet.getUrl());

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithConceptAccumulator.java

@@ -22,9 +22,11 @@ package ca.uhn.fhir.jpa.term;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.term.ex.ExpansionTooCostlyException;
 import ca.uhn.fhir.model.api.annotation.Block;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import org.hl7.fhir.r4.model.StringType;
 import org.hl7.fhir.r4.model.ValueSet;
 
 import javax.annotation.Nullable;
@@ -54,6 +56,13 @@ public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.V
 		return myMaxCapacity - myConceptsCount;
 	}
 
+	@Override
+	public void addMessage(String theMessage) {
+		addExtension()
+			.setUrl(JpaConstants.EXT_VALUESET_EXPANSION_MESSAGE)
+			.setValue(new StringType(theMessage));
+	}
+
 	@Override
 	public void includeConcept(String theSystem, String theCode, String theDisplay) {
 		incrementConceptsCount();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java

@@ -89,6 +89,33 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test {
 
 	}
 
+
+	@Test
+	public void testValidateQuestionnaireResponseWithValueSetIncludingCompleteCodeSystem() throws IOException {
+		CodeSystem cs = loadResourceFromClasspath(CodeSystem.class, "/dstu3/iar/CodeSystem-iar-citizenship-status.xml");
+		myCodeSystemDao.create(cs);
+
+		ValueSet vs = loadResourceFromClasspath(ValueSet.class, "/dstu3/iar/ValueSet-iar-citizenship-status.xml");
+		myValueSetDao.create(vs);
+
+		ValueSet expansion = myValueSetDao.expandByIdentifier("http://ccim.on.ca/fhir/iar/ValueSet/iar-citizenship-status", null);
+			ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
+
+//		Questionnaire q = loadResourceFromClasspath(Questionnaire.class,"/dstu3/iar/Questionnaire-iar-test.xml" );
+//		myQuestionnaireDao.create(q);
+//
+//
+//
+//		Bundle bundleForValidation = loadResourceFromClasspath(Bundle.class, "/dstu3/iar/Bundle-for-validation.xml");
+//		try {
+//			MethodOutcome outcome = myBundleDao.validate(bundleForValidation, null, null, null, null, null, null);
+//			ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome.getOperationOutcome()));
+//		} catch (PreconditionFailedException e) {
+//			ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome()));
+//		}
+	}
+
+
 	@After
 	public void after() {
 		FhirInstanceValidator val = AopTestUtils.getTargetObject(myValidatorModule);

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithLuceneDisabledTest.java

@@ -4,17 +4,21 @@ import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.bulk.IBulkDataExportSvc;
 import ca.uhn.fhir.jpa.config.TestR4WithLuceneDisabledConfig;
 import ca.uhn.fhir.jpa.dao.*;
+import ca.uhn.fhir.jpa.dao.dstu2.FhirResourceDaoDstu2SearchNoFtTest;
 import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc;
 import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
-import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
+import ca.uhn.fhir.parser.IParser;
+import ca.uhn.fhir.rest.api.EncodingEnum;
 import ca.uhn.fhir.rest.api.server.IBundleProvider;
 import ca.uhn.fhir.rest.param.StringParam;
 import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.param.TokenParamModifier;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.util.TestUtil;
+import org.apache.commons.io.IOUtils;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r4.hapi.ctx.IValidationSupport;
@@ -31,11 +35,13 @@ import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.transaction.annotation.Transactional;
 
 import javax.persistence.EntityManager;
-
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
-import static org.mockito.ArgumentMatchers.any;
+import static org.junit.Assert.fail;
 
 @RunWith(SpringJUnit4ClassRunner.class)
 @ContextConfiguration(classes = {TestR4WithLuceneDisabledConfig.class})
@@ -71,7 +77,7 @@ public class FhirResourceDaoR4SearchWithLuceneDisabledTest extends BaseJpaTest {
 	private IFhirResourceDao<CodeSystem> myCodeSystemDao;
 	@Autowired
 	@Qualifier("myValueSetDaoR4")
-	private IFhirResourceDao<ValueSet> myValueSetDao;
+	private IFhirResourceDaoValueSet<ValueSet, ?, ?> myValueSetDao;
 	@Autowired
 	@Qualifier("myObservationDaoR4")
 	private IFhirResourceDao<Observation> myObservationDao;
@@ -219,6 +225,34 @@ public class FhirResourceDaoR4SearchWithLuceneDisabledTest extends BaseJpaTest {
 
 	}
 
+	protected <T extends IBaseResource> T loadResourceFromClasspath(Class<T> type, String resourceName) throws IOException {
+		InputStream stream = FhirResourceDaoDstu2SearchNoFtTest.class.getResourceAsStream(resourceName);
+		if (stream == null) {
+			fail("Unable to load resource: " + resourceName);
+		}
+		String string = IOUtils.toString(stream, StandardCharsets.UTF_8);
+		IParser newJsonParser = EncodingEnum.detectEncodingNoDefault(string).newParser(myFhirCtx);
+		return newJsonParser.parseResource(type, string);
+	}
+
+
+	/**
+	 * A valueset that includes a whole system (i.e. no properties) should expand
+	 */
+	@Test
+	public void testExpandValueSetContainingSystemIncludeWithNoCodes() throws IOException {
+		CodeSystem cs = loadResourceFromClasspath(CodeSystem.class, "/dstu3/iar/CodeSystem-iar-citizenship-status.xml");
+		myCodeSystemDao.create(cs);
+
+		ValueSet vs = loadResourceFromClasspath(ValueSet.class, "/dstu3/iar/ValueSet-iar-citizenship-status.xml");
+		myValueSetDao.create(vs);
+
+		ValueSet expansion = myValueSetDao.expandByIdentifier("http://ccim.on.ca/fhir/iar/ValueSet/iar-citizenship-status", null);
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(expansion));
+
+		assertEquals(4, expansion.getExpansion().getContains().size());
+
+	}
 
 
 	@AfterClass

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java

@@ -730,6 +730,8 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
 		assertEquals("The questionnaire \"Questionnaire/DOES_NOT_EXIST\" could not be resolved, so no validation can be performed against the base questionnaire", oo.getIssueFirstRep().getDiagnostics());
 	}
 
+
+
 	private IBaseResource findResourceByIdInBundle(Bundle vss, String name) {
 		IBaseResource retVal = null;
 		for (BundleEntryComponent next : vss.getEntry()) {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoR5SearchNoFtTest.java

@@ -1,5 +1,7 @@
 package ca.uhn.fhir.jpa.dao.r5;
 
+import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
+import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.util.TestUtil;
 import ca.uhn.fhir.rest.api.Constants;
@@ -9,11 +11,14 @@ import ca.uhn.fhir.rest.param.HasOrListParam;
 import ca.uhn.fhir.rest.param.HasParam;
 import ca.uhn.fhir.rest.param.StringParam;
 import org.hl7.fhir.r5.model.Organization;
+import org.hl7.fhir.r5.model.Patient;
 import org.hl7.fhir.r5.model.Practitioner;
 import org.hl7.fhir.r5.model.PractitionerRole;
 import org.junit.AfterClass;
 import org.junit.Test;
 
+import java.util.Date;
+
 import static org.junit.Assert.assertEquals;
 
 @SuppressWarnings({"unchecked", "Duplicates"})
@@ -98,6 +103,40 @@ public class FhirResourceDaoR5SearchNoFtTest extends BaseJpaR5Test {
 		assertEquals(1, outcome.getResources(0, 1).size());
 	}
 
+	@Test
+	public void testSearchDoesntFailIfResourcesAreDeleted() {
+
+		Patient p = new Patient();
+		p.addIdentifier().setValue("1");
+		myPatientDao.create(p);
+
+		p = new Patient();
+		p.addIdentifier().setValue("2");
+		myPatientDao.create(p);
+
+		p = new Patient();
+		p.addIdentifier().setValue("3");
+		Long id = myPatientDao.create(p).getId().getIdPartAsLong();
+
+		IBundleProvider outcome = myPatientDao.search(new SearchParameterMap());
+		assertEquals(3, outcome.size().intValue());
+
+		runInTransaction(()->{
+			ResourceTable table = myResourceTableDao.findById(id).orElseThrow(() -> new IllegalArgumentException());
+			table.setDeleted(new Date());
+			myResourceTableDao.save(table);
+		});
+
+		assertEquals(2, outcome.getResources(0, 3).size());
+
+		runInTransaction(()->{
+			myResourceHistoryTableDao.deleteAll();
+		});
+
+		assertEquals(0, outcome.getResources(0, 3).size());
+	}
+
+
 	@AfterClass
 	public static void afterClassClearContext() {
 		TestUtil.clearAllStaticFieldsForUnitTest();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java

@@ -68,7 +68,7 @@ public abstract class BaseResourceProviderR4Test extends BaseJpaR4Test {
 	private static SubscriptionMatcherInterceptor ourSubscriptionMatcherInterceptor;
 	protected static Server ourServer;
 	protected IGenericClient ourClient;
-	ResourceCountCache ourResourceCountsCache;
+	ResourceCountCache myResourceCountsCache;
 	private TerminologyUploaderProvider myTerminologyUploaderProvider;
 	private boolean ourRestHookSubscriptionInterceptorRequested;
 
@@ -93,6 +93,7 @@ public abstract class BaseResourceProviderR4Test extends BaseJpaR4Test {
 		myFhirCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
 		myFhirCtx.getRestfulClientFactory().setSocketTimeout(1200 * 1000);
 		myFhirCtx.setParserErrorHandler(new StrictErrorHandler());
+		myResourceCountsCache = (ResourceCountCache) myAppCtx.getBean("myResourceCountsCache");
 
 		if (ourServer == null) {
 			ourRestServer = new RestfulServer(myFhirCtx);
@@ -113,7 +114,6 @@ public abstract class BaseResourceProviderR4Test extends BaseJpaR4Test {
 			ourRestServer.setServerConformanceProvider(confProvider);
 
 			ourPagingProvider = myAppCtx.getBean(DatabaseBackedPagingProvider.class);
-			ourResourceCountsCache = (ResourceCountCache) myAppCtx.getBean("myResourceCountsCache");
 
 			Server server = new Server(0);
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ServerR4Test.java

@@ -1,6 +1,9 @@
 package ca.uhn.fhir.jpa.provider.r4;
 
+import ca.uhn.fhir.jpa.dao.IFhirResourceDao;
+import ca.uhn.fhir.rest.api.EncodingEnum;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
 import ca.uhn.fhir.util.ExtensionConstants;
 import ca.uhn.fhir.util.TestUtil;
 import org.apache.commons.io.IOUtils;
@@ -13,6 +16,7 @@ import org.hl7.fhir.r4.model.Extension;
 import org.hl7.fhir.r4.model.Patient;
 import org.junit.AfterClass;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -25,6 +29,29 @@ public class ServerR4Test extends BaseResourceProviderR4Test {
 
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ServerR4Test.class);
 
+	@Autowired
+	private IFhirResourceDao<CapabilityStatement> myCapabilityStatementDao;
+
+	@Test
+	public void testCapabilityStatementValidates() throws IOException {
+		HttpGet get = new HttpGet(ourServerBase + "/metadata?_pretty=true&_format=json");
+		try (CloseableHttpResponse resp = ourHttpClient.execute(get)) {
+			assertEquals(200, resp.getStatusLine().getStatusCode());
+			String respString = IOUtils.toString(resp.getEntity().getContent(), StandardCharsets.UTF_8);
+
+			ourLog.info(respString);
+
+			CapabilityStatement cs = myFhirCtx.newJsonParser().parseResource(CapabilityStatement.class, respString);
+
+			try {
+				myCapabilityStatementDao.validate(cs, null, respString, EncodingEnum.JSON, null, null, null);
+			} catch (PreconditionFailedException e) {
+				ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome()));
+				fail();
+			}
+		}
+	}
+
 
 	/**
 	 * See #519
@@ -71,7 +98,7 @@ public class ServerR4Test extends BaseResourceProviderR4Test {
 		 * Initial fetch after a clear should return
 		 * no results
 		 */
-		ourResourceCountsCache.clear();
+		myResourceCountsCache.clear();
 
 		CapabilityStatement capabilityStatement = ourClient
 			.capabilities()
@@ -93,7 +120,7 @@ public class ServerR4Test extends BaseResourceProviderR4Test {
 		 * Now run a background pass (the update
 		 * method is called by the scheduler normally)
 		 */
-		ourResourceCountsCache.update();
+		myResourceCountsCache.update();
 
 		capabilityStatement = ourClient
 			.capabilities()

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ServerR5Test.java

@@ -0,0 +1,151 @@
+package ca.uhn.fhir.jpa.provider.r5;
+
+import ca.uhn.fhir.jpa.dao.IFhirResourceDao;
+import ca.uhn.fhir.rest.api.EncodingEnum;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
+import ca.uhn.fhir.util.ExtensionConstants;
+import ca.uhn.fhir.util.TestUtil;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.hl7.fhir.r5.model.CapabilityStatement;
+import org.hl7.fhir.r5.model.CapabilityStatement.CapabilityStatementRestResourceComponent;
+import org.hl7.fhir.r5.model.CapabilityStatement.CapabilityStatementRestResourceSearchParamComponent;
+import org.hl7.fhir.r5.model.Extension;
+import org.hl7.fhir.r5.model.Patient;
+import org.junit.AfterClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.Assert.*;
+
+public class ServerR5Test extends BaseResourceProviderR5Test {
+
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ServerR5Test.class);
+
+	@Autowired
+	private IFhirResourceDao<CapabilityStatement> myCapabilityStatementDao;
+
+	@Test
+	@Ignore
+	public void testCapabilityStatementValidates() throws IOException {
+		HttpGet get = new HttpGet(ourServerBase + "/metadata?_pretty=true&_format=json");
+		try (CloseableHttpResponse resp = ourHttpClient.execute(get)) {
+			assertEquals(200, resp.getStatusLine().getStatusCode());
+			String respString = IOUtils.toString(resp.getEntity().getContent(), StandardCharsets.UTF_8);
+
+			ourLog.info(respString);
+
+			CapabilityStatement cs = myFhirCtx.newJsonParser().parseResource(CapabilityStatement.class, respString);
+
+			try {
+				myCapabilityStatementDao.validate(cs, null, respString, EncodingEnum.JSON, null, null, null);
+			} catch (PreconditionFailedException e) {
+				ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(e.getOperationOutcome()));
+				fail();
+			}
+		}
+	}
+
+
+	/**
+	 * See #519
+	 */
+	@Test
+	public void saveIdParamOnlyAppearsOnce() throws IOException {
+		HttpGet get = new HttpGet(ourServerBase + "/metadata?_pretty=true&_format=xml");
+		CloseableHttpResponse resp = ourHttpClient.execute(get);
+		try {
+			ourLog.info(resp.toString());
+			assertEquals(200, resp.getStatusLine().getStatusCode());
+
+			String respString = IOUtils.toString(resp.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(respString);
+
+			CapabilityStatement cs = myFhirCtx.newXmlParser().parseResource(CapabilityStatement.class, respString);
+
+			for (CapabilityStatementRestResourceComponent nextResource : cs.getRest().get(0).getResource()) {
+				ourLog.info("Testing resource: " + nextResource.getType());
+				Set<String> sps = new HashSet<String>();
+				for (CapabilityStatementRestResourceSearchParamComponent nextSp : nextResource.getSearchParam()) {
+					if (sps.add(nextSp.getName()) == false) {
+						fail("Duplicate search parameter " + nextSp.getName() + " for resource " + nextResource.getType());
+					}
+				}
+
+				if (!sps.contains("_id")) {
+					fail("No search parameter _id for resource " + nextResource.getType());
+				}
+			}
+		} finally {
+			IOUtils.closeQuietly(resp.getEntity().getContent());
+		}
+	}
+
+
+	@Test
+	public void testMetadataIncludesResourceCounts() {
+		Patient p = new Patient();
+		p.setActive(true);
+		ourClient.create().resource(p).execute();
+
+		/*
+		 * Initial fetch after a clear should return
+		 * no results
+		 */
+		myResourceCountsCache.clear();
+
+		CapabilityStatement capabilityStatement = ourClient
+			.capabilities()
+			.ofType(CapabilityStatement.class)
+			.execute();
+
+		Extension patientCountExt = capabilityStatement
+			.getRest()
+			.get(0)
+			.getResource()
+			.stream()
+			.filter(t -> t.getType().equals("Patient"))
+			.findFirst()
+			.orElseThrow(() -> new InternalErrorException("No patient"))
+			.getExtensionByUrl(ExtensionConstants.CONF_RESOURCE_COUNT);
+		assertNull(patientCountExt);
+
+		/*
+		 * Now run a background pass (the update
+		 * method is called by the scheduler normally)
+		 */
+		myResourceCountsCache.update();
+
+		capabilityStatement = ourClient
+			.capabilities()
+			.ofType(CapabilityStatement.class)
+			.execute();
+
+		patientCountExt = capabilityStatement
+			.getRest()
+			.get(0)
+			.getResource()
+			.stream()
+			.filter(t -> t.getType().equals("Patient"))
+			.findFirst()
+			.orElseThrow(() -> new InternalErrorException("No patient"))
+			.getExtensionByUrl(ExtensionConstants.CONF_RESOURCE_COUNT);
+		assertEquals("1", patientCountExt.getValueAsPrimitive().getValueAsString());
+
+	}
+
+
+	@AfterClass
+	public static void afterClassClearContext() {
+		TestUtil.clearAllStaticFieldsForUnitTest();
+	}
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/reindex/ResourceReindexingSvcImplTest.java

@@ -26,6 +26,8 @@ import org.springframework.data.domain.SliceImpl;
 import org.springframework.transaction.PlatformTransactionManager;
 
 import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
@@ -94,6 +96,17 @@ public class ResourceReindexingSvcImplTest extends BaseJpaTest {
 		mySvc.start();
 	}
 
+	@Test
+	public void testNoParallelReindexing() throws InterruptedException {
+		CountDownLatch latch = new CountDownLatch(1);
+		new Thread(()->{
+			mySvc.getIndexingLockForUnitTest().lock();
+			latch.countDown();
+		}).start();
+		latch.await(10, TimeUnit.SECONDS);
+		mySvc.runReindexingPass();
+	}
+
 	@Test
 	public void testReindexPassOnlyReturnsValuesAtLowThreshold() {
 		mockNothingToExpunge();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/resthook/RestHookTestDstu3Test.java

@@ -18,6 +18,7 @@ import ca.uhn.fhir.rest.server.IResourceProvider;
 import ca.uhn.fhir.rest.server.RestfulServer;
 import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
 import ca.uhn.fhir.test.utilities.JettyUtil;
+import ca.uhn.fhir.util.MetaUtil;
 import com.google.common.collect.Lists;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.servlet.ServletContextHandler;
@@ -242,6 +243,33 @@ public class RestHookTestDstu3Test extends BaseResourceProviderDstu3Test {
 		assertEquals(Constants.CT_FHIR_JSON_NEW, ourContentTypes.get(0));
 	}
 
+
+	@Test
+	public void testRestHookSubscriptionSource() throws Exception {
+		String payload = "application/fhir+json";
+
+		String source = "foosource";
+		String criteria = "Observation?_source=" + source;
+
+		Subscription subscription = newSubscription(criteria, payload, ourListenerServerBase, null);
+		MethodOutcome methodOutcome = ourClient.create().resource(subscription).execute();
+		Subscription savedSub = (Subscription) methodOutcome.getResource();
+		assertInMemoryTag(savedSub);
+		mySubscriptionIds.add(methodOutcome.getId());
+
+		waitForQueueToDrain();
+
+		Observation observation = new Observation();
+		MetaUtil.setSource(myFhirCtx, observation, source);
+		ourClient.create().resource(observation).execute();
+
+		// Should see 1 subscription notification
+		waitForQueueToDrain();
+		waitForSize(0, ourCreatedObservations);
+		waitForSize(1, ourUpdatedObservations);
+		assertEquals(Constants.CT_FHIR_JSON_NEW, ourContentTypes.get(0));
+	}
+
 	@Test
 	public void testRestHookSubscriptionApplicationJson() throws Exception {
 		String payload = "application/json";
@@ -474,11 +502,17 @@ public class RestHookTestDstu3Test extends BaseResourceProviderDstu3Test {
 		assertEquals("In-memory", tag.getDisplay());
 
 		// Wait for subscription to be moved to active
-		await().until(()-> Subscription.SubscriptionStatus.ACTIVE.equals(ourClient.read().resource(Subscription.class).withId(subscriptionId.toUnqualifiedVersionless()).execute().getStatus()));
+		await().until(() -> Subscription.SubscriptionStatus.ACTIVE.equals(ourClient.read().resource(Subscription.class).withId(subscriptionId.toUnqualifiedVersionless()).execute().getStatus()));
 
 		Subscription subscriptionActivated = ourClient.read().resource(Subscription.class).withId(subscriptionId.toUnqualifiedVersionless()).execute();
 		assertEquals(Subscription.SubscriptionStatus.ACTIVE, subscriptionActivated.getStatus());
-		tags = subscriptionActivated.getMeta().getTag();
+		assertInMemoryTag(subscriptionActivated);
+	}
+
+	private void assertInMemoryTag(Subscription theSubscription) {
+		List<Coding> tags;
+		Coding tag;
+		tags = theSubscription.getMeta().getTag();
 		assertEquals(1, tags.size());
 		tag = tags.get(0);
 		assertEquals(JpaConstants.EXT_SUBSCRIPTION_MATCHING_STRATEGY, tag.getSystem());
@@ -501,7 +535,7 @@ public class RestHookTestDstu3Test extends BaseResourceProviderDstu3Test {
 		assertEquals("Database", tag.getDisplay());
 
 		// Wait for subscription to be moved to active
-		await().until(()-> Subscription.SubscriptionStatus.ACTIVE.equals(ourClient.read().resource(Subscription.class).withId(subscriptionId.toUnqualifiedVersionless()).execute().getStatus()));
+		await().until(() -> Subscription.SubscriptionStatus.ACTIVE.equals(ourClient.read().resource(Subscription.class).withId(subscriptionId.toUnqualifiedVersionless()).execute().getStatus()));
 
 		Subscription subscription = ourClient.read().resource(Subscription.class).withId(subscriptionId.toUnqualifiedVersionless()).execute();
 		assertEquals(Subscription.SubscriptionStatus.ACTIVE, subscription.getStatus());
@@ -624,9 +658,9 @@ public class RestHookTestDstu3Test extends BaseResourceProviderDstu3Test {
 
 		ourListenerServer.setHandler(proxyHandler);
 		JettyUtil.startServer(ourListenerServer);
-        ourListenerPort = JettyUtil.getPortForStartedServer(ourListenerServer);
+		ourListenerPort = JettyUtil.getPortForStartedServer(ourListenerServer);
-        ourListenerServerBase = "http://localhost:" + ourListenerPort + "/fhir/context";
+		ourListenerServerBase = "http://localhost:" + ourListenerPort + "/fhir/context";
-        ourNotificationListenerServer = "http://localhost:" + ourListenerPort + "/fhir/subscription";
+		ourNotificationListenerServer = "http://localhost:" + ourListenerPort + "/fhir/subscription";
 	}
 
 	@AfterClass

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulatorTest.java

@@ -0,0 +1,75 @@
+package ca.uhn.fhir.jpa.term;
+
+import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao;
+import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDesignationDao;
+import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao;
+import ca.uhn.fhir.jpa.entity.TermValueSet;
+import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
+import ca.uhn.fhir.jpa.entity.TermValueSetConceptDesignation;
+import org.hl7.fhir.r4.model.ValueSet;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Optional;
+
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ValueSetConceptAccumulatorTest {
+
+	private ValueSetConceptAccumulator myAccumulator;
+	private TermValueSet myValueSet;
+	@Mock
+	private ITermValueSetDao myValueSetDao;
+	@Mock
+	private ITermValueSetConceptDesignationDao myValueSetDesignationDao;
+	@Mock
+	private ITermValueSetConceptDao myValueSetConceptDao;
+
+	@Before
+	public void before() {
+		myValueSet = new TermValueSet();
+		myAccumulator = new ValueSetConceptAccumulator(myValueSet, myValueSetDao, myValueSetConceptDao, myValueSetDesignationDao);
+	}
+
+	@Test
+	public void testIncludeConcept() {
+		for (int i = 0; i < 1000; i++) {
+			myAccumulator.includeConcept("sys", "code", "display");
+		}
+		verify(myValueSetConceptDao, times(1000)).save(any());
+	}
+
+	@Test
+	public void testExcludeBlankConcept() {
+		myAccumulator.excludeConcept("", "");
+		verifyNoInteractions(myValueSetConceptDao);
+	}
+
+	@Test
+	public void testAddMessage() {
+		myAccumulator.addMessage("foo");
+		verifyNoInteractions(myValueSetConceptDao);
+	}
+
+	@Test
+	public void testExcludeConceptWithDesignations() {
+		for (int i = 0; i <1000; i++) {
+
+			TermValueSetConcept value = new TermValueSetConcept();
+			value.setCode("code");
+			value.getDesignations().add(new TermValueSetConceptDesignation().setValue("foo"));
+
+			when(myValueSetConceptDao.findByTermValueSetIdSystemAndCode(any(), eq("sys"), eq("code"+i))).thenReturn(Optional.of(value));
+
+			myAccumulator.excludeConcept("sys", "code"+i);
+		}
+
+	}
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4Test.java

@@ -2,11 +2,13 @@ package ca.uhn.fhir.jpa.term;
 
 import ca.uhn.fhir.jpa.entity.*;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import com.google.common.collect.Lists;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r4.model.CodeSystem;
+import org.hl7.fhir.r4.model.Extension;
 import org.hl7.fhir.r4.model.ValueSet;
 import org.hl7.fhir.r4.model.codesystems.HttpVerb;
 import org.junit.Test;
@@ -688,6 +690,21 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
 		assertEquals("Systolische bloeddruk minimaal 1 uur", designationComponent.getValue());
 	}
 
+	@Test
+	public void testExpandValueSetWithUnknownCodeSystem() {
+		ValueSet vs = new ValueSet();
+		ValueSet.ConceptSetComponent include = vs.getCompose().addInclude();
+		include.setSystem("http://unknown-system");
+		ValueSet outcome = myTermSvc.expandValueSetInMemory(vs, null);
+		assertEquals(0, outcome.getExpansion().getContains().size());
+		String encoded = myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome);
+		ourLog.info(encoded);
+
+		Extension extensionByUrl = outcome.getExpansion().getExtensionByUrl(JpaConstants.EXT_VALUESET_EXPANSION_MESSAGE);
+		assertEquals("Unknown CodeSystem URI \"http://unknown-system\" referenced from ValueSet", extensionByUrl.getValueAsPrimitive().getValueAsString());
+	}
+
+
 	@Test
 	public void testExpandTermValueSetAndChildrenWithOffsetAndCountWithClientAssignedId() throws Exception {
 		myDaoConfig.setPreExpandValueSets(true);

hapi-fhir-jpaserver-base/src/test/resources/dstu3/iar/CodeSystem-iar-citizenship-status.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<CodeSystem xmlns="http://hl7.org/fhir">
+	<id value="iar-citizenship-status"/>
+	<url value="http://ccim.on.ca/fhir/iar/CodeSystem/iar-citizenship-status"/>
+	<version value="0.1"/>
+	<name value="Citizenship Status"/>
+	<status value="draft"/>
+	<date value="2019-08-22"/>
+	<publisher value="CCIM"/>
+	<contact>
+		<telecom>
+			<system value="other"/>
+			<value value="https://www.ccim.on.ca"/>
+		</telecom>
+	</contact>
+	<compositional value="false"/>
+	<content value="complete"/>
+	<concept>
+		<code value="CDN"/>
+		<display value="Canadian Citizen"/>
+	</concept>
+	<concept>
+		<code value="PR"/>
+		<display value="Permanent Resident"/>
+	</concept>
+	<concept>
+		<code value="TR"/>
+		<display value="Temporary Resident "/>
+	</concept>
+	<concept>
+		<code value="REF"/>
+		<display value="Refugee"/>
+	</concept>
+</CodeSystem>

hapi-fhir-jpaserver-base/src/test/resources/dstu3/iar/ValueSet-iar-citizenship-status.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ValueSet xmlns="http://hl7.org/fhir">
+	<id value="iar-citizenship-status"/>
+	<url value="http://ccim.on.ca/fhir/iar/ValueSet/iar-citizenship-status"/>
+	<version value="0.1"/>
+	<name value="Citizenship Status"/>
+	<status value="draft"/>
+	<date value="2019-08-08"/>
+	<publisher value="CCIM"/>
+	<contact>
+		<telecom>
+			<system value="other"/>
+			<value value="https://www.ccim.on.ca"/>
+		</telecom>
+	</contact>
+	<compose>
+		<include>
+			<system value="http://ccim.on.ca/fhir/iar/CodeSystem/iar-citizenship-status"/>
+			<version value="0.1"/>
+		</include>
+		<include>
+			<system value="http://hl7.org/fhir/v3/NullFlavor"/>
+			<concept>
+				<code value="UNK"/>
+				<display value="unknown"/>
+			</concept>
+		</include>
+		<include>
+			<system value="http://hl7.org/fhir/v3/NullFlavor"/>
+			<concept>
+				<code value="ASKU"/>
+				<display value="asked but unknown"/>
+			</concept>
+		</include>
+	</compose>
+</ValueSet>

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java

@@ -24,6 +24,13 @@ import ca.uhn.fhir.rest.api.Constants;
 
 public class JpaConstants {
 
+	/**
+	 * Non-instantiable
+	 */
+	private JpaConstants() {
+		// nothing
+	}
+
 	/**
 	 * Operation name for the $apply-codesystem-delta-add operation
 	 */
@@ -243,6 +250,12 @@ public class JpaConstants {
 	 */
 	public static final String EXTENSION_EXT_SYSTEMDEFINED = JpaConstants.class.getName() + "_EXTENSION_EXT_SYSTEMDEFINED";
 
+	/**
+	 * Message added to expansion valueset
+	 */
+	public static final String EXT_VALUESET_EXPANSION_MESSAGE = "http://hapifhir.io/fhir/StructureDefinition/valueset-expansion-message";
+
+
 	/**
 	 * Parameter for the $export operation
 	 */

hapi-fhir-jpaserver-subscription/src/test/java/ca/uhn/fhir/jpa/subscription/module/cache/SubscriptionLoaderTest.java

@@ -11,6 +11,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertEquals;
 
@@ -50,7 +52,17 @@ public class SubscriptionLoaderTest extends BaseBlockingQueueSubscribableChannel
 	@Test
 	public void testMultipleThreadsDontBlock() throws InterruptedException {
 		SubscriptionLoader svc = new SubscriptionLoader();
-		svc.acquireSemaphoreForUnitTest();
+		CountDownLatch latch = new CountDownLatch(1);
+		new Thread(()->{
+			try {
+				svc.acquireSemaphoreForUnitTest();
+				latch.countDown();
+			} catch (InterruptedException theE) {
+				// ignore
+			}
+		}).start();
+
+		latch.await(10, TimeUnit.SECONDS);
 		svc.syncSubscriptions();
 	}
 

hapi-fhir-jpaserver-uhnfhirtest/pom.xml

@@ -51,7 +51,6 @@
 			<artifactId>hapi-fhir-testpage-overlay</artifactId>
 			<version>${project.version}</version>
 			<classifier>classes</classifier>
-			<scope>provided</scope>
 		</dependency>
 
 		<dependency>

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/method/ReadMethodBinding.java

@@ -140,8 +140,7 @@ public class ReadMethodBinding extends BaseResourceReturningMethodBinding {
 		if (Constants.PARAM_HISTORY.equals(theRequest.getOperation())) {
 			if (mySupportsVersion == false) {
 				return false;
-			}
+			} else if (theRequest.getId().hasVersionIdPart() == false) {
-			if (theRequest.getId().hasVersionIdPart() == false) {
 				return false;
 			}
 		} else if (!StringUtils.isBlank(theRequest.getOperation())) {

hapi-fhir-server/src/test/java/ca/uhn/fhir/rest/server/method/ReadMethodBindingTest.java

@@ -101,6 +101,31 @@ public class ReadMethodBindingTest {
 		when(myRequestDetails.getOperation()).thenReturn("$foo");
 		assertFalse(binding.incomingServerRequestMatchesMethod(myRequestDetails));
 
+		// History operation
+		when(myRequestDetails.getId()).thenReturn(new IdDt("Patient/123"));
+		when(myRequestDetails.getOperation()).thenReturn("_history");
+		assertFalse(binding.incomingServerRequestMatchesMethod(myRequestDetails));
+
+	}
+
+	@Test
+	public void testIncomingServerRequestNoMatch_HasCompartment() throws NoSuchMethodException {
+
+		class MyProvider {
+			@Read(version = false)
+			public IBaseResource read(@IdParam IIdType theIdType) {
+				return null;
+			}
+		}
+
+		when(myCtx.getResourceDefinition(any(Class.class))).thenReturn(definition);
+		when(definition.getName()).thenReturn("Patient");
+		when(myRequestDetails.getResourceName()).thenReturn("Patient");
+		when(myRequestDetails.getCompartmentName()).thenReturn("Patient");
+		when(myRequestDetails.getId()).thenReturn(new IdDt("Patient/123"));
+
+		ReadMethodBinding binding = createBinding(new MyProvider());
+		assertFalse(binding.incomingServerRequestMatchesMethod(myRequestDetails));
 	}
 
 	public ReadMethodBinding createBinding(Object theProvider) throws NoSuchMethodException {

hapi-fhir-structures-dstu3/src/test/java/ca/uhn/fhir/util/MetaUtilTest.java

@@ -0,0 +1,19 @@
+package ca.uhn.fhir.util;
+
+import ca.uhn.fhir.context.FhirContext;
+import org.hl7.fhir.dstu3.model.Observation;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class MetaUtilTest {
+	FhirContext ourFhirContext = FhirContext.forDstu3();
+
+	@Test
+	public void testSetGetDstu3() {
+		String source = "testSource";
+		Observation observation = new Observation();
+		MetaUtil.setSource(ourFhirContext, observation, source);
+		assertEquals(source, MetaUtil.getSource(ourFhirContext, observation.getMeta()));
+	}
+}

hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/rest/server/ServerCapabilityStatementProvider.java

@@ -9,7 +9,10 @@ import ca.uhn.fhir.rest.annotation.Metadata;
 import ca.uhn.fhir.rest.annotation.Read;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
-import ca.uhn.fhir.rest.server.*;
+import ca.uhn.fhir.rest.server.Bindings;
+import ca.uhn.fhir.rest.server.IServerConformanceProvider;
+import ca.uhn.fhir.rest.server.RestfulServer;
+import ca.uhn.fhir.rest.server.RestfulServerConfiguration;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.rest.server.method.*;
 import ca.uhn.fhir.rest.server.method.SearchParameter;
@@ -113,7 +116,6 @@ public class ServerCapabilityStatementProvider extends BaseServerCapabilityState
   }
 
 
-
   private DateTimeType conformanceDate(RequestDetails theRequestDetails) {
     IPrimitiveType<Date> buildDate = getServerConfiguration(theRequestDetails).getConformanceDate();
     if (buildDate != null && buildDate.getValue() != null) {
@@ -127,7 +129,6 @@ public class ServerCapabilityStatementProvider extends BaseServerCapabilityState
   }
 
 
-
   /**
    * Gets the value of the "publisher" that will be placed in the generated conformance statement. As this is a mandatory element, the value should not be null (although this is not enforced). The
    * value defaults to "Not provided" but may be set to null, which will cause this element to be omitted.
@@ -194,47 +195,46 @@ public class ServerCapabilityStatementProvider extends BaseServerCapabilityState
         // Map<String, CapabilityStatement.RestResourceSearchParam> nameToSearchParam = new HashMap<String,
         // CapabilityStatement.RestResourceSearchParam>();
         for (BaseMethodBinding<?> nextMethodBinding : nextEntry.getValue()) {
-          if (nextMethodBinding.getRestOperationType() != null) {
+          nextMethodBinding.getRestOperationType();
-            String resOpCode = nextMethodBinding.getRestOperationType().getCode();
+          String resOpCode = nextMethodBinding.getRestOperationType().getCode();
-            if (resOpCode != null) {
+          if (resOpCode != null) {
-              TypeRestfulInteraction resOp;
+            TypeRestfulInteraction resOp;
-              try {
+            try {
-                resOp = TypeRestfulInteraction.fromCode(resOpCode);
+              resOp = TypeRestfulInteraction.fromCode(resOpCode);
-              } catch (Exception e) {
+            } catch (Exception e) {
-                resOp = null;
+              resOp = null;
+            }
+            if (resOp != null) {
+              if (resourceOps.contains(resOp) == false) {
+                resourceOps.add(resOp);
+                resource.addInteraction().setCode(resOp);
               }
-              if (resOp != null) {
+              if ("vread".equals(resOpCode)) {
+                // vread implies read
+                resOp = TypeRestfulInteraction.READ;
                 if (resourceOps.contains(resOp) == false) {
                   resourceOps.add(resOp);
                   resource.addInteraction().setCode(resOp);
                 }
-                if ("vread".equals(resOpCode)) {
+              }
-                  // vread implies read
-                  resOp = TypeRestfulInteraction.READ;
-                  if (resourceOps.contains(resOp) == false) {
-                    resourceOps.add(resOp);
-                    resource.addInteraction().setCode(resOp);
-                  }
-                }
 
-                if (nextMethodBinding.isSupportsConditional()) {
+              if (nextMethodBinding.isSupportsConditional()) {
-                  switch (resOp) {
+                switch (resOp) {
-                    case CREATE:
+                  case CREATE:
-                      resource.setConditionalCreate(true);
+                    resource.setConditionalCreate(true);
-                      break;
+                    break;
-                    case DELETE:
+                  case DELETE:
-                      if (nextMethodBinding.isSupportsConditionalMultiple()) {
+                    if (nextMethodBinding.isSupportsConditionalMultiple()) {
-                        resource.setConditionalDelete(ConditionalDeleteStatus.MULTIPLE);
+                      resource.setConditionalDelete(ConditionalDeleteStatus.MULTIPLE);
-                      } else {
+                    } else {
-                        resource.setConditionalDelete(ConditionalDeleteStatus.SINGLE);
+                      resource.setConditionalDelete(ConditionalDeleteStatus.SINGLE);
-                      }
+                    }
-                      break;
+                    break;
-                    case UPDATE:
+                  case UPDATE:
-                      resource.setConditionalUpdate(true);
+                    resource.setConditionalUpdate(true);
-                      break;
+                    break;
-                    default:
+                  default:
-                      break;
+                    break;
-                  }
                 }
               }
             }
@@ -315,28 +315,18 @@ public class ServerCapabilityStatementProvider extends BaseServerCapabilityState
     }
     sortSearchParameters(searchParameters);
     if (!searchParameters.isEmpty()) {
-      // boolean allOptional = searchParameters.get(0).isRequired() == false;
-      //
-      // OperationDefinition query = null;
-      // if (!allOptional) {
-      // RestOperation operation = rest.addOperation();
-      // query = new OperationDefinition();
-      // operation.setDefinition(new ResourceReferenceDt(query));
-      // query.getDescriptionElement().setValue(searchMethodBinding.getDescription());
-      // query.addUndeclaredExtension(false, ExtensionConstants.QUERY_RETURN_TYPE, new CodeDt(resourceName));
-      // for (String nextInclude : searchMethodBinding.getIncludes()) {
-      // query.addUndeclaredExtension(false, ExtensionConstants.QUERY_ALLOWED_INCLUDE, new StringDt(nextInclude));
-      // }
-      // }
 
       for (SearchParameter nextParameter : searchParameters) {
 
+        if (nextParameter.getParamType() == null) {
+          ourLog.warn("SearchParameter {}:{} does not declare a type - Not exporting in CapabilityStatement", def.getName(), nextParameter.getName());
+          continue;
+        }
+
         String nextParamName = nextParameter.getName();
 
-        String chain = null;
         String nextParamUnchainedName = nextParamName;
         if (nextParamName.contains(".")) {
-          chain = nextParamName.substring(nextParamName.indexOf('.') + 1);
           nextParamUnchainedName = nextParamName.substring(0, nextParamName.indexOf('.'));
         }
 
@@ -352,45 +342,18 @@ public class ServerCapabilityStatementProvider extends BaseServerCapabilityState
           }
         }
 
+
         CapabilityStatementRestResourceSearchParamComponent param = resource.addSearchParam();
+        String typeCode = nextParameter.getParamType().getCode();
+        param.getTypeElement().setValueAsString(typeCode);
         param.setName(nextParamUnchainedName);
-
-//				if (StringUtils.isNotBlank(chain)) {
-//					param.addChain(chain);
-//				}
-//
-//				if (nextParameter.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
-//					for (String nextWhitelist : new TreeSet<String>(nextParameter.getQualifierWhitelist())) {
-//						if (nextWhitelist.startsWith(".")) {
-//							param.addChain(nextWhitelist.substring(1));
-//						}
-//					}
-//				}
-
         param.setDocumentation(nextParamDescription);
-        if (nextParameter.getParamType() != null) {
+
-          param.getTypeElement().setValueAsString(nextParameter.getParamType().getCode());
-        }
-        for (Class<? extends IBaseResource> nextTarget : nextParameter.getDeclaredTypes()) {
-          RuntimeResourceDefinition targetDef = getServerConfiguration(theRequestDetails).getFhirContext().getResourceDefinition(nextTarget);
-          if (targetDef != null) {
-            ResourceType code;
-            try {
-              code = ResourceType.fromCode(targetDef.getName());
-            } catch (FHIRException e) {
-              code = null;
-            }
-//						if (code != null) {
-//							param.addTarget(targetDef.getName());
-//						}
-          }
-        }
       }
     }
   }
 
 
-
   @Read(type = OperationDefinition.class)
   public OperationDefinition readOperationDefinition(@IdParam IdType theId, RequestDetails theRequestDetails) {
     if (theId == null || theId.hasIdPart() == false) {

hapi-fhir-testpage-overlay/src/main/java/ca/uhn/fhir/to/Controller.java

@@ -294,6 +294,9 @@ public class Controller extends BaseController {
 			case R4:
 				haveSearchParams = extractSearchParamsR4CapabilityStatement(conformance, resourceName, includes, revIncludes, sortParams, haveSearchParams, queryIncludes);
 				break;
+			case R5:
+				haveSearchParams = extractSearchParamsR5CapabilityStatement(conformance, resourceName, includes, revIncludes, sortParams, haveSearchParams, queryIncludes);
+				break;
 			default:
 				throw new IllegalStateException("Unknown FHIR version: " + theRequest.getFhirVersion(myConfig));
 		}
@@ -758,6 +761,38 @@ public class Controller extends BaseController {
 		return haveSearchParams;
 	}
 
+	private boolean extractSearchParamsR5CapabilityStatement(IBaseResource theConformance, String resourceName, TreeSet<String> includes, TreeSet<String> theRevIncludes, TreeSet<String> sortParams,
+			boolean haveSearchParams, List<List<String>> queryIncludes) {
+		org.hl7.fhir.r5.model.CapabilityStatement conformance = (org.hl7.fhir.r5.model.CapabilityStatement) theConformance;
+		for (org.hl7.fhir.r5.model.CapabilityStatement.CapabilityStatementRestComponent nextRest : conformance.getRest()) {
+			for (org.hl7.fhir.r5.model.CapabilityStatement.CapabilityStatementRestResourceComponent nextRes : nextRest.getResource()) {
+				if (nextRes.getTypeElement().getValue().equals(resourceName)) {
+					for (org.hl7.fhir.r5.model.StringType next : nextRes.getSearchInclude()) {
+						if (next.isEmpty() == false) {
+							includes.add(next.getValue());
+						}
+					}
+					for (org.hl7.fhir.r5.model.CapabilityStatement.CapabilityStatementRestResourceSearchParamComponent next : nextRes.getSearchParam()) {
+						if (next.getTypeElement().getValue() != org.hl7.fhir.r5.model.Enumerations.SearchParamType.COMPOSITE) {
+							sortParams.add(next.getNameElement().getValue());
+						}
+					}
+					if (nextRes.getSearchParam().size() > 0) {
+						haveSearchParams = true;
+					}
+				} else {
+					// It's a different resource from the one we're searching, so
+					// scan for revinclude candidates
+					for (org.hl7.fhir.r5.model.CapabilityStatement.CapabilityStatementRestResourceSearchParamComponent next : nextRes.getSearchParam()) {
+						if (next.getTypeElement().getValue() == org.hl7.fhir.r5.model.Enumerations.SearchParamType.REFERENCE) {
+						}
+					}
+				}
+			}
+		}
+		return haveSearchParams;
+	}
+
 	private boolean handleSearchParam(String paramIdxString, HttpServletRequest theReq, IQuery theQuery, JsonWriter theClientCodeJsonWriter) throws IOException {
 		String nextName = theReq.getParameter("param." + paramIdxString + ".name");
 		if (isBlank(nextName)) {

pom.xml

@@ -567,6 +567,10 @@
 			<id>jelmerterwal</id>
 			<name>Jelmer ter Wal</name>
 		</developer>
+		<developer>
+			<id>jiaola</id>
+			<name>Dazhi Jiao</name>
+		</developer>
 	</developers>
 
 	<licenses>

src/changes/changes.xml

@@ -476,6 +476,19 @@
 			<action type="fix">
 				Search parameters of type URI did not work in the hapi-fhir-testpage-overlay. This has been corrected.
 			</action>
+			<action type="fix" issue="1568">
+				JPA servers accidentally stripped the type attribute from the server-exported CapabilityStatement
+				when search parameters of type "special" were found. This has been corrected.
+			</action>
+			<action type="fix">
+				When running the JPA server without Lucene indexing enabled and performing ValueSet expansion,
+				the server would incorrectly ignore inclusion rules that specified a system but no codes (i.e.
+				include the whole system). This has been corrected.
+			</action>
+			<action type="fix" issue="1538">
+				The hapi-fhir-testpage-overlay has been updated to support R5 endpoints. Thanks to Dazhi Jiao
+				for the pull request!
+			</action>
 		</release>
 		<release version="4.0.3" date="2019-09-03" description="Igloo (Point Release)">
 			<action type="fix">