Merge remote-tracking branch 'remotes/origin/master' into ks-20200302-near-chain

hapi-fhir-base/src/test/java/ca/uhn/fhir/rest/param/QualifierDetailsTest.java

@@ -0,0 +1,21 @@
+package ca.uhn.fhir.rest.param;
+
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class QualifierDetailsTest {
+
+	@Test
+	public void testBlacklist() {
+
+		QualifierDetails details = new QualifierDetails();
+		details.setColonQualifier(":Patient");
+		assertFalse(details.passes(null, Sets.newHashSet(":Patient")));
+		assertTrue(details.passes(null, Sets.newHashSet(":Observation")));
+
+	}
+
+
+}

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/4_3_0/1742-error-on-missing-profile-validation.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 1742
+title: "When validating a resource, the validator will now report an error if the resource declares conformance
+   to an unknown or invalid profile URL via the `Resource.meta.profile` declaration. Previously this was a warning
+   and did not block successful validation."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/4_3_0/1742-limit-type-on-has.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 1742
+title: "When performing a search in the JPA server where the only parameter was a `_has` parameter,
+   the server did not respect the resource typename being searched for, causing false positive
+   search results. This has been corrected."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/4_3_0/1742-terminology-delta-defer.yaml

@@ -0,0 +1,6 @@
+---
+type: add
+issue: 1742
+title: When performing a terminology delta ADD operation, if the number of codes being added is large
+   the codes will be added in small batches via an asynchronous scheduled task in order to avoid overwhelming
+   the database with a large operation.

hapi-fhir-jpaserver-base/pom.xml

@@ -48,6 +48,11 @@
 			<artifactId>commons-csv</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>co.elastic.apm</groupId>
+			<artifactId>apm-agent-api</artifactId>
+		</dependency>
+
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-base</artifactId>

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java

@@ -102,7 +102,7 @@ public class DaoConfig {
 	/**
 	 * update setter javadoc if default changes
 	 */
-	private int myDeferIndexingForCodesystemsOfSize = 2000;
+	private int myDeferIndexingForCodesystemsOfSize = 100;
 	private boolean myDeleteStaleSearches = true;
 	private boolean myEnforceReferentialIntegrityOnDelete = true;
 	private boolean myUniqueIndexesEnabled = true;
@@ -392,7 +392,7 @@ public class DaoConfig {
 	 * the code system will be indexed later in an incremental process in order to
 	 * avoid overwhelming Lucene with a huge number of codes in a single operation.
 	 * <p>
-	 * Defaults to 2000
+	 * Defaults to 100
 	 * </p>
 	 */
 	public int getDeferIndexingForCodesystemsOfSize() {
@@ -404,7 +404,7 @@ public class DaoConfig {
 	 * the code system will be indexed later in an incremental process in order to
 	 * avoid overwhelming Lucene with a huge number of codes in a single operation.
 	 * <p>
-	 * Defaults to 2000
+	 * Defaults to 100
 	 * </p>
 	 */
 	public void setDeferIndexingForCodesystemsOfSize(int theDeferIndexingForCodesystemsOfSize) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptDao.java

@@ -50,4 +50,8 @@ public interface ITermConceptDao extends JpaRepository<TermConcept, Long> {
 	@Query("SELECT t FROM TermConcept t WHERE t.myIndexStatus = null")
 	Page<TermConcept> findResourcesRequiringReindexing(Pageable thePageRequest);
 
+	@Modifying
+	@Query("DELETE FROM TermConcept t WHERE t.myId = :pid")
+	void deleteByPid(@Param("pid") Long theId);
+
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java

@@ -4,6 +4,7 @@ import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Slice;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 
@@ -39,4 +40,9 @@ public interface ITermConceptParentChildLinkDao extends JpaRepository<TermConcep
 
 	@Query("SELECT t.myPid FROM TermConceptParentChildLink t WHERE t.myCodeSystem.myId = :cs_pid")
 	Slice<Long> findIdsByCodeSystemVersion(Pageable thePage, @Param("cs_pid") Long thePid);
+
+	@Modifying
+	@Query("DELETE FROM TermConceptParentChildLink t WHERE t.myChildPid = :pid OR t.myParentPid = :pid")
+	void deleteByConceptPid(@Param("pid") Long theId);
+
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/predicate/PredicateBuilderReference.java

@@ -430,7 +430,7 @@ class PredicateBuilderReference extends BasePredicateBuilder {
 				break;
 
 			case Constants.PARAM_HAS:
-				addPredicateHas(theAndOrParams, theRequest);
+				addPredicateHas(theResourceName, theAndOrParams, theRequest);
 				break;
 
 			case Constants.PARAM_TAG:
@@ -756,7 +756,7 @@ class PredicateBuilderReference extends BasePredicateBuilder {
 		return retVal;
 	}
 
-	private void addPredicateHas(List<List<IQueryParameterType>> theHasParameters, RequestDetails theRequest) {
+	private void addPredicateHas(String theResourceType, List<List<IQueryParameterType>> theHasParameters, RequestDetails theRequest) {
 
 		for (List<? extends IQueryParameterType> nextOrList : theHasParameters) {
 
@@ -811,8 +811,9 @@ class PredicateBuilderReference extends BasePredicateBuilder {
 
 			Join<ResourceTable, ResourceLink> join = myQueryRoot.join("myResourceLinksAsTarget", JoinType.LEFT);
 			Predicate pathPredicate = myPredicateBuilder.createResourceLinkPathPredicate(targetResourceType, paramReference, join);
-			Predicate pidPredicate = join.get("mySourceResourcePid").in(subQ);
+			Predicate sourceTypePredicate = myBuilder.equal(join.get("myTargetResourceType"), theResourceType);
-			Predicate andPredicate = myBuilder.and(pathPredicate, pidPredicate);
+			Predicate sourcePidPredicate = join.get("mySourceResourcePid").in(subQ);
+			Predicate andPredicate = myBuilder.and(pathPredicate, sourcePidPredicate, sourceTypePredicate);
 			myQueryRoot.addPredicate(andPredicate);
 		}
 	}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermCodeSystem.java

@@ -47,7 +47,7 @@ public class TermCodeSystem implements Serializable {
 	@Column(name = "CODE_SYSTEM_URI", nullable = false, length = MAX_URL_LENGTH)
 	private String myCodeSystemUri;
 
-	@OneToOne()
+	@OneToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "CURRENT_VERSION_PID", referencedColumnName = "PID", nullable = true, foreignKey = @ForeignKey(name = "FK_TRMCODESYSTEM_CURVER"))
 	private TermCodeSystemVersion myCurrentVersion;
 	@Column(name = "CURRENT_VERSION_PID", nullable = true, insertable = false, updatable = false)
@@ -57,7 +57,7 @@ public class TermCodeSystem implements Serializable {
 	@GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CODESYSTEM_PID")
 	@Column(name = "PID")
 	private Long myPid;
-	@OneToOne()
+	@OneToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "RES_ID", referencedColumnName = "RES_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_TRMCODESYSTEM_RES"))
 	private ResourceTable myResource;
 	@Column(name = "RES_ID", insertable = false, updatable = false)

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermCodeSystemVersion.java

@@ -50,7 +50,7 @@ public class TermCodeSystemVersion implements Serializable {
 	@Column(name = "PID")
 	private Long myId;
 
-	@OneToOne()
+	@OneToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "RES_ID", referencedColumnName = "RES_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_CODESYSVER_RES_ID"))
 	private ResourceTable myResource;
 
@@ -64,7 +64,7 @@ public class TermCodeSystemVersion implements Serializable {
 	 * This was added in HAPI FHIR 3.3.0 and is nullable just to avoid migration
 	 * issued. It should be made non-nullable at some point.
 	 */
-	@ManyToOne
+	@ManyToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "CODESYSTEM_PID", referencedColumnName = "PID", nullable = true, foreignKey = @ForeignKey(name = "FK_CODESYSVER_CS_ID"))
 	private TermCodeSystem myCodeSystem;
 
@@ -72,7 +72,7 @@ public class TermCodeSystemVersion implements Serializable {
 	private Long myCodeSystemPid;
 
 	@SuppressWarnings("unused")
-	@OneToOne(mappedBy = "myCurrentVersion", optional = true)
+	@OneToOne(mappedBy = "myCurrentVersion", optional = true, fetch = FetchType.LAZY)
 	private TermCodeSystem myCodeSystemHavingThisVersionAsCurrentVersionIfAny;
 
 	@Column(name = "CS_DISPLAY", nullable = true, updatable = false, length = MAX_VERSION_LENGTH)

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java

@@ -65,7 +65,7 @@ public class TermConcept implements Serializable {
 	@Temporal(TemporalType.TIMESTAMP)
 	@Column(name = "CONCEPT_UPDATED", nullable = true)
 	private Date myUpdated;
-	@ManyToOne()
+	@ManyToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "CODESYSTEM_PID", referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_CONCEPT_PID_CS_PID"))
 	private TermCodeSystemVersion myCodeSystem;
 	@Column(name = "CODESYSTEM_PID", insertable = false, updatable = false)
@@ -79,11 +79,11 @@ public class TermConcept implements Serializable {
 		@Field(name = "myDisplayPhonetic", index = org.hibernate.search.annotations.Index.YES, store = Store.NO, analyze = Analyze.YES, analyzer = @Analyzer(definition = "autocompletePhoneticAnalyzer"))
 	})
 	private String myDisplay;
-	@OneToMany(mappedBy = "myConcept", orphanRemoval = false)
+	@OneToMany(mappedBy = "myConcept", orphanRemoval = false, fetch = FetchType.LAZY)
 	@Field(name = "PROPmyProperties", analyzer = @Analyzer(definition = "termConceptPropertyAnalyzer"))
 	@FieldBridge(impl = TermConceptPropertyFieldBridge.class)
 	private Collection<TermConceptProperty> myProperties;
-	@OneToMany(mappedBy = "myConcept", orphanRemoval = false)
+	@OneToMany(mappedBy = "myConcept", orphanRemoval = false, fetch = FetchType.LAZY)
 	private Collection<TermConceptDesignation> myDesignations;
 	@Id()
 	@SequenceGenerator(name = "SEQ_CONCEPT_PID", sequenceName = "SEQ_CONCEPT_PID")

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptDesignation.java

@@ -39,7 +39,7 @@ public class TermConceptDesignation implements Serializable {
 	public static final int MAX_LENGTH = 500;
 	public static final int MAX_VAL_LENGTH = 2000;
 
-	@ManyToOne
+	@ManyToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "CONCEPT_PID", referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_CONCEPTDESIG_CONCEPT"))
 	private TermConcept myConcept;
 	@Id()
@@ -62,7 +62,7 @@ public class TermConceptDesignation implements Serializable {
 	 *
 	 * @since 3.5.0
 	 */
-	@ManyToOne
+	@ManyToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "CS_VER_PID", nullable = true, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_CONCEPTDESIG_CSV"))
 	private TermCodeSystemVersion myCodeSystemVersion;
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptParentChildLink.java

@@ -32,14 +32,14 @@ import java.io.Serializable;
 public class TermConceptParentChildLink implements Serializable {
 	private static final long serialVersionUID = 1L;
 
-	@ManyToOne()
+	@ManyToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "CHILD_PID", nullable = false, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_TERM_CONCEPTPC_CHILD"))
 	private TermConcept myChild;
 
 	@Column(name = "CHILD_PID", insertable = false, updatable = false)
 	private Long myChildPid;
 
-	@ManyToOne()
+	@ManyToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "CODESYSTEM_PID", nullable = false, foreignKey = @ForeignKey(name = "FK_TERM_CONCEPTPC_CS"))
 	private TermCodeSystemVersion myCodeSystem;
 
@@ -47,7 +47,7 @@ public class TermConceptParentChildLink implements Serializable {
 	@Fields({@Field(name = "myCodeSystemVersionPid")})
 	private long myCodeSystemVersionPid;
 
-	@ManyToOne(cascade = {})
+	@ManyToOne(fetch = FetchType.LAZY, cascade = {})
 	@JoinColumn(name = "PARENT_PID", nullable = false, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_TERM_CONCEPTPC_PARENT"))
 	private TermConcept myParent;
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptProperty.java

@@ -44,7 +44,7 @@ public class TermConceptProperty implements Serializable {
 	private static final int MAX_LENGTH = 500;
 	static final int MAX_PROPTYPE_ENUM_LENGTH = 6;
 
-	@ManyToOne
+	@ManyToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "CONCEPT_PID", referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_CONCEPTPROP_CONCEPT"))
 	private TermConcept myConcept;
 	/**
@@ -52,7 +52,7 @@ public class TermConceptProperty implements Serializable {
 	 *
 	 * @since 3.5.0
 	 */
-	@ManyToOne
+	@ManyToOne(fetch = FetchType.LAZY)
 	@JoinColumn(name = "CS_VER_PID", nullable = true, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_CONCEPTPROP_CSV"))
 	private TermCodeSystemVersion myCodeSystemVersion;
 	@Id()

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java

@@ -56,6 +56,9 @@ import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
 import ca.uhn.fhir.rest.server.util.ICachedSearchDetails;
 import ca.uhn.fhir.util.AsyncUtil;
 import ca.uhn.fhir.util.StopWatch;
+import co.elastic.apm.api.ElasticApm;
+import co.elastic.apm.api.Span;
+import co.elastic.apm.api.Transaction;
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang3.Validate;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -601,7 +604,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 		private List<ResourcePersistentId> myPreviouslyAddedResourcePids;
 		private Integer myMaxResultsToFetch;
 		private SearchRuntimeDetails mySearchRuntimeDetails;
-
+		private Transaction myParentTransaction;
 		/**
 		 * Constructor
 		 */
@@ -614,6 +617,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 			mySearchRuntimeDetails = new SearchRuntimeDetails(theRequest, mySearch.getUuid());
 			mySearchRuntimeDetails.setQueryString(theParams.toNormalizedQueryString(theCallingDao.getContext()));
 			myRequest = theRequest;
+			myParentTransaction = ElasticApm.currentTransaction();
 		}
 
 		/**
@@ -841,7 +845,8 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 		@Override
 		public Void call() {
 			StopWatch sw = new StopWatch();
-
+			Span span = myParentTransaction.startSpan("db", "query", "search");
+			span.setName("FHIR Database Search");
 			try {
 				// Create an initial search in the DB and give it an ID
 				saveSearch();
@@ -897,7 +902,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 					ourLog.error("Failed during search loading after {}ms", sw.getMillis(), t);
 				}
 				myUnsyncedPids.clear();
-
 				Throwable rootCause = ExceptionUtils.getRootCause(t);
 				rootCause = defaultIfNull(rootCause, t);
 
@@ -924,12 +928,13 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 				JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FAILED, params);
 
 				saveSearch();
-
+				span.captureException(t);
 			} finally {
 
 				myIdToSearchTask.remove(mySearch.getUuid());
 				myInitialCollectionLatch.countDown();
 				markComplete();
+				span.end();
 
 			}
 			return null;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java

@@ -1246,6 +1246,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 		return retVal;
 	}
 
+	@Transactional
 	@Override
 	public List<VersionIndependentConcept> findCodesAbove(String theSystem, String theCode) {
 		TermCodeSystem cs = getCodeSystem(theSystem);
@@ -1277,6 +1278,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc, ApplicationCo
 		return retVal;
 	}
 
+	@Transactional
 	@Override
 	public List<VersionIndependentConcept> findCodesBelow(String theSystem, String theCode) {
 		TermCodeSystem cs = getCodeSystem(theSystem);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcImpl.java

@@ -223,8 +223,16 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 		 * save parent concepts first (it's way too slow to do that)
 		 */
 		if (theConcept.getId() == null) {
+			boolean needToSaveParents = false;
+			for (TermConceptParentChildLink next : theConcept.getParents()) {
+				if (next.getParent().getId() == null) {
+					needToSaveParents = true;
+				}
+			}
+			if (needToSaveParents) {
 				retVal += ensureParentsSaved(theConcept.getParents());
 			}
+		}
 
 		if (theConcept.getId() == null || theConcept.getIndexStatus() == null) {
 			retVal++;
@@ -485,7 +493,7 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 		conceptToAdd.setCodeSystemVersion(theCsv);
 
 		if (theStatisticsTracker.getUpdatedConceptCount() <= myDaoConfig.getDeferIndexingForCodesystemsOfSize()) {
-			conceptToAdd = myConceptDao.save(conceptToAdd);
+			saveConcept(conceptToAdd);
 			Long nextConceptPid = conceptToAdd.getId();
 			Validate.notNull(nextConceptPid);
 		} else {
@@ -655,12 +663,15 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
 	private void deleteConceptChildrenAndConcept(TermConcept theConcept, AtomicInteger theRemoveCounter) {
 		for (TermConceptParentChildLink nextChildLink : theConcept.getChildren()) {
 			deleteConceptChildrenAndConcept(nextChildLink.getChild(), theRemoveCounter);
-			myConceptParentChildLinkDao.delete(nextChildLink);
 		}
 
+		myConceptParentChildLinkDao.deleteByConceptPid(theConcept.getId());
+
 		myConceptDesignationDao.deleteAll(theConcept.getDesignations());
 		myConceptPropertyDao.deleteAll(theConcept.getProperties());
-		myConceptDao.delete(theConcept);
+
+		ourLog.info("Deleting concept {} - Code {}", theConcept.getId(), theConcept.getCode());
+		myConceptDao.deleteByPid(theConcept.getId());
 		theRemoveCounter.incrementAndGet();
 	}
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImpl.java

@@ -98,6 +98,13 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
 		myDeferredValueSets.addAll(theValueSets);
 	}
 
+	@Override
+	public void saveAllDeferred() {
+		while (!isStorageQueueEmpty()) {
+			saveDeferred();
+		}
+	}
+
 	@Override
 	public void setProcessDeferred(boolean theProcessDeferred) {
 		myProcessDeferred = theProcessDeferred;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/api/ITermDeferredStorageSvc.java

@@ -50,4 +50,9 @@ public interface ITermDeferredStorageSvc {
 	void addConceptMapsToStorageQueue(List<ConceptMap> theConceptMaps);
 
 	void addValueSetsToStorageQueue(List<ValueSet> theValueSets);
+
+	/**
+	 * This is mostly here for unit tests - Saves any and all deferred concepts and links
+	 */
+	void saveAllDeferred();
 }

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java

@@ -9,6 +9,7 @@ import ca.uhn.fhir.jpa.model.cross.ResourcePersistentId;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.term.TermReindexingSvcImpl;
+import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
 import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.param.TokenParamModifier;
@@ -134,8 +135,13 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test {
 		}
 
 		myTermCodeSystemStorageSvc.storeNewCodeSystemVersion(new ResourcePersistentId(table.getId()), URL_MY_CODE_SYSTEM, "SYSTEM NAME", "SYSTEM VERSION" , cs, table);
+
+		myTermDeferredStorageSvc.saveAllDeferred();
 	}
 
+	@Autowired
+	private ITermDeferredStorageSvc myTermDeferredStorageSvc;
+
 	private void createExternalCsAndLocalVs() {
 		CodeSystem codeSystem = createExternalCs();
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3ValidateTest.java

@@ -15,7 +15,6 @@ import org.hl7.fhir.dstu3.hapi.validation.FhirInstanceValidator;
 import org.hl7.fhir.dstu3.model.*;
 import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
 import org.hl7.fhir.dstu3.model.Observation.ObservationStatus;
-import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r5.utils.IResourceValidator;
@@ -323,12 +322,16 @@ public class FhirResourceDaoDstu3ValidateTest extends BaseJpaDstu3Test {
 
 		ValidationModeEnum mode = ValidationModeEnum.CREATE;
 		String encoded = myFhirCtx.newJsonParser().encodeResourceToString(input);
-		MethodOutcome output = myObservationDao.validate(input, null, encoded, EncodingEnum.JSON, mode, null, mySrd);
+		try {
-		OperationOutcome oo = (OperationOutcome) output.getOperationOutcome();
+			myObservationDao.validate(input, null, encoded, EncodingEnum.JSON, mode, null, mySrd);
+			fail();
+		} catch (PreconditionFailedException e) {
+			OperationOutcome oo = (OperationOutcome) e.getOperationOutcome();
 			String outputString = myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo);
 			ourLog.info(outputString);
 			assertThat(outputString, containsString("Profile reference 'http://example.com/StructureDefinition/testValidateResourceContainingProfileDeclarationInvalid' could not be resolved, so has not been checked"));
 		}
+	}
 
 	@Test
 	public void testValidateForCreate() {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java

@@ -172,6 +172,9 @@ public abstract class BaseJpaR4Test extends BaseJpaTest {
 	@Qualifier("myDeviceDaoR4")
 	protected IFhirResourceDao<Device> myDeviceDao;
 	@Autowired
+	@Qualifier("myProvenanceDaoR4")
+	protected IFhirResourceDao<Provenance> myProvenanceDao;
+	@Autowired
 	@Qualifier("myDiagnosticReportDaoR4")
 	protected IFhirResourceDao<DiagnosticReport> myDiagnosticReportDao;
 	@Autowired

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java

@@ -589,6 +589,41 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
 
 	}
 
+	@Test
+	public void testHasLimitsByType() {
+
+		Patient patient = new Patient();
+		patient.setActive(true);
+		IIdType patientId = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		Encounter encounter = new Encounter();
+		encounter.setStatus(Encounter.EncounterStatus.ARRIVED);
+		IIdType encounterId = myEncounterDao.create(encounter).getId().toUnqualifiedVersionless();
+
+		Device device = new Device();
+		device.setManufacturer("Acme");
+		IIdType deviceId = myDeviceDao.create(device).getId().toUnqualifiedVersionless();
+
+		Provenance provenance = new Provenance();
+		provenance.addTarget().setReferenceElement(patientId);
+		provenance.addTarget().setReferenceElement(encounterId);
+		provenance.addAgent().setWho(new Reference(deviceId));
+		myProvenanceDao.create(provenance);
+
+		String criteria = "_has:Provenance:target:agent=" + deviceId.getValue();
+		SearchParameterMap map = myMatchUrlService.translateMatchUrl(criteria, myFhirCtx.getResourceDefinition(Encounter.class));
+
+		map.setLoadSynchronous(true);
+
+		myCaptureQueriesListener.clear();
+		IBundleProvider results = myEncounterDao.search(map);
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread(0);
+
+		List<String> ids = toUnqualifiedVersionlessIdValues(results);
+		assertThat(ids, containsInAnyOrder(encounterId.getValue()));
+
+	}
+
 	@Test
 	public void testHasParameter() {
 		IIdType pid0;

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java

@@ -164,6 +164,8 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
 		}
 
 		myTermCodeSystemStorageSvc.storeNewCodeSystemVersion(new ResourcePersistentId(table.getId()), URL_MY_CODE_SYSTEM, "SYSTEM NAME", "SYSTEM VERSION", cs, table);
+
+		myTerminologyDeferredStorageSvc.saveAllDeferred();
 	}
 
 	private void createLocalCsAndVs() {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java

@@ -147,6 +147,8 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
 		}
 		myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://loinc.org", codesToAdd);
 
+		myTerminologyDeferredStorageSvc.saveAllDeferred();
+
 		// Create a valueset
 		ValueSet vs = new ValueSet();
 		vs.setUrl("http://example.com/fhir/ValueSet/observation-vitalsignresult");
@@ -395,12 +397,15 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
 		ValidationModeEnum mode = ValidationModeEnum.CREATE;
 		String encoded = myFhirCtx.newJsonParser().encodeResourceToString(input);
 
-		MethodOutcome output = myObservationDao.validate(input, null, encoded, EncodingEnum.JSON, mode, null, mySrd);
+		try {
-		org.hl7.fhir.r4.model.OperationOutcome oo = (org.hl7.fhir.r4.model.OperationOutcome) output.getOperationOutcome();
+			myObservationDao.validate(input, null, encoded, EncodingEnum.JSON, mode, null, mySrd);
+			fail();
+		} catch (PreconditionFailedException e) {
+			org.hl7.fhir.r4.model.OperationOutcome oo = (org.hl7.fhir.r4.model.OperationOutcome) e.getOperationOutcome();
 			String outputString = myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo);
 			ourLog.info(outputString);
 			assertThat(outputString, containsString("Profile reference 'http://example.com/StructureDefinition/testValidateResourceContainingProfileDeclarationInvalid' could not be resolved, so has not been checked"));
-
+		}
 	}
 
 	@Test
@@ -614,6 +619,8 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test {
 		upload("/r4/uscore/ValueSet-omb-race-category.json");
 		upload("/r4/uscore/ValueSet-us-core-usps-state.json");
 
+		myTerminologyDeferredStorageSvc.saveAllDeferred();
+
 		{
 			String resource = loadResource("/r4/uscore/patient-resource-badcode.json");
 			IBaseResource parsedResource = myFhirCtx.newJsonParser().parseResource(resource);

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java

@@ -15,6 +15,7 @@ import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.parser.StrictErrorHandler;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.*;
+import ca.uhn.fhir.rest.client.apache.ResourceEntity;
 import ca.uhn.fhir.rest.client.api.IClientInterceptor;
 import ca.uhn.fhir.rest.client.api.IGenericClient;
 import ca.uhn.fhir.rest.client.api.IHttpRequest;
@@ -26,6 +27,9 @@ import ca.uhn.fhir.rest.server.exceptions.*;
 import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor;
 import ca.uhn.fhir.util.StopWatch;
 import ca.uhn.fhir.util.UrlUtil;
+import ca.uhn.fhir.validation.ResultSeverityEnum;
+import ca.uhn.fhir.validation.SingleValidationMessage;
+import ca.uhn.fhir.validation.ValidationResult;
 import com.google.common.base.Charsets;
 import com.google.common.collect.Lists;
 import org.apache.commons.io.IOUtils;
@@ -52,6 +56,7 @@ import org.hl7.fhir.r4.model.Observation.ObservationStatus;
 import org.hl7.fhir.r4.model.Questionnaire.QuestionnaireItemType;
 import org.hl7.fhir.r4.model.Subscription.SubscriptionChannelType;
 import org.hl7.fhir.r4.model.Subscription.SubscriptionStatus;
+import org.hl7.fhir.utilities.xhtml.XhtmlNode;
 import org.junit.*;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.util.AopTestUtils;
@@ -2260,6 +2265,28 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 		}
 	}
 
+
+	@Test
+	public void testValidateResourceContainingProfileDeclarationDoesntResolve() throws IOException {
+		Observation input = new Observation();
+		input.getText().setDiv(new XhtmlNode().setValue("<div>AA</div>")).setStatus(Narrative.NarrativeStatus.GENERATED);
+		input.getMeta().addProfile("http://foo/structuredefinition/myprofile");
+
+		input.getCode().setText("Hello");
+		input.setStatus(ObservationStatus.FINAL);
+
+		HttpPost post = new HttpPost(ourServerBase + "/Observation/$validate?_pretty=true");
+		post.setEntity(new ResourceEntity(myFhirCtx, input));
+
+		try (CloseableHttpResponse resp = ourHttpClient.execute(post)) {
+			String respString = IOUtils.toString(resp.getEntity().getContent(), Charsets.UTF_8);
+			ourLog.info(respString);
+			assertEquals(412, resp.getStatusLine().getStatusCode());
+			assertThat(respString, containsString("Profile reference 'http://foo/structuredefinition/myprofile' could not be resolved, so has not been checked"));
+		}
+	}
+
+
 	@SuppressWarnings("unused")
 	@Test
 	public void testFullTextSearch() throws Exception {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcDeltaR4Test.java

@@ -31,8 +31,10 @@ import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import static org.apache.commons.lang3.StringUtils.countMatches;
 import static org.apache.commons.lang3.StringUtils.leftPad;
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.matchesPattern;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
@@ -126,17 +128,23 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
 			ourLog.info("All concepts: {}", myTermConceptDao.findAll());
 		});
 
+		myCaptureQueriesListener.clear();
+
 		delta = new CustomTerminologySet();
 		TermConcept root = delta.addRootConcept("RootA", "Root A");
 		root.addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA).setCode("ChildAA").setDisplay("Child AA");
 		root.addChild(TermConceptParentChildLink.RelationshipTypeEnum.ISA).setCode("ChildAB").setDisplay("Child AB");
 		myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd("http://foo/cs", delta);
+
+		myCaptureQueriesListener.logAllQueriesForCurrentThread();
+
 		assertHierarchyContains(
 			"RootA seq=0",
 			" ChildAA seq=0",
 			" ChildAB seq=1",
 			"RootB seq=0"
 		);
+
 	}
 
 	@Test
@@ -175,6 +183,12 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
 		);
 		assertEquals(2, outcome.getUpdatedConceptCount());
 
+		runInTransaction(() -> {
+			TermConcept concept = myTermSvc.findCode("http://foo/cs", "ChildAA").orElseThrow(() -> new IllegalStateException());
+			assertEquals(2, concept.getParents().size());
+			assertThat(concept.getParentPidsAsString(), matchesPattern("^[0-9]+ [0-9]+$"));
+		});
+
 	}
 
 	@Test
@@ -538,9 +552,15 @@ public class TerminologySvcDeltaR4Test extends BaseJpaR4Test {
 		assertEquals(true, runInTransaction(() -> myTermSvc.findCode("http://foo/cs", "codeAAA").isPresent()));
 
 		// Remove CodeA
-		delta = new CustomTerminologySet();
+		myCaptureQueriesListener.clear();
-		delta.addRootConcept("codeA");
+		runInTransaction(()->{
-		myTermCodeSystemStorageSvc.applyDeltaCodeSystemsRemove("http://foo/cs", delta);
+			CustomTerminologySet delta2 = new CustomTerminologySet();
+			delta2.addRootConcept("codeA");
+			myTermCodeSystemStorageSvc.applyDeltaCodeSystemsRemove("http://foo/cs", delta2);
+		});
+		myCaptureQueriesListener.logAllQueriesForCurrentThread();
+
+		ourLog.info("*** Done removing");
 
 		assertEquals(false, runInTransaction(() -> myTermSvc.findCode("http://foo/cs", "codeB").isPresent()));
 		assertEquals(false, runInTransaction(() -> myTermSvc.findCode("http://foo/cs", "codeA").isPresent()));

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java

@@ -100,6 +100,8 @@ public class TerminologySvcImplDstu3Test extends BaseJpaDstu3Test {
 
 		myTermCodeSystemStorageSvc.storeNewCodeSystemVersion(new ResourcePersistentId(table.getId()), CS_URL, "SYSTEM NAME", "SYSTEM VERSION", cs, table);
 
+		myTerminologyDeferredStorageSvc.saveAllDeferred();
+
 		return id;
 	}
 

hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/ValidatorWrapper.java

@@ -171,11 +171,18 @@ public class ValidatorWrapper {
 		for (int i = 0; i < messages.size(); i++) {
 			ValidationMessage next = messages.get(i);
 			String message = next.getMessage();
+
+			// TODO: are these still needed?
 			if ("Binding has no source, so can't be checked".equals(message) ||
 				"ValueSet http://hl7.org/fhir/ValueSet/mimetypes not found".equals(message)) {
 				messages.remove(i);
 				i--;
 			}
+
+			if (message.endsWith("' could not be resolved, so has not been checked") && next.getLevel() == ValidationMessage.IssueSeverity.WARNING) {
+				next.setLevel(ValidationMessage.IssueSeverity.ERROR);
+			}
+
 		}
 
 		return messages;

hapi-fhir-validation/src/test/java/org/hl7/fhir/r4/validation/FhirInstanceValidatorR4Test.java

@@ -66,10 +66,13 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.*;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class FhirInstanceValidatorR4Test extends BaseTest {
 
@@ -280,6 +283,33 @@ public class FhirInstanceValidatorR4Test extends BaseTest {
 		assertEquals("Primitive types must have a value that is not empty", all.get(0).getMessage());
 	}
 
+	/**
+	 * See #1740
+	 */
+	@Ignore
+	@Test
+	public void testValidateScalarInRepeatableField() {
+		String operationDefinition = "{\n" +
+			"  \"resourceType\": \"OperationDefinition\",\n" +
+			"  \"name\": \"Questionnaire\",\n" +
+			"  \"status\": \"draft\",\n" +
+			"  \"kind\" : \"operation\",\n" +
+			"  \"code\": \"populate\",\n" +
+			"  \"resource\": \"Patient\",\n" + // should be array
+			"  \"system\": false,\n" + " " +
+			" \"type\": false,\n" +
+			"  \"instance\": true\n" +
+			"}";
+
+		FhirValidator val = ourCtx.newValidator();
+		val.registerValidatorModule(new FhirInstanceValidator(myDefaultValidationSupport));
+
+		ValidationResult result = val.validateWithResult(operationDefinition);
+		List<SingleValidationMessage> all = logResultsAndReturnAll(result);
+		assertFalse(result.isSuccessful());
+		assertEquals("Primitive types must have a value that is not empty", all.get(0).getMessage());
+	}
+
 	/**
 	 * See #1676 - We should ignore schema location
 	 */
@@ -400,7 +430,7 @@ public class FhirInstanceValidatorR4Test extends BaseTest {
 		List<SingleValidationMessage> errors = logResultsAndReturnNonInformationalOnes(output);
 		errors = errors
 			.stream()
-			.filter(t->t.getMessage().contains("Bundle entry missing fullUrl"))
+			.filter(t -> t.getMessage().contains("Bundle entry missing fullUrl"))
 			.collect(Collectors.toList());
 		assertEquals(5, errors.size());
 	}
@@ -658,7 +688,6 @@ public class FhirInstanceValidatorR4Test extends BaseTest {
 			.setCode("acd");
 
 
-
 		// Should pass
 		ValidationResult output = val.validateWithResult(input);
 		List<SingleValidationMessage> all = logResultsAndReturnErrorOnes(output);
@@ -1038,7 +1067,10 @@ public class FhirInstanceValidatorR4Test extends BaseTest {
 		myInstanceVal.setValidationSupport(myMockSupport);
 		ValidationResult output = myVal.validateWithResult(input);
 		List<SingleValidationMessage> errors = logResultsAndReturnNonInformationalOnes(output);
-		assertThat(errors.toString(), containsString("Profile reference 'http://foo/structuredefinition/myprofile' could not be resolved, so has not been checked"));
+
+		assertEquals(1, errors.size());
+		assertEquals("Profile reference 'http://foo/structuredefinition/myprofile' could not be resolved, so has not been checked", errors.get(0).getMessage());
+		assertEquals(ResultSeverityEnum.ERROR, errors.get(0).getSeverity());
 	}
 
 	@Test
@@ -1110,7 +1142,7 @@ public class FhirInstanceValidatorR4Test extends BaseTest {
 			"            },\n" +
 			"            \"text\": \"210.0-925.\"\n" +
 			"          }\n" +
-			"        ]"+
+			"        ]" +
 			"}";
 		ourLog.info(input);
 		ValidationResult output = myVal.validateWithResult(input);
@@ -1263,7 +1295,7 @@ public class FhirInstanceValidatorR4Test extends BaseTest {
 		String encoded = loadResource("/r4/r4-caredove-bundle.json");
 
 		IResourceValidator.IValidatorResourceFetcher resourceFetcher = mock(IResourceValidator.IValidatorResourceFetcher.class);
-		when(resourceFetcher.validationPolicy(any(),anyString(), anyString())).thenReturn(IResourceValidator.ReferenceValidationPolicy.CHECK_TYPE_IF_EXISTS);
+		when(resourceFetcher.validationPolicy(any(), anyString(), anyString())).thenReturn(IResourceValidator.ReferenceValidationPolicy.CHECK_TYPE_IF_EXISTS);
 		myInstanceVal.setValidatorResourceFetcher(resourceFetcher);
 		myVal.validateWithResult(encoded);
 

pom.xml

@@ -679,6 +679,7 @@
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 		<ebay_cors_filter_version>1.0.1</ebay_cors_filter_version>
 
+		<elastic_apm_version>1.13.0</elastic_apm_version>
 		<!-- Site properties -->
 		<fontawesomeVersion>5.4.1</fontawesomeVersion>
 	</properties>
@@ -1036,6 +1037,11 @@
 				<artifactId>httpcore</artifactId>
 				<version>${httpcore_version}</version>
 			</dependency>
+			<dependency>
+				<groupId>co.elastic.apm</groupId>
+				<artifactId>apm-agent-api</artifactId>
+				<version>${elastic_apm_version}</version>
+			</dependency>
 			<dependency>
 				<groupId>org.apache.jena</groupId>
 				<artifactId>apache-jena-libs</artifactId>