Copy identifiers to placeholder resources (#1675)

* Copy identifiers to placeholder resources

* Add tests

* Maven cleanup

* Still messing around with azure

* Adding to logging

* More azure work

* One more attempt

* More messing around with azure

* Test fix

* Another caching attempt

* More work on azure pipeline

* Fix pipeline file

* Keep working on pipeline

* More work on azure

* More azure

* More azure work

* More azure

azure-pipelines.yml

@@ -2,7 +2,9 @@
 
 variables:
   MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
-  MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+  #MAVEN_CACHE_FOLDER: $(Agent.TempDirectory)/.m2/repository
+  #MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+  MAVEN_OPTS: ''
 
 trigger:
 - master
@@ -15,17 +17,21 @@ jobs:
     timeoutInMinutes: 360
     container: maven:3-jdk-11
     steps:
-      - task: CacheBeta@0
+      - task: Cache@2
         inputs:
-          key: maven
+          key: 'maven | "$(Agent.OS)" | **/pom.xml'
           path: $(MAVEN_CACHE_FOLDER)
+      - task: Bash@3
+        inputs:
+          targetType: 'inline'
+          script: mkdir -p $(MAVEN_CACHE_FOLDER); pwd; ls -al $(MAVEN_CACHE_FOLDER)
       - task: Maven@3
         env:
           JAVA_HOME_11_X64: /usr/local/openjdk-11
         inputs:
-          goals: 'clean dependency:resolve install'
+          goals: 'clean install'
           # These are Maven CLI options (and show up in the build logs) - "-nsu"=Don't update snapshots. We can remove this when Maven OSS is more healthy
-          options: '-P ALLMODULES,JACOCO,CI,ERRORPRONE -nsu'
+          options: '-P ALLMODULES,JACOCO,CI,ERRORPRONE -nsu -e -B -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
           # These are JVM options (and don't show up in the build logs)
           mavenOptions: '-Xmx2048m $(MAVEN_OPTS) -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss,SSS -Duser.timezone=America/Toronto'
           jdkVersionOption: 1.11

hapi-fhir-base/src/main/java/org/hl7/fhir/instance/model/api/IBaseReference.java

@@ -34,4 +34,9 @@ public interface IBaseReference extends ICompositeType {
 	IBase setDisplay(String theValue);
 
 	IPrimitiveType<String> getDisplayElement();
+
+	default boolean hasIdentifier() {
+		return false;
+	}
+	
 }

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/4_2_0/1675-copy-identifiers-to-placeholder.yaml

@@ -0,0 +1,5 @@
+---
+type: add
+title: "In the JPA server, a new setting called `setPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets(boolean)` has
+   been added to the DaoConfig. If this setting is enabled, when creating placeholder resources, the Reference.identifier
+   value is copied to the target resource if possible."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseStorageDao.java

@@ -53,6 +53,7 @@ import java.util.Set;
 
 import static ca.uhn.fhir.jpa.dao.BaseHapiFhirDao.OO_SEVERITY_ERROR;
 import static ca.uhn.fhir.jpa.dao.BaseHapiFhirDao.OO_SEVERITY_INFO;
+import static org.apache.commons.lang3.StringUtils.defaultString;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
@@ -95,7 +96,8 @@ public abstract class BaseStorageDao {
 		if ("Bundle".equals(type)) {
 			Set<String> allowedBundleTypes = getConfig().getBundleTypesAllowedForStorage();
 			String bundleType = BundleUtil.getBundleType(getContext(), (IBaseBundle) theResource);
-			if (isBlank(bundleType) || !allowedBundleTypes.contains(bundleType)) {
+			bundleType = defaultString(bundleType);
+			if (!allowedBundleTypes.contains(bundleType)) {
 				String message = "Unable to store a Bundle resource on this server with a Bundle.type value of: " + (isNotBlank(bundleType) ? bundleType : "(missing)");
 				throw new UnprocessableEntityException(message);
 			}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java

@@ -15,7 +15,12 @@ import org.hl7.fhir.r4.model.Bundle;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 
 /*
  * #%L
@@ -53,12 +58,6 @@ public class DaoConfig {
 	 * See {@link #setStatusBasedReindexingDisabled(boolean)}
 	 */
 	public static final String DISABLE_STATUS_BASED_REINDEX = "disable_status_based_reindex";
-	/**
-	 * Default value for {@link #setMaximumSearchResultCountInTransaction(Integer)}
-	 *
-	 * @see #setMaximumSearchResultCountInTransaction(Integer)
-	 */
-	private static final Integer DEFAULT_MAXIMUM_SEARCH_RESULT_COUNT_IN_TRANSACTION = null;
 	/**
 	 * Default {@link #setBundleTypesAllowedForStorage(Set)} value:
 	 * <ul>
@@ -73,12 +72,16 @@ public class DaoConfig {
 		Bundle.BundleType.DOCUMENT.toCode(),
 		Bundle.BundleType.MESSAGE.toCode()
 	)));
-	private static final Logger ourLog = LoggerFactory.getLogger(DaoConfig.class);
-	private static final int DEFAULT_EXPUNGE_BATCH_SIZE = 800;
-
 	// update setter javadoc if default changes
 	public static final int DEFAULT_MAX_EXPANSION_SIZE = 1000;
-
+	/**
+	 * Default value for {@link #setMaximumSearchResultCountInTransaction(Integer)}
+	 *
+	 * @see #setMaximumSearchResultCountInTransaction(Integer)
+	 */
+	private static final Integer DEFAULT_MAXIMUM_SEARCH_RESULT_COUNT_IN_TRANSACTION = null;
+	private static final Logger ourLog = LoggerFactory.getLogger(DaoConfig.class);
+	private static final int DEFAULT_EXPUNGE_BATCH_SIZE = 800;
 	private IndexEnabledEnum myIndexMissingFieldsEnabled = IndexEnabledEnum.DISABLED;
 
 	/**
@@ -175,6 +178,11 @@ public class DaoConfig {
 	 */
 	private int myPreExpandValueSetsMaxCount = 1000;
 
+	/**
+	 * @since 4.2.0
+	 */
+	private boolean myPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets;
+
 	/**
 	 * Constructor
 	 */
@@ -993,6 +1001,108 @@ public class DaoConfig {
 		myAutoCreatePlaceholderReferenceTargets = theAutoCreatePlaceholderReferenceTargets;
 	}
 
+	/**
+	 * When {@link #setAutoCreatePlaceholderReferenceTargets(boolean)} is enabled, if this
+	 * setting is set to <code>true</code> (default is <code>false</code>) and the source
+	 * reference has an identifier populated, the identifier will be copied to the target
+	 * resource.
+	 * <p>
+	 * When enabled, if an Observation contains a reference like the one below,
+	 * and no existing resource was found that matches the given ID, a new
+	 * one will be created and its <code>Patient.identifier</code> value will be
+	 * populated using the value from <code>Observation.subject.identifier</code>.
+	 * </p>
+	 * <pre>
+	 * {
+	 *   "resourceType": "Observation",
+	 *   "subject": {
+	 *     "reference": "Patient/ABC",
+	 *     "identifier": {
+	 *       "system": "http://foo",
+	 *       "value": "123"
+	 *     }
+	 *   }
+	 * }
+	 * </pre>
+	 * <p>
+	 * This method is often combined with {@link #setAllowInlineMatchUrlReferences(boolean)}.
+	 * </p>
+	 * <p>
+	 * In other words if an Observation contains a reference like the one below,
+	 * and no existing resource was found that matches the given match URL, a new
+	 * one will be created and its <code>Patient.identifier</code> value will be
+	 * populated using the value from <code>Observation.subject.identifier</code>.
+	 * </p>
+	 * <pre>
+	 * {
+	 *   "resourceType": "Observation",
+	 *   "subject": {
+	 *     "reference": "Patient?identifier=http://foo|123",
+	 *     "identifier": {
+	 *       "system": "http://foo",
+	 *       "value": "123"
+	 *     }
+	 *   }
+	 * }
+	 * </pre>
+	 *
+	 * @since 4.2.0
+	 */
+	public boolean isPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets() {
+		return myPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets;
+	}
+
+	/**
+	 * When {@link #setAutoCreatePlaceholderReferenceTargets(boolean)} is enabled, if this
+	 * setting is set to <code>true</code> (default is <code>false</code>) and the source
+	 * reference has an identifier populated, the identifier will be copied to the target
+	 * resource.
+	 * <p>
+	 * When enabled, if an Observation contains a reference like the one below,
+	 * and no existing resource was found that matches the given ID, a new
+	 * one will be created and its <code>Patient.identifier</code> value will be
+	 * populated using the value from <code>Observation.subject.identifier</code>.
+	 * </p>
+	 * <pre>
+	 * {
+	 *   "resourceType": "Observation",
+	 *   "subject": {
+	 *     "reference": "Patient/ABC",
+	 *     "identifier": {
+	 *       "system": "http://foo",
+	 *       "value": "123"
+	 *     }
+	 *   }
+	 * }
+	 * </pre>
+	 * <p>
+	 * This method is often combined with {@link #setAllowInlineMatchUrlReferences(boolean)}.
+	 * </p>
+	 * <p>
+	 * In other words if an Observation contains a reference like the one below,
+	 * and no existing resource was found that matches the given match URL, a new
+	 * one will be created and its <code>Patient.identifier</code> value will be
+	 * populated using the value from <code>Observation.subject.identifier</code>.
+	 * </p>
+	 * <pre>
+	 * {
+	 *   "resourceType": "Observation",
+	 *   "subject": {
+	 *     "reference": "Patient?identifier=http://foo|123",
+	 *     "identifier": {
+	 *       "system": "http://foo",
+	 *       "value": "123"
+	 *     }
+	 *   }
+	 * }
+	 * </pre>
+	 *
+	 * @since 4.2.0
+	 */
+	public void setPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets(boolean thePopulateIdentifierInAutoCreatedPlaceholderReferenceTargets) {
+		myPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets = thePopulateIdentifierInAutoCreatedPlaceholderReferenceTargets;
+	}
+
 	/**
 	 * If set to <code>false</code> (default is <code>true</code>) resources will be permitted to be
 	 * deleted even if other resources currently contain references to them.
@@ -1682,29 +1792,6 @@ public class DaoConfig {
 		myStoreMetaSourceInformation = theStoreMetaSourceInformation;
 	}
 
-	public enum StoreMetaSourceInformationEnum {
-		NONE(false, false),
-		SOURCE_URI(true, false),
-		REQUEST_ID(false, true),
-		SOURCE_URI_AND_REQUEST_ID(true, true);
-
-		private final boolean myStoreSourceUri;
-		private final boolean myStoreRequestId;
-
-		StoreMetaSourceInformationEnum(boolean theStoreSourceUri, boolean theStoreRequestId) {
-			myStoreSourceUri = theStoreSourceUri;
-			myStoreRequestId = theStoreRequestId;
-		}
-
-		public boolean isStoreSourceUri() {
-			return myStoreSourceUri;
-		}
-
-		public boolean isStoreRequestId() {
-			return myStoreRequestId;
-		}
-	}
-
 	/**
 	 * <p>
 	 * If set to {@code true}, ValueSets and expansions are stored in terminology tables. This is to facilitate
@@ -1820,6 +1907,29 @@ public class DaoConfig {
 		setPreExpandValueSetsDefaultCount(Math.min(getPreExpandValueSetsDefaultCount(), getPreExpandValueSetsMaxCount()));
 	}
 
+	public enum StoreMetaSourceInformationEnum {
+		NONE(false, false),
+		SOURCE_URI(true, false),
+		REQUEST_ID(false, true),
+		SOURCE_URI_AND_REQUEST_ID(true, true);
+
+		private final boolean myStoreSourceUri;
+		private final boolean myStoreRequestId;
+
+		StoreMetaSourceInformationEnum(boolean theStoreSourceUri, boolean theStoreRequestId) {
+			myStoreSourceUri = theStoreSourceUri;
+			myStoreRequestId = theStoreRequestId;
+		}
+
+		public boolean isStoreSourceUri() {
+			return myStoreSourceUri;
+		}
+
+		public boolean isStoreRequestId() {
+			return myStoreRequestId;
+		}
+	}
+
 	public enum IndexEnabledEnum {
 		ENABLED,
 		DISABLED

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ResourceExpungeService.java

@@ -159,7 +159,7 @@ class ResourceExpungeService implements IResourceExpungeService {
 	private void callHooks(RequestDetails theRequestDetails, AtomicInteger theRemainingCount, ResourceHistoryTable theVersion, IdDt theId) {
 		final AtomicInteger counter = new AtomicInteger();
 		if (JpaInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_RESOURCE, myInterceptorBroadcaster, theRequestDetails)) {
-			IFhirResourceDao resourceDao = myDaoRegistry.getResourceDao(theId.getResourceType());
+			IFhirResourceDao<?> resourceDao = myDaoRegistry.getResourceDao(theId.getResourceType());
 			IBaseResource resource = resourceDao.toResource(theVersion, false);
 			HookParams params = new HookParams()
 				.add(AtomicInteger.class, counter)

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/DaoResourceLinkResolver.java

@@ -20,6 +20,9 @@ package ca.uhn.fhir.jpa.dao.index;
  * #L%
  */
 
+import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
+import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
+import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
 import ca.uhn.fhir.context.RuntimeSearchParam;
@@ -33,19 +36,24 @@ import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
+import org.hl7.fhir.instance.model.api.IBase;
+import org.hl7.fhir.instance.model.api.IBaseReference;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.Nullable;
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
 import javax.persistence.PersistenceContextType;
+import java.util.Optional;
 
 @Service
 public class DaoResourceLinkResolver implements IResourceLinkResolver {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(DaoResourceLinkResolver.class);
-
+	@PersistenceContext(type = PersistenceContextType.TRANSACTION)
+	protected EntityManager myEntityManager;
 	@Autowired
 	private DaoConfig myDaoConfig;
 	@Autowired
@@ -55,38 +63,37 @@ public class DaoResourceLinkResolver implements IResourceLinkResolver {
 	@Autowired
 	private DaoRegistry myDaoRegistry;
 
-	@PersistenceContext(type = PersistenceContextType.TRANSACTION)
-	protected EntityManager myEntityManager;
-
 	@Override
-	public ResourceTable findTargetResource(RuntimeSearchParam theNextSpDef, String theNextPathsUnsplit, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, String theId, RequestDetails theRequest) {
+	public ResourceTable findTargetResource(RuntimeSearchParam theNextSpDef, String theNextPathsUnsplit, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, IBaseReference theReference, RequestDetails theRequest) {
 		ResourceTable target;
 		ResourcePersistentId valueOf;
+		String idPart = theNextId.getIdPart();
 		try {
-			valueOf = myIdHelperService.translateForcedIdToPid(theTypeString, theId, theRequest);
+			valueOf = myIdHelperService.translateForcedIdToPid(theTypeString, idPart, theRequest);
-			ourLog.trace("Translated {}/{} to resource PID {}", theType, theId, valueOf);
+			ourLog.trace("Translated {}/{} to resource PID {}", theType, idPart, valueOf);
 		} catch (ResourceNotFoundException e) {
+
+			Optional<ResourcePersistentId> pidOpt = createPlaceholderTargetIfConfiguredToDoSo(theType, theReference, idPart);
+			if (!pidOpt.isPresent()) {
+
 				if (myDaoConfig.isEnforceReferentialIntegrityOnWrite() == false) {
 					return null;
 				}
+
 				RuntimeResourceDefinition missingResourceDef = myContext.getResourceDefinition(theType);
 				String resName = missingResourceDef.getName();
+				throw new InvalidRequestException("Resource " + resName + "/" + idPart + " not found, specified in path: " + theNextPathsUnsplit);
 
-			if (myDaoConfig.isAutoCreatePlaceholderReferenceTargets()) {
-				IBaseResource newResource = missingResourceDef.newInstance();
-				newResource.setId(resName + "/" + theId);
-				IFhirResourceDao<IBaseResource> placeholderResourceDao = (IFhirResourceDao<IBaseResource>) myDaoRegistry.getResourceDao(newResource.getClass());
-				ourLog.debug("Automatically creating empty placeholder resource: {}", newResource.getIdElement().getValue());
-				valueOf = placeholderResourceDao.update(newResource).getEntity().getPersistentId();
-			} else {
-				throw new InvalidRequestException("Resource " + resName + "/" + theId + " not found, specified in path: " + theNextPathsUnsplit);
 			}
+
+			valueOf = pidOpt.get();
 		}
+
 		target = myEntityManager.find(ResourceTable.class, valueOf.getIdAsLong());
 		RuntimeResourceDefinition targetResourceDef = myContext.getResourceDefinition(theType);
 		if (target == null) {
 			String resName = targetResourceDef.getName();
-			throw new InvalidRequestException("Resource " + resName + "/" + theId + " not found, specified in path: " + theNextPathsUnsplit);
+			throw new InvalidRequestException("Resource " + resName + "/" + idPart + " not found, specified in path: " + theNextPathsUnsplit);
 		}
 
 		ourLog.trace("Resource PID {} is of type {}", valueOf, target.getResourceType());
@@ -98,7 +105,7 @@ public class DaoResourceLinkResolver implements IResourceLinkResolver {
 
 		if (target.getDeleted() != null) {
 			String resName = targetResourceDef.getName();
-			throw new InvalidRequestException("Resource " + resName + "/" + theId + " is deleted, specified in path: " + theNextPathsUnsplit);
+			throw new InvalidRequestException("Resource " + resName + "/" + idPart + " is deleted, specified in path: " + theNextPathsUnsplit);
 		}
 
 		if (!theNextSpDef.hasTargets() && theNextSpDef.getTargets().contains(theTypeString)) {
@@ -107,8 +114,60 @@ public class DaoResourceLinkResolver implements IResourceLinkResolver {
 		return target;
 	}
 
+	/**
+	 * @param theIdToAssignToPlaceholder If specified, the placeholder resource created will be given a specific ID
+	 */
+	public <T extends IBaseResource> Optional<ResourcePersistentId> createPlaceholderTargetIfConfiguredToDoSo(Class<T> theType, IBaseReference theReference, @Nullable String theIdToAssignToPlaceholder) {
+		ResourcePersistentId valueOf = null;
+
+		if (myDaoConfig.isAutoCreatePlaceholderReferenceTargets()) {
+			RuntimeResourceDefinition missingResourceDef = myContext.getResourceDefinition(theType);
+			String resName = missingResourceDef.getName();
+
+			@SuppressWarnings("unchecked")
+			T newResource = (T) missingResourceDef.newInstance();
+
+			IFhirResourceDao<T> placeholderResourceDao = myDaoRegistry.getResourceDao(theType);
+			ourLog.debug("Automatically creating empty placeholder resource: {}", newResource.getIdElement().getValue());
+
+			if (myDaoConfig.isPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets()) {
+				tryToCopyIdentifierFromReferenceToTargetResource(theReference, missingResourceDef, newResource);
+			}
+
+			if (theIdToAssignToPlaceholder != null) {
+				newResource.setId(resName + "/" + theIdToAssignToPlaceholder);
+				valueOf = placeholderResourceDao.update(newResource).getEntity().getPersistentId();
+			} else {
+				valueOf = placeholderResourceDao.create(newResource).getEntity().getPersistentId();
+			}
+		}
+
+		return Optional.ofNullable(valueOf);
+	}
+
+	private <T extends IBaseResource> void tryToCopyIdentifierFromReferenceToTargetResource(IBaseReference theSourceReference, RuntimeResourceDefinition theTargetResourceDef, T theTargetResource) {
+		boolean referenceHasIdentifier = theSourceReference.hasIdentifier();
+		if (referenceHasIdentifier) {
+			BaseRuntimeChildDefinition targetIdentifier = theTargetResourceDef.getChildByName("identifier");
+			if (targetIdentifier != null) {
+				BaseRuntimeElementDefinition<?> identifierElement = targetIdentifier.getChildByName("identifier");
+				String identifierElementName = identifierElement.getName();
+				boolean targetHasIdentifierElement = identifierElementName.equals("Identifier");
+				if (targetHasIdentifierElement) {
+
+					BaseRuntimeElementCompositeDefinition<?> referenceElement = (BaseRuntimeElementCompositeDefinition<?>) myContext.getElementDefinition(theSourceReference.getClass());
+					BaseRuntimeChildDefinition referenceIdentifierChild = referenceElement.getChildByName("identifier");
+					Optional<IBase> identifierOpt = referenceIdentifierChild.getAccessor().getFirstValueOrNull(theSourceReference);
+					identifierOpt.ifPresent(theIBase -> targetIdentifier.getMutator().addValue(theTargetResource, theIBase));
+
+				}
+			}
+		}
+	}
+
 	@Override
 	public void validateTypeOrThrowException(Class<? extends IBaseResource> theType) {
 		myDaoRegistry.getDaoOrThrowException(theType);
 	}
+
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/SearchParamWithInlineReferencesExtractor.java

@@ -26,8 +26,8 @@ import ca.uhn.fhir.context.RuntimeSearchParam;
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
 import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.dao.MatchResourceUrlService;
-import ca.uhn.fhir.jpa.model.cross.ResourcePersistentId;
 import ca.uhn.fhir.jpa.dao.data.IResourceIndexedCompositeStringUniqueDao;
+import ca.uhn.fhir.jpa.model.cross.ResourcePersistentId;
 import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedCompositeStringUnique;
 import ca.uhn.fhir.jpa.model.entity.ResourceLink;
@@ -61,6 +61,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
@@ -81,17 +82,16 @@ public class SearchParamWithInlineReferencesExtractor {
 	@Autowired
 	private ISearchParamRegistry mySearchParamRegistry;
 	@Autowired
-	SearchParamExtractorService mySearchParamExtractorService;
+	private SearchParamExtractorService mySearchParamExtractorService;
 	@Autowired
-	ResourceLinkExtractor myResourceLinkExtractor;
+	private ResourceLinkExtractor myResourceLinkExtractor;
 	@Autowired
-	DaoResourceLinkResolver myDaoResourceLinkResolver;
+	private DaoResourceLinkResolver myDaoResourceLinkResolver;
 	@Autowired
-	DaoSearchParamSynchronizer myDaoSearchParamSynchronizer;
+	private DaoSearchParamSynchronizer myDaoSearchParamSynchronizer;
 	@Autowired
 	private IResourceIndexedCompositeStringUniqueDao myResourceIndexedCompositeStringUniqueDao;
 
-
 	@PersistenceContext(type = PersistenceContextType.TRANSACTION)
 	protected EntityManager myEntityManager;
 
@@ -246,16 +246,25 @@ public class SearchParamWithInlineReferencesExtractor {
 				}
 				Class<? extends IBaseResource> matchResourceType = matchResourceDef.getImplementingClass();
 				Set<ResourcePersistentId> matches = myMatchResourceUrlService.processMatchUrl(nextIdText, matchResourceType, theRequest);
+
+				ResourcePersistentId match;
 				if (matches.isEmpty()) {
+
+					Optional<ResourcePersistentId> placeholderOpt = myDaoResourceLinkResolver.createPlaceholderTargetIfConfiguredToDoSo(matchResourceType, nextRef, null);
+					if (placeholderOpt.isPresent()) {
+						match = placeholderOpt.get();
+					} else {
 						String msg = myContext.getLocalizer().getMessage(BaseHapiFhirDao.class, "invalidMatchUrlNoMatches", nextId.getValue());
 						throw new ResourceNotFoundException(msg);
 					}
-				if (matches.size() > 1) {
+				} else if (matches.size() > 1) {
 					String msg = myContext.getLocalizer().getMessage(BaseHapiFhirDao.class, "invalidMatchUrlMultipleMatches", nextId.getValue());
 					throw new PreconditionFailedException(msg);
+				} else {
+					match = matches.iterator().next();
 				}
-				ResourcePersistentId next = matches.iterator().next();
+
-				String newId = myIdHelperService.translatePidIdToForcedId(resourceTypeString, next);
+				String newId = myIdHelperService.translatePidIdToForcedId(resourceTypeString, match);
 				ourLog.debug("Replacing inline match URL[{}] with ID[{}}", nextId.getValue(), newId);
 				nextRef.setReference(newId);
 			}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoCreatePlaceholdersR4Test.java

@@ -8,10 +8,14 @@ import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.util.TestUtil;
+import com.google.common.collect.Sets;
 import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.r4.model.AuditEvent;
 import org.hl7.fhir.r4.model.IdType;
 import org.hl7.fhir.r4.model.Observation;
 import org.hl7.fhir.r4.model.Observation.ObservationStatus;
+import org.hl7.fhir.r4.model.Patient;
+import org.hl7.fhir.r4.model.Reference;
 import org.hl7.fhir.r4.model.Task;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -21,9 +25,12 @@ import java.util.List;
 
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.Matchers.contains;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
 
-@SuppressWarnings({"unchecked", "deprecation"})
+@SuppressWarnings({"ConstantConditions"})
 public class FhirResourceDaoCreatePlaceholdersR4Test extends BaseJpaR4Test {
 
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoCreatePlaceholdersR4Test.class);
@@ -32,6 +39,9 @@ public class FhirResourceDaoCreatePlaceholdersR4Test extends BaseJpaR4Test {
 	public final void afterResetDao() {
 		myDaoConfig.setAutoCreatePlaceholderReferenceTargets(new DaoConfig().isAutoCreatePlaceholderReferenceTargets());
 		myDaoConfig.setResourceClientIdStrategy(new DaoConfig().getResourceClientIdStrategy());
+		myDaoConfig.setPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets(new DaoConfig().isPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets());
+		myDaoConfig.setBundleTypesAllowedForStorage(new DaoConfig().getBundleTypesAllowedForStorage());
+
 	}
 
 	@Test
@@ -163,6 +173,90 @@ public class FhirResourceDaoCreatePlaceholdersR4Test extends BaseJpaR4Test {
 		assertEquals("Patient/999999999999999", outcome.getResources(0,1).get(0).getIdElement().toUnqualifiedVersionless().getValue());
 	}
 
+	@Test
+	public void testCreatePlaceholderWithMatchUrl_IdentifierNotCopiedByDefault() {
+		myDaoConfig.setAutoCreatePlaceholderReferenceTargets(true);
+		myDaoConfig.setAllowInlineMatchUrlReferences(true);
+
+		Observation obsToCreate = new Observation();
+		obsToCreate.setStatus(ObservationStatus.FINAL);
+		obsToCreate.getSubject().setReference("Patient?identifier=http://foo|123");
+		obsToCreate.getSubject().getIdentifier().setSystem("http://foo").setValue("123");
+		IIdType id = myObservationDao.create(obsToCreate, mySrd).getId();
+
+		Observation createdObs = myObservationDao.read(id);
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(createdObs));
+
+		Patient patient = myPatientDao.read(new IdType(createdObs.getSubject().getReference()));
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(patient));
+		assertEquals(0, patient.getIdentifier().size());
+	}
+
+
+	@Test
+	public void testCreatePlaceholderWithMatchUrl_IdentifierCopied_NotPreExisting() {
+		myDaoConfig.setAutoCreatePlaceholderReferenceTargets(true);
+		myDaoConfig.setAllowInlineMatchUrlReferences(true);
+		myDaoConfig.setPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets(true);
+
+		Observation obsToCreate = new Observation();
+		obsToCreate.setStatus(ObservationStatus.FINAL);
+		obsToCreate.getSubject().setReference("Patient?identifier=http://foo|123");
+		obsToCreate.getSubject().getIdentifier().setSystem("http://foo").setValue("123");
+		IIdType id = myObservationDao.create(obsToCreate, mySrd).getId();
+
+		Observation createdObs = myObservationDao.read(id);
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(createdObs));
+
+		Patient patient = myPatientDao.read(new IdType(createdObs.getSubject().getReference()));
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(patient));
+		assertEquals(1, patient.getIdentifier().size());
+		assertEquals("http://foo", patient.getIdentifier().get(0).getSystem());
+		assertEquals("123", patient.getIdentifier().get(0).getValue());
+	}
+
+	@Test
+	public void testCreatePlaceholderWithMatchUrl_IdentifierNotCopiedBecauseNoFieldMatches() {
+		myDaoConfig.setAutoCreatePlaceholderReferenceTargets(true);
+		myDaoConfig.setAllowInlineMatchUrlReferences(true);
+		myDaoConfig.setPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets(true);
+		myDaoConfig.setBundleTypesAllowedForStorage(Sets.newHashSet(""));
+
+		AuditEvent eventToCreate = new AuditEvent();
+		Reference what = eventToCreate.addEntity().getWhat();
+		what.setReference("Bundle/ABC");
+		what.getIdentifier().setSystem("http://foo");
+		what.getIdentifier().setValue("123");
+		IIdType id = myAuditEventDao.create(eventToCreate, mySrd).getId();
+
+		AuditEvent createdEvent = myAuditEventDao.read(id);
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(createdEvent));
+
+	}
+
+	@Test
+	public void testCreatePlaceholderWithMatchUrl_PreExisting() {
+		myDaoConfig.setAutoCreatePlaceholderReferenceTargets(true);
+		myDaoConfig.setAllowInlineMatchUrlReferences(true);
+		myDaoConfig.setPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets(true);
+
+		Patient patient = new Patient();
+		patient.setId("ABC");
+		patient.addIdentifier().setSystem("http://foo").setValue("123");
+		myPatientDao.update(patient);
+
+		Observation obsToCreate = new Observation();
+		obsToCreate.setStatus(ObservationStatus.FINAL);
+		obsToCreate.getSubject().setReference("Patient?identifier=http://foo|123");
+		obsToCreate.getSubject().getIdentifier().setSystem("http://foo").setValue("123");
+		IIdType id = myObservationDao.create(obsToCreate, mySrd).getId();
+
+		Observation createdObs = myObservationDao.read(id);
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(createdObs));
+		assertEquals("Patient/ABC", obsToCreate.getSubject().getReference());
+
+	}
+
 	@AfterClass
 	public static void afterClassClearContext() {
 		TestUtil.clearAllStaticFieldsForUnitTest();

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/IResourceLinkResolver.java

@@ -23,11 +23,12 @@ package ca.uhn.fhir.jpa.searchparam.extractor;
 import ca.uhn.fhir.context.RuntimeSearchParam;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
+import org.hl7.fhir.instance.model.api.IBaseReference;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 
 public interface IResourceLinkResolver {
-	ResourceTable findTargetResource(RuntimeSearchParam theNextSpDef, String theNextPathsUnsplit, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, String theId, RequestDetails theRequest);
+	ResourceTable findTargetResource(RuntimeSearchParam theNextSpDef, String theNextPathsUnsplit, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, IBaseReference theReference, RequestDetails theRequest);
 
 	void validateTypeOrThrowException(Class<? extends IBaseResource> theType);
 }

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/ResourceLinkExtractor.java

@@ -39,7 +39,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.util.Date;
-import java.util.Map;
 
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
@@ -73,8 +72,8 @@ public class ResourceLinkExtractor {
 	}
 
 	private void extractResourceLinks(ResourceIndexedSearchParams theParams, ResourceTable theEntity, Date theUpdateTime, IResourceLinkResolver theResourceLinkResolver, RuntimeSearchParam theRuntimeSearchParam, PathAndRef thePathAndRef, boolean theFailOnInvalidReference, RequestDetails theRequest) {
-		IBaseReference nextObject = thePathAndRef.getRef();
+		IBaseReference nextReference = thePathAndRef.getRef();
-		IIdType nextId = nextObject.getReferenceElement();
+		IIdType nextId = nextReference.getReferenceElement();
 		String path = thePathAndRef.getPath();
 
 		/*
@@ -82,8 +81,8 @@ public class ResourceLinkExtractor {
 		 * programmatically with a Bundle (not through the FHIR REST API)
 		 * but Smile does this
 		 */
-		if (nextId.isEmpty() && nextObject.getResource() != null) {
+		if (nextId.isEmpty() && nextReference.getResource() != null) {
-			nextId = nextObject.getResource().getIdElement();
+			nextId = nextReference.getResource().getIdElement();
 		}
 
 		theParams.myPopulatedResourceLinkParameters.add(thePathAndRef.getSearchParamName());
@@ -152,15 +151,15 @@ public class ResourceLinkExtractor {
 		}
 
 		theResourceLinkResolver.validateTypeOrThrowException(type);
-		ResourceLink resourceLink = createResourceLink(theEntity, theUpdateTime, theResourceLinkResolver, theRuntimeSearchParam, path, thePathAndRef, nextId, typeString, type, id, theRequest);
+		ResourceLink resourceLink = createResourceLink(theEntity, theUpdateTime, theResourceLinkResolver, theRuntimeSearchParam, path, thePathAndRef, nextId, typeString, type, nextReference, theRequest);
 		if (resourceLink == null) {
 			return;
 		}
 		theParams.myLinks.add(resourceLink);
 	}
 
-	private ResourceLink createResourceLink(ResourceTable theEntity, Date theUpdateTime, IResourceLinkResolver theResourceLinkResolver, RuntimeSearchParam nextSpDef, String theNextPathsUnsplit, PathAndRef nextPathAndRef, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, String theId, RequestDetails theRequest) {
+	private ResourceLink createResourceLink(ResourceTable theEntity, Date theUpdateTime, IResourceLinkResolver theResourceLinkResolver, RuntimeSearchParam nextSpDef, String theNextPathsUnsplit, PathAndRef nextPathAndRef, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, IBaseReference theReference, RequestDetails theRequest) {
-		ResourceTable targetResource = theResourceLinkResolver.findTargetResource(nextSpDef, theNextPathsUnsplit, theNextId, theTypeString, theType, theId, theRequest);
+		ResourceTable targetResource = theResourceLinkResolver.findTargetResource(nextSpDef, theNextPathsUnsplit, theNextId, theTypeString, theType, theReference, theRequest);
 
 		if (targetResource == null) {
 			return null;
@@ -169,7 +168,4 @@ public class ResourceLinkExtractor {
 		return new ResourceLink(nextPathAndRef.getPath(), theEntity, targetResource, theUpdateTime);
 	}
 
-	private String toResourceName(Class<? extends IBaseResource> theResourceType) {
-		return myContext.getResourceDefinition(theResourceType).getName();
-	}
 }

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/matcher/InlineResourceLinkResolver.java

@@ -25,6 +25,7 @@ import ca.uhn.fhir.jpa.model.entity.ForcedId;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.searchparam.extractor.IResourceLinkResolver;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
+import org.hl7.fhir.instance.model.api.IBaseReference;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.springframework.stereotype.Service;
@@ -33,7 +34,7 @@ import org.springframework.stereotype.Service;
 public class InlineResourceLinkResolver implements IResourceLinkResolver {
 
 	@Override
-	public ResourceTable findTargetResource(RuntimeSearchParam theNextSpDef, String theNextPathsUnsplit, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, String theId, RequestDetails theRequest) {
+	public ResourceTable findTargetResource(RuntimeSearchParam theNextSpDef, String theNextPathsUnsplit, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, IBaseReference theReference, RequestDetails theRequest) {
 		ResourceTable target;
 		target = new ResourceTable();
 		target.setResourceType(theTypeString);
@@ -41,7 +42,7 @@ public class InlineResourceLinkResolver implements IResourceLinkResolver {
 			target.setId(theNextId.getIdPartAsLong());
 		} else {
 			ForcedId forcedId = new ForcedId();
-			forcedId.setForcedId(theId);
+			forcedId.setForcedId(theNextId.getIdPart());
 			target.setForcedId(forcedId);
 		}
 		return target;

hapi-fhir-structures-dstu2/src/test/java/ca/uhn/fhir/rest/client/ClientServerValidationDstu2Test.java

@@ -265,7 +265,7 @@ public class ClientServerValidationDstu2Test {
 
 	@Test
 	public void testServerReturnsWrongVersionForDstu2() throws Exception {
-		String wrongFhirVersion = "3.0.1";
+		String wrongFhirVersion = FhirVersionEnum.DSTU3.getFhirVersionString();
 		assertThat(wrongFhirVersion, is(FhirVersionEnum.DSTU3.getFhirVersionString())); // asserting that what we assume to be the DSTU3 FHIR version is still correct
 		Conformance conf = new Conformance();
 		conf.setFhirVersion(wrongFhirVersion);
@@ -285,7 +285,7 @@ public class ClientServerValidationDstu2Test {
 			fail();
 		} catch (FhirClientInappropriateForServerException e) {
 			String out = e.toString();
-			String want = "The server at base URL \"http://foo/metadata\" returned a conformance statement indicating that it supports FHIR version \"3.0.1\" which corresponds to DSTU3, but this client is configured to use DSTU2 (via the FhirContext)";
+			String want = "The server at base URL \"http://foo/metadata\" returned a conformance statement indicating that it supports FHIR version \"" + wrongFhirVersion + "\" which corresponds to DSTU3, but this client is configured to use DSTU2 (via the FhirContext)";
 			ourLog.info(out);
 			ourLog.info(want);
 			assertThat(out, containsString(want));