Correct two JPA processing bugs (#1801)

* Work on search params on contained

* Add workaround for stored decimals with leading decimal point

* Add changelog

* Cleanup

* Test fix

* Test fix

* One more test fix

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/BaseParser.java

@@ -58,6 +58,14 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
 @SuppressWarnings("WeakerAccess")
 public abstract class BaseParser implements IParser {
 
+	/**
+	 * Any resources that were created by the parser (i.e. by parsing a serialized resource) will have
+	 * a {@link IBaseResource#getUserData(String) user data} property with this key.
+	 *
+	 * @since 5.0.0
+	 */
+	public static final String RESOURCE_CREATED_BY_PARSER = BaseParser.class.getName() + "_" + "RESOURCE_CREATED_BY_PARSER";
+
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseParser.class);
 
 	private static final Set<String> notEncodeForContainedResource = new HashSet<>(Arrays.asList("security", "versionId", "lastUpdated"));

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/ParserState.java

@@ -43,6 +43,7 @@ import ca.uhn.fhir.util.ReflectionUtil;
 import ca.uhn.fhir.util.XmlUtil;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
+import org.apache.commons.lang3.math.NumberUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.hl7.fhir.instance.model.api.*;
 
@@ -456,7 +457,7 @@ class ParserState<T> {
 					RuntimePrimitiveDatatypeDefinition primitiveTarget = (RuntimePrimitiveDatatypeDefinition) target;
 					IPrimitiveType<?> newChildInstance = newPrimitiveInstance(myDefinition, primitiveTarget);
 					myDefinition.getMutator().addValue(myParentInstance, newChildInstance);
-					PrimitiveState newState = new PrimitiveState(getPreResourceState(), newChildInstance, theLocalPart);
+					PrimitiveState newState = new PrimitiveState(getPreResourceState(), newChildInstance, theLocalPart, primitiveTarget.getName());
 					push(newState);
 					return;
 				}
@@ -495,10 +496,10 @@ class ParserState<T> {
 
 	private class ElementCompositeState extends BaseState {
 
-		private BaseRuntimeElementCompositeDefinition<?> myDefinition;
+		private final BaseRuntimeElementCompositeDefinition<?> myDefinition;
-		private IBase myInstance;
+		private final IBase myInstance;
-		private Set<String> myParsedNonRepeatableNames = new HashSet<>();
+		private final Set<String> myParsedNonRepeatableNames = new HashSet<>();
-		private String myElementName;
+		private final String myElementName;
 
 		ElementCompositeState(PreResourceState thePreResourceState, String theElementName, BaseRuntimeElementCompositeDefinition<?> theDef, IBase theInstance) {
 			super(thePreResourceState);
@@ -585,7 +586,7 @@ class ParserState<T> {
 					IPrimitiveType<?> newChildInstance;
 					newChildInstance = getPrimitiveInstance(child, primitiveTarget, theChildName);
 					child.getMutator().addValue(myInstance, newChildInstance);
-					PrimitiveState newState = new PrimitiveState(getPreResourceState(), newChildInstance, theChildName);
+					PrimitiveState newState = new PrimitiveState(getPreResourceState(), newChildInstance, theChildName, primitiveTarget.getName());
 					push(newState);
 					return;
 				}
@@ -668,7 +669,7 @@ class ParserState<T> {
 
 	public class ElementIdState extends BaseState {
 
-		private IBaseElement myElement;
+		private final IBaseElement myElement;
 
 		ElementIdState(ParserState<T>.PreResourceState thePreResourceState, IBaseElement theElement) {
 			super(thePreResourceState);
@@ -689,7 +690,7 @@ class ParserState<T> {
 
 	private class ExtensionState extends BaseState {
 
-		private IBaseExtension<?, ?> myExtension;
+		private final IBaseExtension<?, ?> myExtension;
 
 		ExtensionState(PreResourceState thePreResourceState, IBaseExtension<?, ?> theExtension) {
 			super(thePreResourceState);
@@ -752,7 +753,7 @@ class ParserState<T> {
 						RuntimePrimitiveDatatypeDefinition primitiveTarget = (RuntimePrimitiveDatatypeDefinition) target;
 						IPrimitiveType<?> newChildInstance = newInstance(primitiveTarget);
 						myExtension.setValue(newChildInstance);
-						PrimitiveState newState = new PrimitiveState(getPreResourceState(), newChildInstance, theLocalPart);
+						PrimitiveState newState = new PrimitiveState(getPreResourceState(), newChildInstance, theLocalPart, primitiveTarget.getName());
 						push(newState);
 						return;
 					}
@@ -782,7 +783,7 @@ class ParserState<T> {
 
 	public class IdentifiableElementIdState extends BaseState {
 
-		private IIdentifiableElement myElement;
+		private final IIdentifiableElement myElement;
 
 		public IdentifiableElementIdState(ParserState<T>.PreResourceState thePreResourceState, IIdentifiableElement theElement) {
 			super(thePreResourceState);
@@ -802,7 +803,7 @@ class ParserState<T> {
 	}
 
 	private class MetaElementState extends BaseState {
-		private ResourceMetadataMap myMap;
+		private final ResourceMetadataMap myMap;
 
 		public MetaElementState(ParserState<T>.PreResourceState thePreResourceState, ResourceMetadataMap theMap) {
 			super(thePreResourceState);
@@ -824,7 +825,7 @@ class ParserState<T> {
 					break;
 				case "lastUpdated":
 					InstantDt updated = new InstantDt();
-					push(new PrimitiveState(getPreResourceState(), updated, theLocalPart));
+					push(new PrimitiveState(getPreResourceState(), updated, theLocalPart, "instant"));
 					myMap.put(ResourceMetadataKeyEnum.UPDATED, updated);
 					break;
 				case "security":
@@ -850,7 +851,7 @@ class ParserState<T> {
 						newProfiles = new ArrayList<>(1);
 					}
 					IdDt profile = new IdDt();
-					push(new PrimitiveState(getPreResourceState(), profile, theLocalPart));
+					push(new PrimitiveState(getPreResourceState(), profile, theLocalPart, "id"));
 					newProfiles.add(profile);
 					myMap.put(ResourceMetadataKeyEnum.PROFILES, Collections.unmodifiableList(newProfiles));
 					break;
@@ -1048,6 +1049,8 @@ class ParserState<T> {
 				}
 			}
 
+			myInstance.setUserData(BaseParser.RESOURCE_CREATED_BY_PARSER, Boolean.TRUE);
+
 			populateTarget();
 		}
 
@@ -1269,41 +1272,50 @@ class ParserState<T> {
 
 	private class PrimitiveState extends BaseState {
 		private final String myChildName;
+		private final String myTypeName;
 		private IPrimitiveType<?> myInstance;
 
-		PrimitiveState(PreResourceState thePreResourceState, IPrimitiveType<?> theInstance, String theChildName) {
+		PrimitiveState(PreResourceState thePreResourceState, IPrimitiveType<?> theInstance, String theChildName, String theTypeName) {
 			super(thePreResourceState);
 			myInstance = theInstance;
 			myChildName = theChildName;
+			myTypeName = theTypeName;
 		}
 
 		@Override
 		public void attributeValue(String theName, String theValue) throws DataFormatException {
+			String value = theValue;
 			if ("value".equals(theName)) {
-				if ("".equals(theValue)) {
+				if ("".equals(value)) {
 					ParseLocation location = ParseLocation.fromElementName(myChildName);
-					myErrorHandler.invalidValue(location, theValue, "Attribute value must not be empty (\"\")");
+					myErrorHandler.invalidValue(location, value, "Attribute value must not be empty (\"\")");
 				} else {
+					if ("decimal".equals(myTypeName)) {
+						if (value != null && value.startsWith(".") && NumberUtils.isDigits(value.substring(1))) {
+							value = "0" + value;
+						}
+					}
+
 					try {
-						myInstance.setValueAsString(theValue);
+						myInstance.setValueAsString(value);
 					} catch (DataFormatException | IllegalArgumentException e) {
 						ParseLocation location = ParseLocation.fromElementName(myChildName);
-						myErrorHandler.invalidValue(location, theValue, e.getMessage());
+						myErrorHandler.invalidValue(location, value, e.getMessage());
 					}
 				}
 			} else if ("id".equals(theName)) {
 				if (myInstance instanceof IIdentifiableElement) {
-					((IIdentifiableElement) myInstance).setElementSpecificId(theValue);
+					((IIdentifiableElement) myInstance).setElementSpecificId(value);
 				} else if (myInstance instanceof IBaseElement) {
-					((IBaseElement) myInstance).setId(theValue);
+					((IBaseElement) myInstance).setId(value);
 				} else if (myInstance instanceof IBaseResource) {
-					new IdDt(theValue).applyTo((org.hl7.fhir.instance.model.api.IBaseResource) myInstance);
+					new IdDt(value).applyTo((org.hl7.fhir.instance.model.api.IBaseResource) myInstance);
 				} else {
 					ParseLocation location = ParseLocation.fromElementName(myChildName);
 					myErrorHandler.unknownAttribute(location, theName);
 				}
 			} else {
-				super.attributeValue(theName, theValue);
+				super.attributeValue(theName, value);
 			}
 		}
 
@@ -1348,7 +1360,7 @@ class ParserState<T> {
 		@Override
 		public void enteringNewElement(String theNamespace, String theChildName) throws DataFormatException {
 			if ("id".equals(theChildName)) {
-				push(new PrimitiveState(getPreResourceState(), myInstance.getId(), theChildName));
+				push(new PrimitiveState(getPreResourceState(), myInstance.getId(), theChildName, "id"));
 			} else if ("meta".equals(theChildName)) {
 				push(new MetaElementState(getPreResourceState(), myInstance.getResourceMetadata()));
 			} else {

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/json/jackson/JacksonStructure.java

@@ -35,7 +35,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.DecimalNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.jena.tdb.setup.BuilderStdDB;
 
 import java.io.IOException;
 import java.io.PushbackReader;

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/4_3_0/1801-allow-parsing-contained-in-jpa.yaml

@@ -0,0 +1,7 @@
+---
+type: fix
+issue: 1801
+title: "When invoking JPA DAO methods programatically to store a resource (as opposed to using the FHIR REST API), if
+  the resource being stored had any contained resources, these would sometimes not be visible to the search parameter
+  indexer, leading to missing search params. This is a very fringe use case, but a workaround has been put in place to
+  solve it."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/4_3_0/1801-handle-persisted-leading-decimals.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 1801
+title: "In previous versions of HAPI FHIR, the server incorrectly silently accepted decimal numbers in JSON with no
+   leading numbers (e.g. `.123`). These will now be rejected by the JSON parser. Any values with this string that
+   have previously been stored in the JPA server database will now automatically convert the value to `0.123`."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java

@@ -11,6 +11,7 @@ import ca.uhn.fhir.jpa.bulk.BulkDataExportProvider;
 import ca.uhn.fhir.jpa.bulk.BulkDataExportSvcImpl;
 import ca.uhn.fhir.jpa.bulk.IBulkDataExportSvc;
 import ca.uhn.fhir.jpa.dao.ISearchBuilder;
+import ca.uhn.fhir.jpa.dao.index.DaoResourceLinkResolver;
 import ca.uhn.fhir.jpa.entity.Search;
 import ca.uhn.fhir.jpa.graphql.JpaStorageServices;
 import ca.uhn.fhir.jpa.interceptor.JpaConsentContextServices;
@@ -33,6 +34,7 @@ import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc;
 import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
 import ca.uhn.fhir.jpa.search.reindex.ResourceReindexingSvcImpl;
 import ca.uhn.fhir.jpa.searchparam.config.SearchParamConfig;
+import ca.uhn.fhir.jpa.searchparam.extractor.IResourceLinkResolver;
 import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.jpa.searchparam.registry.SearchParamRegistryImpl;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
@@ -46,6 +48,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Lazy;
+import org.springframework.context.annotation.Primary;
 import org.springframework.context.annotation.Scope;
 import org.springframework.core.env.Environment;
 import org.springframework.core.task.AsyncTaskExecutor;
@@ -158,6 +161,12 @@ public abstract class BaseConfig {
 		return new BinaryStorageInterceptor();
 	}
 
+	@Bean
+	@Primary
+	public IResourceLinkResolver daoResourceLinkResolver() {
+		return new DaoResourceLinkResolver();
+	}
+
 	@Bean
 	public ISearchCacheSvc searchCacheSvc() {
 		return new DatabaseSearchCacheSvcImpl();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java

@@ -1,6 +1,13 @@
 package ca.uhn.fhir.jpa.dao;
 
-import ca.uhn.fhir.context.*;
+import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
+import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
+import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.context.FhirVersionEnum;
+import ca.uhn.fhir.context.RuntimeChildResourceDefinition;
+import ca.uhn.fhir.context.RuntimeResourceDefinition;
+import ca.uhn.fhir.context.RuntimeSearchParam;
 import ca.uhn.fhir.interceptor.api.HookParams;
 import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
 import ca.uhn.fhir.interceptor.api.Pointcut;
@@ -8,7 +15,12 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.dao.IDao;
 import ca.uhn.fhir.jpa.api.dao.IJpaDao;
-import ca.uhn.fhir.jpa.dao.data.*;
+import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
+import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
+import ca.uhn.fhir.jpa.dao.data.IResourceHistoryTableDao;
+import ca.uhn.fhir.jpa.dao.data.IResourceProvenanceDao;
+import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
+import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
 import ca.uhn.fhir.jpa.dao.expunge.ExpungeService;
 import ca.uhn.fhir.jpa.dao.index.DaoSearchParamSynchronizer;
 import ca.uhn.fhir.jpa.dao.index.IdHelperService;
@@ -22,7 +34,6 @@ import ca.uhn.fhir.jpa.model.entity.*;
 import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
 import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
 import ca.uhn.fhir.jpa.model.util.JpaConstants;
-import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
 import ca.uhn.fhir.jpa.search.PersistedJpaBundleProviderFactory;
 import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc;
 import ca.uhn.fhir.jpa.searchparam.ResourceMetaParams;
@@ -78,7 +89,11 @@ import org.springframework.transaction.support.TransactionSynchronizationAdapter
 import org.springframework.transaction.support.TransactionSynchronizationManager;
 
 import javax.annotation.PostConstruct;
-import javax.persistence.*;
+import javax.persistence.EntityManager;
+import javax.persistence.NoResultException;
+import javax.persistence.PersistenceContext;
+import javax.persistence.PersistenceContextType;
+import javax.persistence.TypedQuery;
 import javax.persistence.criteria.CriteriaBuilder;
 import javax.persistence.criteria.CriteriaQuery;
 import javax.persistence.criteria.Root;
@@ -87,7 +102,12 @@ import javax.xml.stream.events.XMLEvent;
 import java.util.*;
 import java.util.Map.Entry;
 
-import static org.apache.commons.lang3.StringUtils.*;
+import static org.apache.commons.lang3.StringUtils.defaultIfBlank;
+import static org.apache.commons.lang3.StringUtils.defaultString;
+import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.commons.lang3.StringUtils.left;
+import static org.apache.commons.lang3.StringUtils.trim;
 
 /*
  * #%L
@@ -850,8 +870,9 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> extends BaseStora
 		// 4. parse the text to FHIR
 		R retVal;
 		if (resourceEncoding != ResourceEncodingEnum.DEL) {
-			IParser parser = resourceEncoding.newParser(getContext(theEntity.getFhirVersion()));
+
-			parser.setParserErrorHandler(new LenientErrorHandler(false).setErrorOnInvalidValue(false));
+			LenientErrorHandler errorHandler = new LenientErrorHandler(false).setErrorOnInvalidValue(false);
+			IParser parser = new TolerantJsonParser(getContext(theEntity.getFhirVersion()), errorHandler);
 
 			try {
 				retVal = parser.parseResource(resourceType, resourceText);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/TolerantJsonParser.java

@@ -0,0 +1,57 @@
+package ca.uhn.fhir.jpa.dao;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.parser.DataFormatException;
+import ca.uhn.fhir.parser.IParserErrorHandler;
+import ca.uhn.fhir.parser.JsonParser;
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.commons.lang3.StringUtils.defaultString;
+
+class TolerantJsonParser extends JsonParser {
+
+	private static final Logger ourLog = LoggerFactory.getLogger(TolerantJsonParser.class);
+
+	TolerantJsonParser(FhirContext theContext, IParserErrorHandler theParserErrorHandler) {
+		super(theContext, theParserErrorHandler);
+	}
+
+	@Override
+	public <T extends IBaseResource> T parseResource(Class<T> theResourceType, String theMessageString) {
+		try {
+			return super.parseResource(theResourceType, theMessageString);
+		} catch (DataFormatException e) {
+			if (defaultString(e.getMessage()).contains("Unexpected character ('.' (code 46))")) {
+
+				/*
+				 * The following is a hacky and gross workaround until the following PR is hopefully merged:
+				 * https://github.com/FasterXML/jackson-core/pull/611
+				 *
+				 * The issue this solves is that under Gson it was possible to store JSON containing
+				 * decimal numbers with no leading integer, e.g. .123
+				 *
+				 * These don't parse in Jackson, meaning we can be stuck with data in the database
+				 * that can't be loaded back out.
+				 *
+				 * Note that if we fix this in the future to rely on Jackson natively handing this
+				 * nicely we may or may not be able to remove some code from
+				 * ParserState.Primitive state too.
+				 */
+
+				Gson gson = new Gson();
+
+				JsonObject object = gson.fromJson(theMessageString, JsonObject.class);
+				String corrected = gson.toJson(object);
+
+				return super.parseResource(theResourceType, corrected);
+
+			}
+			throw e;
+		}
+	}
+
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/DaoResourceLinkResolver.java

@@ -49,7 +49,6 @@ import javax.persistence.PersistenceContext;
 import javax.persistence.PersistenceContextType;
 import java.util.Optional;
 
-@Service
 public class DaoResourceLinkResolver implements IResourceLinkResolver {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(DaoResourceLinkResolver.class);
 	@PersistenceContext(type = PersistenceContextType.TRANSACTION)

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/SearchParamWithInlineReferencesExtractor.java

@@ -23,8 +23,8 @@ package ca.uhn.fhir.jpa.dao.index;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
 import ca.uhn.fhir.context.RuntimeSearchParam;
-import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
+import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
 import ca.uhn.fhir.jpa.dao.MatchResourceUrlService;
 import ca.uhn.fhir.jpa.dao.data.IResourceIndexedCompositeStringUniqueDao;
 import ca.uhn.fhir.jpa.model.cross.ResourcePersistentId;
@@ -34,7 +34,6 @@ import ca.uhn.fhir.jpa.model.entity.ResourceLink;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.searchparam.JpaRuntimeSearchParam;
 import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
-import ca.uhn.fhir.jpa.searchparam.extractor.ResourceLinkExtractor;
 import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorService;
 import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.model.api.IQueryParameterType;
@@ -84,8 +83,6 @@ public class SearchParamWithInlineReferencesExtractor {
 	@Autowired
 	private SearchParamExtractorService mySearchParamExtractorService;
 	@Autowired
-	private ResourceLinkExtractor myResourceLinkExtractor;
-	@Autowired
 	private DaoResourceLinkResolver myDaoResourceLinkResolver;
 	@Autowired
 	private DaoSearchParamSynchronizer myDaoSearchParamSynchronizer;
@@ -96,19 +93,15 @@ public class SearchParamWithInlineReferencesExtractor {
 	protected EntityManager myEntityManager;
 
 	public void populateFromResource(ResourceIndexedSearchParams theParams, Date theUpdateTime, ResourceTable theEntity, IBaseResource theResource, ResourceIndexedSearchParams theExistingParams, RequestDetails theRequest) {
-		mySearchParamExtractorService.extractFromResource(theRequest, theParams, theEntity, theResource);
+		extractInlineReferences(theResource, theRequest);
+
+		mySearchParamExtractorService.extractFromResource(theRequest, theParams, theEntity, theResource, theUpdateTime, true);
 
 		Set<Map.Entry<String, RuntimeSearchParam>> activeSearchParams = mySearchParamRegistry.getActiveSearchParams(theEntity.getResourceType()).entrySet();
 		if (myDaoConfig.getIndexMissingFields() == DaoConfig.IndexEnabledEnum.ENABLED) {
 			theParams.findMissingSearchParams(myDaoConfig.getModelConfig(), theEntity, activeSearchParams);
 		}
 
-		theParams.setUpdatedTime(theUpdateTime);
-
-		extractInlineReferences(theResource, theRequest);
-
-		myResourceLinkExtractor.extractResourceLinks(theParams, theEntity, theResource, theUpdateTime, myDaoResourceLinkResolver, true, theRequest);
-
 		/*
 		 * If the existing resource already has links and those match links we still want, use them instead of removing them and re adding them
 		 */

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/xmlpatch/XmlPatchUtils.java

@@ -1,12 +1,14 @@
 package ca.uhn.fhir.jpa.util.xmlpatch;
 
-import java.io.*;
+import ca.uhn.fhir.context.FhirContext;
-
+import ca.uhn.fhir.rest.api.Constants;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import com.github.dnault.xmlpatch.Patcher;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 
-import com.github.dnault.xmlpatch.Patcher;
+import java.io.ByteArrayInputStream;
-
+import java.io.ByteArrayOutputStream;
-import ca.uhn.fhir.context.FhirContext;
+import java.io.IOException;
 
 /*
  * #%L
@@ -28,9 +30,6 @@ import ca.uhn.fhir.context.FhirContext;
  * #L%
  */
 
-import ca.uhn.fhir.rest.api.Constants;
-import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
-
 public class XmlPatchUtils {
 
 	public static <T extends IBaseResource> T apply(FhirContext theCtx, T theResourceToUpdate, String thePatchBody) {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/TolerantJsonParserR4Test.java

@@ -0,0 +1,50 @@
+package ca.uhn.fhir.jpa.dao;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.parser.DataFormatException;
+import ca.uhn.fhir.parser.LenientErrorHandler;
+import org.hl7.fhir.r4.model.Observation;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TolerantJsonParserR4Test {
+
+	private FhirContext myFhirContext = FhirContext.forR4();
+
+	@Test
+	public void testParseInvalidNumeric() {
+		String input = "{\n" +
+			"\"resourceType\": \"Observation\",\n" +
+			"\"valueQuantity\": {\n" +
+			"      \"value\": .5\n" +
+			"   }\n" +
+			"}";
+
+
+		TolerantJsonParser parser = new TolerantJsonParser(myFhirContext, new LenientErrorHandler());
+		Observation obs = parser.parseResource(Observation.class, input);
+
+		assertEquals("0.5", obs.getValueQuantity().getValueElement().getValueAsString());
+	}
+
+	@Test
+	public void testParseInvalidNumeric2() {
+		String input = "{\n" +
+			"\"resourceType\": \"Observation\",\n" +
+			"\"valueQuantity\": {\n" +
+			"      \"value\": .\n" +
+			"   }\n" +
+			"}";
+
+
+		TolerantJsonParser parser = new TolerantJsonParser(myFhirContext, new LenientErrorHandler());
+		try {
+			parser.parseResource(Observation.class, input);
+		} catch (DataFormatException e) {
+			assertEquals("[element=\"value\"] Invalid attribute value \".\": No digits found.", e.getMessage());
+		}
+
+	}
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/BaseJpaDstu3Test.java

@@ -218,6 +218,8 @@ public abstract class BaseJpaDstu3Test extends BaseJpaTest {
 	@Autowired
 	protected IResourceIndexedSearchParamStringDao myResourceIndexedSearchParamStringDao;
 	@Autowired
+	protected IResourceIndexedSearchParamTokenDao myResourceIndexedSearchParamTokenDao;
+	@Autowired
 	protected IResourceTableDao myResourceTableDao;
 	@Autowired
 	protected IResourceTagDao myResourceTagDao;

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchCustomSearchParamTest.java

@@ -1,8 +1,10 @@
 package ca.uhn.fhir.jpa.dao.dstu3;
 
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
+import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.model.api.Include;
+import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.rest.api.server.IBundleProvider;
 import ca.uhn.fhir.rest.param.*;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
@@ -20,6 +22,7 @@ import org.junit.Before;
 import org.junit.Test;
 
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.*;
 import static org.junit.Assert.*;
@@ -1054,6 +1057,48 @@ public class FhirResourceDaoDstu3SearchCustomSearchParamTest extends BaseJpaDstu
 
 	}
 
+	@Test
+	public void testProgramaticallyContainedByReferenceAreStillResolvable() {
+		SearchParameter sp = new SearchParameter();
+		sp.setUrl("http://hapifhir.io/fhir/StructureDefinition/sp-unique");
+		sp.setName("MEDICATIONADMINISTRATION-INGREDIENT-MEDICATION");
+		sp.setStatus(Enumerations.PublicationStatus.ACTIVE);
+		sp.setCode("medicationadministration-ingredient-medication");
+		sp.addBase("MedicationAdministration");
+		sp.setType(Enumerations.SearchParamType.TOKEN);
+		sp.setExpression("MedicationAdministration.medication.resolve().ingredient.item.as(Reference).resolve().code");
+		mySearchParameterDao.create(sp);
+		mySearchParamRegistry.forceRefresh();
+
+		Medication ingredient = new Medication();
+		ingredient.getCode().addCoding().setSystem("system").setCode("code");
+
+		Medication medication = new Medication();
+		medication.addIngredient().setItem(new Reference(ingredient));
+
+		MedicationAdministration medAdmin = new MedicationAdministration();
+		medAdmin.setMedication(new Reference(medication));
+
+		myMedicationAdministrationDao.create(medAdmin);
+		
+		runInTransaction(()->{
+			List<ResourceIndexedSearchParamToken> tokens = myResourceIndexedSearchParamTokenDao
+				.findAll()
+				.stream()
+				.filter(t -> t.getParamName().equals("medicationadministration-ingredient-medication"))
+				.collect(Collectors.toList());
+			ourLog.info("Tokens: {}", tokens);
+			assertEquals(tokens.toString(), 1, tokens.size());
+
+		});
+
+		SearchParameterMap map = new SearchParameterMap();
+		map.add("medicationadministration-ingredient-medication", new TokenParam("system","code"));
+		assertEquals(1, myMedicationAdministrationDao.search(map).size().intValue());
+
+	}
+
+
 	@AfterClass
 	public static void afterClassClearContext() {
 		TestUtil.clearAllStaticFieldsForUnitTest();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchCustomSearchParamTest.java

@@ -253,6 +253,7 @@ public class FhirResourceDaoR4SearchCustomSearchParamTest extends BaseJpaR4Test
 
 		MessageHeader messageHeader = new MessageHeader();
 		messageHeader.setId("123");
+		messageHeader.setDefinition("Hello");
 		Bundle bundle = new Bundle();
 		bundle.setType(Bundle.BundleType.MESSAGE);
 		bundle.addEntry()

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/config/SearchParamConfig.java

@@ -22,9 +22,7 @@ package ca.uhn.fhir.jpa.searchparam.config;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.searchparam.MatchUrlService;
-import ca.uhn.fhir.jpa.searchparam.extractor.IResourceLinkResolver;
 import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor;
-import ca.uhn.fhir.jpa.searchparam.extractor.ResourceLinkExtractor;
 import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorDstu2;
 import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorDstu3;
 import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorR4;
@@ -32,13 +30,11 @@ import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorR5;
 import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorService;
 import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryResourceMatcher;
 import ca.uhn.fhir.jpa.searchparam.matcher.IndexedSearchParamExtractor;
-import ca.uhn.fhir.jpa.searchparam.matcher.InlineResourceLinkResolver;
 import ca.uhn.fhir.jpa.searchparam.matcher.SearchParamMatcher;
 import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.jpa.searchparam.registry.SearchParamRegistryImpl;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.scheduling.annotation.EnableScheduling;
@@ -79,11 +75,6 @@ public class SearchParamConfig {
 		return new MatchUrlService();
 	}
 
-	@Bean
-	public ResourceLinkExtractor resourceLinkExtractor() {
-		return new ResourceLinkExtractor();
-	}
-
 	@Bean
 	@Lazy
 	public SearchParamExtractorService searchParamExtractorService(){
@@ -95,11 +86,6 @@ public class SearchParamConfig {
 		return new IndexedSearchParamExtractor();
 	}
 
-	@Bean
-	public InlineResourceLinkResolver inlineResourceLinkResolver() {
-		return new InlineResourceLinkResolver();
-	}
-
 	@Bean
 	public InMemoryResourceMatcher InMemoryResourceMatcher() {
 		return new InMemoryResourceMatcher();

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java

@@ -1015,6 +1015,10 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
 					systemAsString = myUseSystem;
 				}
 
+				if (value instanceof IIdType) {
+					valueAsString = ((IIdType) value).getIdPart();
+				}
+
 				BaseSearchParamExtractor.this.createTokenIndexIfNotBlank(myResourceTypeName, params, searchParam, systemAsString, valueAsString);
 				return;
 			}

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/ResourceIndexedSearchParams.java

@@ -26,18 +26,22 @@ import ca.uhn.fhir.model.api.IQueryParameterType;
 import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
 import ca.uhn.fhir.rest.param.ReferenceParam;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.context.annotation.DependsOn;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.function.Predicate;
 
 import static org.apache.commons.lang3.StringUtils.compare;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 public final class ResourceIndexedSearchParams {
-	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceIndexedSearchParams.class);
-
 	final public Collection<ResourceIndexedSearchParamString> myStringParams = new ArrayList<>();
 	final public Collection<ResourceIndexedSearchParamToken> myTokenParams = new HashSet<>();
 	final public Collection<ResourceIndexedSearchParamNumber> myNumberParams = new ArrayList<>();
@@ -110,7 +114,7 @@ public final class ResourceIndexedSearchParams {
 		theEntity.setHasLinks(myLinks.isEmpty() == false);
 	}
 
-	public void setUpdatedTime(Date theUpdateTime) {
+	void setUpdatedTime(Date theUpdateTime) {
 		setUpdatedTime(myStringParams, theUpdateTime);
 		setUpdatedTime(myNumberParams, theUpdateTime);
 		setUpdatedTime(myQuantityParams, theUpdateTime);

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/ResourceLinkExtractor.java

@@ -1,192 +0,0 @@
-package ca.uhn.fhir.jpa.searchparam.extractor;
-
-/*-
- * #%L
- * HAPI FHIR Search Parameters
- * %%
- * Copyright (C) 2014 - 2020 University Health Network
- * %%
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * #L%
- */
-
-import ca.uhn.fhir.context.FhirContext;
-import ca.uhn.fhir.context.RuntimeResourceDefinition;
-import ca.uhn.fhir.context.RuntimeSearchParam;
-import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
-import ca.uhn.fhir.jpa.model.cross.IResourceLookup;
-import ca.uhn.fhir.jpa.model.entity.ModelConfig;
-import ca.uhn.fhir.jpa.model.entity.ResourceLink;
-import ca.uhn.fhir.jpa.model.entity.ResourceTable;
-import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
-import ca.uhn.fhir.parser.DataFormatException;
-import ca.uhn.fhir.rest.api.server.RequestDetails;
-import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
-import org.apache.commons.lang3.StringUtils;
-import org.hl7.fhir.instance.model.api.IBaseReference;
-import org.hl7.fhir.instance.model.api.IBaseResource;
-import org.hl7.fhir.instance.model.api.IIdType;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.commons.lang3.StringUtils.isBlank;
-import static org.apache.commons.lang3.StringUtils.isNotBlank;
-
-public class ResourceLinkExtractor {
-	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceLinkExtractor.class);
-
-	@Autowired
-	private ModelConfig myModelConfig;
-	@Autowired
-	private FhirContext myContext;
-	@Autowired
-	private ISearchParamRegistry mySearchParamRegistry;
-	@Autowired
-	private ISearchParamExtractor mySearchParamExtractor;
-	@Autowired
-	private IInterceptorBroadcaster myInterceptorBroadcaster;
-
-	public void extractResourceLinks(ResourceIndexedSearchParams theParams, ResourceTable theEntity, IBaseResource theResource, Date theUpdateTime, IResourceLinkResolver theResourceLinkResolver, boolean theFailOnInvalidReference, RequestDetails theRequest) {
-		String resourceName = myContext.getResourceDefinition(theResource).getName();
-
-		ISearchParamExtractor.SearchParamSet<PathAndRef> refs = mySearchParamExtractor.extractResourceLinks(theResource);
-		SearchParamExtractorService.handleWarnings(theRequest, myInterceptorBroadcaster, refs);
-
-		Map<String, IResourceLookup> resourceIdToResolvedTarget = new HashMap<>();
-		for (PathAndRef nextPathAndRef : refs) {
-			RuntimeSearchParam searchParam = mySearchParamRegistry.getActiveSearchParam(resourceName, nextPathAndRef.getSearchParamName());
-			extractResourceLinks(theParams, theEntity, theUpdateTime, theResourceLinkResolver, searchParam, nextPathAndRef, theFailOnInvalidReference, theRequest, resourceIdToResolvedTarget);
-		}
-
-		theEntity.setHasLinks(theParams.myLinks.size() > 0);
-	}
-
-	private void extractResourceLinks(ResourceIndexedSearchParams theParams, ResourceTable theEntity, Date theUpdateTime, IResourceLinkResolver theResourceLinkResolver, RuntimeSearchParam theRuntimeSearchParam, PathAndRef thePathAndRef, boolean theFailOnInvalidReference, RequestDetails theRequest, Map<String, IResourceLookup> theResourceIdToResolvedTarget) {
-		IBaseReference nextReference = thePathAndRef.getRef();
-		IIdType nextId = nextReference.getReferenceElement();
-		String path = thePathAndRef.getPath();
-
-		/*
-		 * This can only really happen if the DAO is being called
-		 * programmatically with a Bundle (not through the FHIR REST API)
-		 * but Smile does this
-		 */
-		if (nextId.isEmpty() && nextReference.getResource() != null) {
-			nextId = nextReference.getResource().getIdElement();
-		}
-
-		theParams.myPopulatedResourceLinkParameters.add(thePathAndRef.getSearchParamName());
-
-		boolean canonical = thePathAndRef.isCanonical();
-		if (LogicalReferenceHelper.isLogicalReference(myModelConfig, nextId) || canonical) {
-			String value = nextId.getValue();
-			ResourceLink resourceLink = ResourceLink.forLogicalReference(thePathAndRef.getPath(), theEntity, value, theUpdateTime);
-			if (theParams.myLinks.add(resourceLink)) {
-				ourLog.debug("Indexing remote resource reference URL: {}", nextId);
-			}
-			return;
-		}
-
-		String baseUrl = nextId.getBaseUrl();
-		String typeString = nextId.getResourceType();
-		if (isBlank(typeString)) {
-			String msg = "Invalid resource reference found at path[" + path + "] - Does not contain resource type - " + nextId.getValue();
-			if (theFailOnInvalidReference) {
-				throw new InvalidRequestException(msg);
-			} else {
-				ourLog.debug(msg);
-				return;
-			}
-		}
-		RuntimeResourceDefinition resourceDefinition;
-		try {
-			resourceDefinition = myContext.getResourceDefinition(typeString);
-		} catch (DataFormatException e) {
-			String msg = "Invalid resource reference found at path[" + path + "] - Resource type is unknown or not supported on this server - " + nextId.getValue();
-			if (theFailOnInvalidReference) {
-				throw new InvalidRequestException(msg);
-			} else {
-				ourLog.debug(msg);
-				return;
-			}
-		}
-
-		if (theRuntimeSearchParam.hasTargets()) {
-			if (!theRuntimeSearchParam.getTargets().contains(typeString)) {
-				return;
-			}
-		}
-
-		if (isNotBlank(baseUrl)) {
-			if (!myModelConfig.getTreatBaseUrlsAsLocal().contains(baseUrl) && !myModelConfig.isAllowExternalReferences()) {
-				String msg = myContext.getLocalizer().getMessage(BaseSearchParamExtractor.class, "externalReferenceNotAllowed", nextId.getValue());
-				throw new InvalidRequestException(msg);
-			} else {
-				ResourceLink resourceLink = ResourceLink.forAbsoluteReference(thePathAndRef.getPath(), theEntity, nextId, theUpdateTime);
-				if (theParams.myLinks.add(resourceLink)) {
-					ourLog.debug("Indexing remote resource reference URL: {}", nextId);
-				}
-				return;
-			}
-		}
-
-		Class<? extends IBaseResource> type = resourceDefinition.getImplementingClass();
-		String id = nextId.getIdPart();
-		if (StringUtils.isBlank(id)) {
-			String msg = "Invalid resource reference found at path[" + path + "] - Does not contain resource ID - " + nextId.getValue();
-			if (theFailOnInvalidReference) {
-				throw new InvalidRequestException(msg);
-			} else {
-				ourLog.debug(msg);
-				return;
-			}
-		}
-
-		theResourceLinkResolver.validateTypeOrThrowException(type);
-		ResourceLink resourceLink = createResourceLink(theEntity, theUpdateTime, theResourceLinkResolver, theRuntimeSearchParam, path, thePathAndRef, nextId, typeString, type, nextReference, theRequest, theResourceIdToResolvedTarget);
-		if (resourceLink == null) {
-			return;
-		}
-		theParams.myLinks.add(resourceLink);
-	}
-
-	private ResourceLink createResourceLink(ResourceTable theEntity, Date theUpdateTime, IResourceLinkResolver theResourceLinkResolver, RuntimeSearchParam nextSpDef, String theNextPathsUnsplit, PathAndRef nextPathAndRef, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, IBaseReference theReference, RequestDetails theRequest, Map<String, IResourceLookup> theResourceIdToResolvedTarget) {
-		/*
-		 * We keep a cache of resolved target resources. This is good since for some resource types, there
-		 * are multiple search parameters that map to the same element path within a resource (e.g.
-		 * Observation:patient and Observation.subject and we don't want to force a resolution of the
-		 * target any more times than we have to.
-		 */
-
-		IResourceLookup targetResource = theResourceIdToResolvedTarget.get(theNextId.getValue());
-		if (targetResource == null) {
-			targetResource = theResourceLinkResolver.findTargetResource(nextSpDef, theNextPathsUnsplit, theNextId, theTypeString, theType, theReference, theRequest);
-		}
-
-		if (targetResource == null) {
-			return null;
-		}
-
-		theResourceIdToResolvedTarget.put(theNextId.getValue(), targetResource);
-
-		String targetResourceType = targetResource.getResourceType();
-		Long targetResourcePid = targetResource.getResourceId();
-		String targetResourceIdPart = theNextId.getIdPart();
-		return ResourceLink.forLocalReference(nextPathAndRef.getPath(), theEntity, targetResourceType, targetResourcePid, targetResourceIdPart, theUpdateTime);
-	}
-
-}

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/SearchParamExtractorService.java

@@ -20,21 +20,36 @@ package ca.uhn.fhir.jpa.searchparam.extractor;
  * #L%
  */
 
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.context.RuntimeResourceDefinition;
+import ca.uhn.fhir.context.RuntimeSearchParam;
 import ca.uhn.fhir.interceptor.api.HookParams;
 import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
 import ca.uhn.fhir.interceptor.api.Pointcut;
+import ca.uhn.fhir.jpa.model.cross.IResourceLookup;
 import ca.uhn.fhir.jpa.model.entity.*;
 import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
+import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster;
+import ca.uhn.fhir.parser.DataFormatException;
+import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
+import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.StringUtils;
+import org.hl7.fhir.instance.model.api.IBaseReference;
 import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.instance.model.api.IIdType;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.stereotype.Service;
 
 import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 public class SearchParamExtractorService {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchParamExtractorService.class);
@@ -43,30 +58,60 @@ public class SearchParamExtractorService {
 	private ISearchParamExtractor mySearchParamExtractor;
 	@Autowired
 	private IInterceptorBroadcaster myInterceptorBroadcaster;
+	@Autowired
+	private ModelConfig myModelConfig;
+	@Autowired
+	private FhirContext myContext;
+	@Autowired
+	private ISearchParamRegistry mySearchParamRegistry;
+	@Autowired(required = false)
+	private IResourceLinkResolver myResourceLinkResolver;
 
-	public void extractFromResource(RequestDetails theRequestDetails, ResourceIndexedSearchParams theParams, ResourceTable theEntity, IBaseResource theResource) {
+	/**
+	 * This method is responsible for scanning a resource for all of the search parameter instances. I.e. for all search parameters defined for
+	 * a given resource type, it extracts the associated indexes and populates {@literal theParams}.
+	 */
+	public void extractFromResource(RequestDetails theRequestDetails, ResourceIndexedSearchParams theParams, ResourceTable theEntity, IBaseResource theResource, Date theUpdateTime, boolean theFailOnInvalidReference) {
+		IBaseResource resource = normalizeResource(theResource);
+
+		// All search parameter types except Reference
+		extractSearchIndexParameters(theRequestDetails, theParams, resource, theEntity);
+
+		// Reference search parameters
+		extractResourceLinks(theParams, theEntity, resource, theUpdateTime, theFailOnInvalidReference, theRequestDetails);
+
+		theParams.setUpdatedTime(theUpdateTime);
+	}
+
+	private void extractSearchIndexParameters(RequestDetails theRequestDetails, ResourceIndexedSearchParams theParams, IBaseResource theResource, ResourceTable theEntity) {
+
+		// Strings
 		ISearchParamExtractor.SearchParamSet<ResourceIndexedSearchParamString> strings = extractSearchParamStrings(theResource);
 		handleWarnings(theRequestDetails, myInterceptorBroadcaster, strings);
 		theParams.myStringParams.addAll(strings);
 
+		// Numbers
 		ISearchParamExtractor.SearchParamSet<ResourceIndexedSearchParamNumber> numbers = extractSearchParamNumber(theResource);
 		handleWarnings(theRequestDetails, myInterceptorBroadcaster, numbers);
 		theParams.myNumberParams.addAll(numbers);
 
+		// Quantities
 		ISearchParamExtractor.SearchParamSet<ResourceIndexedSearchParamQuantity> quantities = extractSearchParamQuantity(theResource);
 		handleWarnings(theRequestDetails, myInterceptorBroadcaster, quantities);
 		theParams.myQuantityParams.addAll(quantities);
 
+		// Dates
 		ISearchParamExtractor.SearchParamSet<ResourceIndexedSearchParamDate> dates = extractSearchParamDates(theResource);
 		handleWarnings(theRequestDetails, myInterceptorBroadcaster, dates);
 		theParams.myDateParams.addAll(dates);
 
+		// URIs
 		ISearchParamExtractor.SearchParamSet<ResourceIndexedSearchParamUri> uris = extractSearchParamUri(theResource);
 		handleWarnings(theRequestDetails, myInterceptorBroadcaster, uris);
 		theParams.myUriParams.addAll(uris);
 
-		ourLog.trace("Storing date indexes: {}", theParams.myDateParams);
+		// Tokens (can result in both Token and String, as we index the display name for
-
+		// the types: Coding, CodeableConcept)
 		for (BaseResourceIndexedSearchParam next : extractSearchParamTokens(theResource)) {
 			if (next instanceof ResourceIndexedSearchParamToken) {
 				theParams.myTokenParams.add((ResourceIndexedSearchParamToken) next);
@@ -77,21 +122,182 @@ public class SearchParamExtractorService {
 			}
 		}
 
+		// Specials
 		for (BaseResourceIndexedSearchParam next : extractSearchParamSpecial(theResource)) {
 			if (next instanceof ResourceIndexedSearchParamCoords) {
 				theParams.myCoordsParams.add((ResourceIndexedSearchParamCoords) next);
 			}
 		}
 
-		populateResourceTable(theParams.myStringParams, theEntity);
+		// Do this after, because we add to strings during both string and token processing
 		populateResourceTable(theParams.myNumberParams, theEntity);
 		populateResourceTable(theParams.myQuantityParams, theEntity);
 		populateResourceTable(theParams.myDateParams, theEntity);
 		populateResourceTable(theParams.myUriParams, theEntity);
-		populateResourceTable(theParams.myCoordsParams, theEntity);
 		populateResourceTable(theParams.myTokenParams, theEntity);
+		populateResourceTable(theParams.myStringParams, theEntity);
+		populateResourceTable(theParams.myCoordsParams, theEntity);
+
 	}
 
+	/**
+	 * This is a bit hacky, but if someone has manually populated a resource (ie. my working directly with the model
+	 * as opposed to by parsing a serialized instance) it's possible that they have put in contained resources
+	 * using {@link IBaseReference#setResource(IBaseResource)}, and those contained resources have not yet
+	 * ended up in the Resource.contained array, meaning that FHIRPath expressions won't be able to find them.
+	 *
+	 * As a result, we to a serialize-and-parse to normalize the object. This really only affects people who
+	 * are calling the JPA DAOs directly, but there are a few of those...
+	 */
+	private IBaseResource normalizeResource(IBaseResource theResource) {
+		IParser parser = myContext.newJsonParser().setPrettyPrint(false);
+		theResource = parser.parseResource(parser.encodeResourceToString(theResource));
+		return theResource;
+	}
+
+	private void extractResourceLinks(ResourceIndexedSearchParams theParams, ResourceTable theEntity, IBaseResource theResource, Date theUpdateTime, boolean theFailOnInvalidReference, RequestDetails theRequest) {
+		String resourceName = myContext.getResourceDefinition(theResource).getName();
+
+		ISearchParamExtractor.SearchParamSet<PathAndRef> refs = mySearchParamExtractor.extractResourceLinks(theResource);
+		SearchParamExtractorService.handleWarnings(theRequest, myInterceptorBroadcaster, refs);
+
+		Map<String, IResourceLookup> resourceIdToResolvedTarget = new HashMap<>();
+		for (PathAndRef nextPathAndRef : refs) {
+			RuntimeSearchParam searchParam = mySearchParamRegistry.getActiveSearchParam(resourceName, nextPathAndRef.getSearchParamName());
+			extractResourceLinks(theParams, theEntity, theUpdateTime, searchParam, nextPathAndRef, theFailOnInvalidReference, theRequest, resourceIdToResolvedTarget);
+		}
+
+		theEntity.setHasLinks(theParams.myLinks.size() > 0);
+	}
+
+	private void extractResourceLinks(ResourceIndexedSearchParams theParams, ResourceTable theEntity, Date theUpdateTime, RuntimeSearchParam theRuntimeSearchParam, PathAndRef thePathAndRef, boolean theFailOnInvalidReference, RequestDetails theRequest, Map<String, IResourceLookup> theResourceIdToResolvedTarget) {
+		IBaseReference nextReference = thePathAndRef.getRef();
+		IIdType nextId = nextReference.getReferenceElement();
+		String path = thePathAndRef.getPath();
+
+		/*
+		 * This can only really happen if the DAO is being called
+		 * programmatically with a Bundle (not through the FHIR REST API)
+		 * but Smile does this
+		 */
+		if (nextId.isEmpty() && nextReference.getResource() != null) {
+			nextId = nextReference.getResource().getIdElement();
+		}
+
+		theParams.myPopulatedResourceLinkParameters.add(thePathAndRef.getSearchParamName());
+
+		boolean canonical = thePathAndRef.isCanonical();
+		if (LogicalReferenceHelper.isLogicalReference(myModelConfig, nextId) || canonical) {
+			String value = nextId.getValue();
+			ResourceLink resourceLink = ResourceLink.forLogicalReference(thePathAndRef.getPath(), theEntity, value, theUpdateTime);
+			if (theParams.myLinks.add(resourceLink)) {
+				ourLog.debug("Indexing remote resource reference URL: {}", nextId);
+			}
+			return;
+		}
+
+		String baseUrl = nextId.getBaseUrl();
+		String typeString = nextId.getResourceType();
+		if (isBlank(typeString)) {
+			String msg = "Invalid resource reference found at path[" + path + "] - Does not contain resource type - " + nextId.getValue();
+			if (theFailOnInvalidReference) {
+				throw new InvalidRequestException(msg);
+			} else {
+				ourLog.debug(msg);
+				return;
+			}
+		}
+		RuntimeResourceDefinition resourceDefinition;
+		try {
+			resourceDefinition = myContext.getResourceDefinition(typeString);
+		} catch (DataFormatException e) {
+			String msg = "Invalid resource reference found at path[" + path + "] - Resource type is unknown or not supported on this server - " + nextId.getValue();
+			if (theFailOnInvalidReference) {
+				throw new InvalidRequestException(msg);
+			} else {
+				ourLog.debug(msg);
+				return;
+			}
+		}
+
+		if (theRuntimeSearchParam.hasTargets()) {
+			if (!theRuntimeSearchParam.getTargets().contains(typeString)) {
+				return;
+			}
+		}
+
+		if (isNotBlank(baseUrl)) {
+			if (!myModelConfig.getTreatBaseUrlsAsLocal().contains(baseUrl) && !myModelConfig.isAllowExternalReferences()) {
+				String msg = myContext.getLocalizer().getMessage(BaseSearchParamExtractor.class, "externalReferenceNotAllowed", nextId.getValue());
+				throw new InvalidRequestException(msg);
+			} else {
+				ResourceLink resourceLink = ResourceLink.forAbsoluteReference(thePathAndRef.getPath(), theEntity, nextId, theUpdateTime);
+				if (theParams.myLinks.add(resourceLink)) {
+					ourLog.debug("Indexing remote resource reference URL: {}", nextId);
+				}
+				return;
+			}
+		}
+
+		Class<? extends IBaseResource> type = resourceDefinition.getImplementingClass();
+		String id = nextId.getIdPart();
+		if (StringUtils.isBlank(id)) {
+			String msg = "Invalid resource reference found at path[" + path + "] - Does not contain resource ID - " + nextId.getValue();
+			if (theFailOnInvalidReference) {
+				throw new InvalidRequestException(msg);
+			} else {
+				ourLog.debug(msg);
+				return;
+			}
+		}
+
+		ResourceLink resourceLink;
+		if (theFailOnInvalidReference) {
+
+			myResourceLinkResolver.validateTypeOrThrowException(type);
+			resourceLink = resolveTargetAndCreateResourceLinkOrReturnNull(theEntity, theUpdateTime, theRuntimeSearchParam, path, thePathAndRef, nextId, typeString, type, nextReference, theRequest, theResourceIdToResolvedTarget);
+			if (resourceLink == null) {
+				return;
+			}
+
+		} else {
+
+			ResourceTable target;
+			target = new ResourceTable();
+			target.setResourceType(typeString);
+			resourceLink = ResourceLink.forLocalReference(thePathAndRef.getPath(), theEntity, typeString, null, nextId.getIdPart(), theUpdateTime);
+
+		}
+
+		theParams.myLinks.add(resourceLink);
+	}
+
+	private ResourceLink resolveTargetAndCreateResourceLinkOrReturnNull(ResourceTable theEntity, Date theUpdateTime, RuntimeSearchParam nextSpDef, String theNextPathsUnsplit, PathAndRef nextPathAndRef, IIdType theNextId, String theTypeString, Class<? extends IBaseResource> theType, IBaseReference theReference, RequestDetails theRequest, Map<String, IResourceLookup> theResourceIdToResolvedTarget) {
+		/*
+		 * We keep a cache of resolved target resources. This is good since for some resource types, there
+		 * are multiple search parameters that map to the same element path within a resource (e.g.
+		 * Observation:patient and Observation.subject and we don't want to force a resolution of the
+		 * target any more times than we have to.
+		 */
+
+		IResourceLookup targetResource = theResourceIdToResolvedTarget.get(theNextId.getValue());
+		if (targetResource == null) {
+			targetResource = myResourceLinkResolver.findTargetResource(nextSpDef, theNextPathsUnsplit, theNextId, theTypeString, theType, theReference, theRequest);
+		}
+
+		if (targetResource == null) {
+			return null;
+		}
+
+		theResourceIdToResolvedTarget.put(theNextId.getValue(), targetResource);
+
+		String targetResourceType = targetResource.getResourceType();
+		Long targetResourcePid = targetResource.getResourceId();
+		String targetResourceIdPart = theNextId.getIdPart();
+		return ResourceLink.forLocalReference(nextPathAndRef.getPath(), theEntity, targetResourceType, targetResourcePid, targetResourceIdPart, theUpdateTime);
+	}
+
+
 	static void handleWarnings(RequestDetails theRequestDetails, IInterceptorBroadcaster theInterceptorBroadcaster, ISearchParamExtractor.SearchParamSet<?> theSearchParamSet) {
 		if (theSearchParamSet.getWarnings().isEmpty()) {
 			return;

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/matcher/IndexedSearchParamExtractor.java

@@ -23,30 +23,23 @@ package ca.uhn.fhir.jpa.searchparam.matcher;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
-import ca.uhn.fhir.jpa.searchparam.extractor.ResourceLinkExtractor;
 import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorService;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
 
 public class IndexedSearchParamExtractor {
 	@Autowired
 	private FhirContext myContext;
 	@Autowired
 	private SearchParamExtractorService mySearchParamExtractorService;
-	@Autowired
-	private ResourceLinkExtractor myResourceLinkExtractor;
-	@Autowired
-	private InlineResourceLinkResolver myInlineResourceLinkResolver;
 
 	public ResourceIndexedSearchParams extractIndexedSearchParams(IBaseResource theResource, RequestDetails theRequest) {
 		ResourceTable entity = new ResourceTable();
 		String resourceType = myContext.getResourceDefinition(theResource).getName();
 		entity.setResourceType(resourceType);
 		ResourceIndexedSearchParams resourceIndexedSearchParams = new ResourceIndexedSearchParams();
-		mySearchParamExtractorService.extractFromResource(theRequest, resourceIndexedSearchParams, entity, theResource);
+		mySearchParamExtractorService.extractFromResource(theRequest, resourceIndexedSearchParams, entity, theResource, theResource.getMeta().getLastUpdated(), false);
-		myResourceLinkExtractor.extractResourceLinks(resourceIndexedSearchParams, entity, theResource, theResource.getMeta().getLastUpdated(), myInlineResourceLinkResolver, false, theRequest);
 		return resourceIndexedSearchParams;
 	}
 }

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/matcher/InlineResourceLinkResolver.java

@@ -1,54 +0,0 @@
-package ca.uhn.fhir.jpa.searchparam.matcher;
-
-/*-
- * #%L
- * HAPI FHIR Search Parameters
- * %%
- * Copyright (C) 2014 - 2020 University Health Network
- * %%
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * #L%
- */
-
-import ca.uhn.fhir.context.RuntimeSearchParam;
-import ca.uhn.fhir.jpa.model.cross.IResourceLookup;
-import ca.uhn.fhir.jpa.model.cross.ResourceLookup;
-import ca.uhn.fhir.jpa.model.entity.ResourceTable;
-import ca.uhn.fhir.jpa.searchparam.extractor.IResourceLinkResolver;
-import ca.uhn.fhir.rest.api.server.RequestDetails;
-import org.hl7.fhir.instance.model.api.IBaseReference;
-import org.hl7.fhir.instance.model.api.IBaseResource;
-import org.hl7.fhir.instance.model.api.IIdType;
-import org.springframework.stereotype.Service;
-
-public class InlineResourceLinkResolver implements IResourceLinkResolver {
-
-	@Override
-	public IResourceLookup findTargetResource(RuntimeSearchParam theSearchParam, String theSourcePath, IIdType theSourceResourceId, String theTypeString, Class<? extends IBaseResource> theType, IBaseReference theReference, RequestDetails theRequest) {
-
-		/*
-		 * TODO: JA - This gets used during runtime in-memory matching for subscription. It's not
-		 * really clear if it's useful or not.
-		 */
-
-		ResourceTable target;
-		target = new ResourceTable();
-		target.setResourceType(theTypeString);
-		return new ResourceLookup(theTypeString, null, null);
-	}
-
-	@Override
-	public void validateTypeOrThrowException(Class<? extends IBaseResource> theType) {
-		// When resolving reference in-memory for a single resource, there's nothing to validate
-	}
-}

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/matcher/SearchParamMatcher.java

@@ -33,7 +33,9 @@ public class SearchParamMatcher {
 	private InMemoryResourceMatcher myInMemoryResourceMatcher;
 
 	public InMemoryMatchResult match(String theCriteria, IBaseResource theResource, RequestDetails theRequest) {
+
 		ResourceIndexedSearchParams resourceIndexedSearchParams = myIndexedSearchParamExtractor.extractIndexedSearchParams(theResource, theRequest);
+
 		return myInMemoryResourceMatcher.match(theCriteria, theResource, resourceIndexedSearchParams);
 	}
 }

hapi-fhir-jpaserver-searchparam/src/test/java/ca/uhn/fhir/jpa/searchparam/IndexStressTest.java

@@ -41,6 +41,7 @@ public class IndexStressTest {
 
 		FhirContext ctx = FhirContext.forDstu3();
 		IValidationSupport mockValidationSupport = mock(IValidationSupport.class);
+		when(mockValidationSupport.getFhirContext()).thenReturn(ctx);
 		IValidationSupport validationSupport = new CachingValidationSupport(new ValidationSupportChain(new DefaultProfileValidationSupport(ctx), mockValidationSupport));
 		ISearchParamRegistry searchParamRegistry = mock(ISearchParamRegistry.class);
 		SearchParamExtractorDstu3 extractor = new SearchParamExtractorDstu3(new ModelConfig(), ctx, validationSupport, searchParamRegistry);

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/parser/JsonParserR4Test.java

@@ -5,10 +5,7 @@ import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.test.BaseTest;
 import ca.uhn.fhir.util.StopWatch;
 import ca.uhn.fhir.util.TestUtil;
-import com.google.common.base.Charsets;
 import com.google.common.collect.Sets;
-import com.google.common.io.Resources;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.output.NullWriter;
 import org.apache.commons.lang.StringUtils;
@@ -21,7 +18,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.net.URL;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.Set;
@@ -30,8 +26,10 @@ import java.util.concurrent.TimeUnit;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.stringContainsInOrder;
 import static org.hamcrest.core.IsNot.not;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
 
 public class JsonParserR4Test extends BaseTest {
 	private static final Logger ourLog = LoggerFactory.getLogger(JsonParserR4Test.class);
@@ -156,10 +154,12 @@ public class JsonParserR4Test extends BaseTest {
 
 	@Test
 	public void testParseBundleWithMultipleNestedContainedResources() throws Exception {
-		URL url = Resources.getResource("bundle-with-two-patient-resources.json");
+		String text = loadResource("/bundle-with-two-patient-resources.json");
-		String text = Resources.toString(url, Charsets.UTF_8);
 
 		Bundle bundle = ourCtx.newJsonParser().parseResource(Bundle.class, text);
+		assertEquals(Boolean.TRUE, bundle.getUserData(BaseParser.RESOURCE_CREATED_BY_PARSER));
+		assertEquals(Boolean.TRUE, bundle.getEntry().get(0).getResource().getUserData(BaseParser.RESOURCE_CREATED_BY_PARSER));
+		assertEquals(Boolean.TRUE, bundle.getEntry().get(1).getResource().getUserData(BaseParser.RESOURCE_CREATED_BY_PARSER));
 
 		assertEquals("12346", getPatientIdValue(bundle, 0));
 		assertEquals("12345", getPatientIdValue(bundle, 1));