Allow canonical references to support treating given base URLs as local. (#3050)

* Add failling test, works on #2843

* Partial hack to get this working

* Move implementation into pre-processing

* Add changelog

* Move implementation

* Minify Test

* Remove fixmes

* Tidying

* Ensure this only operates on STU versions which support canonical

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_6_0/2843-treat-canonicals-as-local.yaml

@@ -0,0 +1,6 @@
+---
+type: change
+issue: 2843
+title: "HAPi-FHIR provides indexing on Canonical Types as references. However, the option to [treat absolute references as local](/docs/server_jpa/configuration.html#jpa-server-configuration-options)
+ was being ignored for those indexed canonicals. This has been corrected. Now, if you have set `getTreatBaseUrlsAsLocal()` and HAPI-FHIR detects a canonical which starts with such a url, that prefix will be stripped,
+   and indexing will occur normally."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java

@@ -39,6 +39,7 @@ import ca.uhn.fhir.jpa.dao.BaseStorageDao;
 import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
 import ca.uhn.fhir.jpa.dao.IResultIterator;
 import ca.uhn.fhir.jpa.dao.ISearchBuilder;
+import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
 import ca.uhn.fhir.jpa.dao.index.IdHelperService;

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderQuestionnaireResponseR4Test.java

@@ -1,16 +1,26 @@
 package ca.uhn.fhir.jpa.provider.r4;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.startsWith;
 import static org.junit.jupiter.api.Assertions.*;
 
 import java.nio.charset.StandardCharsets;
 import java.util.Collection;
+import java.util.Collections;
 
+import ca.uhn.fhir.context.RuntimeResourceDefinition;
+import ca.uhn.fhir.jpa.entity.Search;
+import ca.uhn.fhir.jpa.searchparam.MatchUrlService;
+import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.rest.api.server.IBundleProvider;
+import ca.uhn.fhir.rest.param.TokenParam;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.client.methods.*;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
+import org.apache.jena.base.Sys;
 import org.hl7.fhir.r4.model.*;
 import org.hl7.fhir.r4.model.Questionnaire.QuestionnaireItemType;
 import org.hl7.fhir.r4.model.QuestionnaireResponse.QuestionnaireResponseStatus;
@@ -21,6 +31,7 @@ import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
 import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor;
 import ca.uhn.fhir.validation.IValidatorModule;
 import ca.uhn.fhir.validation.ResultSeverityEnum;
+import org.springframework.beans.factory.annotation.Autowired;
 
 public class ResourceProviderQuestionnaireResponseR4Test extends BaseResourceProviderR4Test {
 
@@ -52,8 +63,29 @@ public class ResourceProviderQuestionnaireResponseR4Test extends BaseResourcePro
 		ourRestServer.getInterceptorService().registerInterceptor(ourValidatingInterceptor);
 	}
 
-	
-	
+	@Test
+	public void testCreateWithNonLocalReferenceWorksWithIncludes() {
+		String baseUrl = "https://hapi.fhir.org/baseR4/";
+
+		myModelConfig.setTreatBaseUrlsAsLocal(Collections.singleton(baseUrl));
+		Questionnaire questionnaire = new Questionnaire();
+		questionnaire.setId("my-questionnaire");
+
+		QuestionnaireResponse questionnaireResponse = new QuestionnaireResponse();
+		questionnaireResponse.setQuestionnaire(baseUrl + "Questionnaire/my-questionnaire");
+		questionnaireResponse.setId("my-questionnaire-response");
+
+		myQuestionnaireDao.update(questionnaire);
+		myQuestionnaireResponseDao.update(questionnaireResponse);
+
+		SearchParameterMap spMap = new SearchParameterMap();
+		spMap.setLoadSynchronous(true);
+		spMap.addInclude(QuestionnaireResponse.INCLUDE_QUESTIONNAIRE);
+		spMap.add("_id", new TokenParam("my-questionnaire-response"));
+		IBundleProvider search = myQuestionnaireResponseDao.search(spMap);
+		assertThat(search.size(), is(equalTo(2)));
+	}
+
 	@SuppressWarnings("unused")
 	@Test
 	public void testCreateWithLocalReference() {

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java

@@ -89,6 +89,7 @@ import java.util.stream.Collectors;
 
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.commons.lang3.StringUtils.strip;
 import static org.apache.commons.lang3.StringUtils.trim;
 
 public abstract class BaseSearchParamExtractor implements ISearchParamExtractor {
@@ -1312,6 +1313,11 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
 			}
 		}
 
+		private boolean isOrCanBeTreatedAsLocal(IIdType theId) {
+			boolean acceptableAsLocalReference = !theId.isAbsolute() ||  myModelConfig.getTreatBaseUrlsAsLocal().contains(theId.getBaseUrl());
+			return acceptableAsLocalReference;
+		}
+
 		public PathAndRef get(IBase theValue, String thePath) {
 			extract(new SearchParamSet<>(),
 				new RuntimeSearchParam(null, null, "Reference", null, null, null, null, null, null, null),

hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/BaseStorageDao.java

@@ -20,6 +20,7 @@ package ca.uhn.fhir.jpa.dao;
  * #L%
  */
 
+import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeSearchParam;
 import ca.uhn.fhir.interceptor.api.HookParams;
@@ -67,6 +68,7 @@ import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
 import org.hl7.fhir.instance.model.api.IBaseReference;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.instance.model.api.IPrimitiveType;
 import org.hl7.fhir.r4.model.InstantType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.transaction.annotation.Propagation;
@@ -77,6 +79,7 @@ import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
@@ -170,7 +173,11 @@ public abstract class BaseStorageDao {
 
 		verifyBundleTypeIsAppropriateForStorage(theResource);
 
-		replaceAbsoluteReferencesWithRelative(theResource);
+		if(!getConfig().getTreatBaseUrlsAsLocal().isEmpty()) {
+			FhirTerser terser = myFhirContext.newTerser();
+			replaceAbsoluteReferencesWithRelative(theResource, terser);
+			replaceAbsoluteUrisWithRelative(theResource, terser);
+		}
 
 		performAutoVersioning(theResource, thePerformIndexing);
 
@@ -215,10 +222,8 @@ public abstract class BaseStorageDao {
 	/**
 	 * Replace absolute references with relative ones if configured to do so
 	 */
-	private void replaceAbsoluteReferencesWithRelative(IBaseResource theResource) {
-		if (getConfig().getTreatBaseUrlsAsLocal().isEmpty() == false) {
-			FhirTerser t = getContext().newTerser();
-			List<ResourceReferenceInfo> refs = t.getAllResourceReferences(theResource);
+	private void replaceAbsoluteReferencesWithRelative(IBaseResource theResource, FhirTerser theTerser) {
+			List<ResourceReferenceInfo> refs = theTerser.getAllResourceReferences(theResource);
 			for (ResourceReferenceInfo nextRef : refs) {
 				IIdType refId = nextRef.getResourceReference().getReferenceElement();
 				if (refId != null && refId.hasBaseUrl()) {
@@ -228,6 +233,28 @@ public abstract class BaseStorageDao {
 					}
 				}
 			}
+	}
+
+	/**
+	 * Replace Canonical URI's with local references, if we find that the canonical should be treated as local.
+	 */
+	private void replaceAbsoluteUrisWithRelative(IBaseResource theResource, FhirTerser theTerser) {
+
+		BaseRuntimeElementDefinition<?> canonicalElementDefinition = myFhirContext.getElementDefinition("canonical");
+		if (canonicalElementDefinition != null) {
+			Class<? extends IPrimitiveType<String>> canonicalType = (Class<? extends IPrimitiveType<String>>) canonicalElementDefinition.getImplementingClass();
+			List<? extends IPrimitiveType<String>> canonicals = theTerser.getAllPopulatedChildElementsOfType(theResource, canonicalType);
+
+			//Try to offset the N^2 by maintaining a visited list.
+			Set<IPrimitiveType<String>> visited = new HashSet<>();
+			for (String baseUrl : myModelConfig.getTreatBaseUrlsAsLocal()) {
+				for (IPrimitiveType<String> canonical : canonicals) {
+					if (!visited.contains(canonical) && canonical.getValue().startsWith(baseUrl)) {
+						canonical.setValue(canonical.getValue().substring(baseUrl.length() + 1));
+						visited.add(canonical);
+					}
+				}
+			}
 		}
 	}