Fix #1583 - Index local refs in canonical types (#1593)

* Fix #1583 - Index local refs in canonical types

* Add fix to pre-expansion

* Test fix

* Rework ID handling

* Test fixes

* Fix test

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/BaseParser.java

@@ -22,10 +22,16 @@ package ca.uhn.fhir.parser;
 
 import ca.uhn.fhir.context.*;
 import ca.uhn.fhir.context.BaseRuntimeElementDefinition.ChildTypeEnum;
-import ca.uhn.fhir.model.api.*;
+import ca.uhn.fhir.model.api.IIdentifiableElement;
+import ca.uhn.fhir.model.api.IResource;
+import ca.uhn.fhir.model.api.ISupportsUndeclaredExtensions;
+import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
+import ca.uhn.fhir.model.api.Tag;
+import ca.uhn.fhir.model.api.TagList;
 import ca.uhn.fhir.model.primitive.IdDt;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import ca.uhn.fhir.util.BundleUtil;
 import ca.uhn.fhir.util.UrlUtil;
 import com.google.common.base.Charsets;
 import org.apache.commons.lang3.StringUtils;
@@ -35,7 +41,13 @@ import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.hl7.fhir.instance.model.api.*;
 
 import javax.annotation.Nullable;
-import java.io.*;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.io.Writer;
 import java.lang.reflect.Modifier;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -530,11 +542,6 @@ public abstract class BaseParser implements IParser {
 		return tags;
 	}
 
-	@Override
-	public Boolean getOverrideResourceIdWithBundleEntryFullUrl() {
-		return myOverrideResourceIdWithBundleEntryFullUrl;
-	}
-
 	@Override
 	public List<Class<? extends IBaseResource>> getPreferTypes() {
 		return myPreferTypes;
@@ -705,39 +712,38 @@ public abstract class BaseParser implements IParser {
 		RuntimeResourceDefinition def = myContext.getResourceDefinition(retVal);
 		if ("Bundle".equals(def.getName())) {
 
-			BaseRuntimeChildDefinition entryChild = def.getChildByName("entry");
+			if (isOverrideResourceIdWithBundleEntryFullUrl()) {
-			BaseRuntimeElementCompositeDefinition<?> entryDef = (BaseRuntimeElementCompositeDefinition<?>) entryChild.getChildByName("entry");
+				BundleUtil.processEntries(myContext, (IBaseBundle) retVal, t -> {
-			List<IBase> entries = entryChild.getAccessor().getValues(retVal);
+					String fullUrl = t.getFullUrl();
-			if (entries != null) {
+					if (fullUrl != null) {
-				for (IBase nextEntry : entries) {
+						IBaseResource resource = t.getResource();
-
+						if (resource != null) {
-					/**
+							IIdType resourceId = resource.getIdElement();
-					 * If Bundle.entry.fullUrl is populated, set the resource ID to that
+							if (isBlank(resourceId.getValue())) {
-					 */
+								resourceId.setValue(fullUrl);
-					// TODO: should emit a warning and maybe notify the error handler if the resource ID doesn't match the
+							} else {
-					// fullUrl idPart
+								if (fullUrl.startsWith("urn:") && fullUrl.endsWith(":" + resourceId.getIdPart())) {
-					BaseRuntimeChildDefinition fullUrlChild = entryDef.getChildByName("fullUrl");
+									resourceId.setValue(fullUrl);
-					if (fullUrlChild == null) {
+								} else {
-						continue; // TODO: remove this once the data model in tinder plugin catches up to 1.2
+									IIdType fullUrlId = myContext.getVersion().newIdType();
-					}
+									fullUrlId.setValue(fullUrl);
-					if (isOverrideResourceIdWithBundleEntryFullUrl()) {
+									if (myContext.getVersion().getVersion().isOlderThan(FhirVersionEnum.DSTU3)) {
-						List<IBase> fullUrl = fullUrlChild.getAccessor().getValues(nextEntry);
+										IIdType newId = fullUrlId;
-						if (fullUrl != null && !fullUrl.isEmpty()) {
+										if (!newId.hasVersionIdPart() && resourceId.hasVersionIdPart()) {
-							IPrimitiveType<?> value = (IPrimitiveType<?>) fullUrl.get(0);
+											newId = newId.withVersion(resourceId.getVersionIdPart());
-							if (value.isEmpty() == false) {
+										}
-								List<IBase> entryResources = entryDef.getChildByName("resource").getAccessor().getValues(nextEntry);
+										resourceId.setValue(newId.getValue());
-								if (entryResources != null && entryResources.size() > 0) {
+									} else if (StringUtils.equals(fullUrlId.getIdPart(), resourceId.getIdPart())) {
-									IBaseResource res = (IBaseResource) entryResources.get(0);
+										if (fullUrlId.hasBaseUrl()) {
-									String versionId = res.getIdElement().getVersionIdPart();
+											IIdType newResourceId = resourceId.withServerBase(fullUrlId.getBaseUrl(), resourceId.getResourceType());
-									res.setId(value.getValueAsString());
+											resourceId.setValue(newResourceId.getValue());
-									if (isNotBlank(versionId) && res.getIdElement().hasVersionIdPart() == false) {
+										}
-										res.setId(res.getIdElement().withVersion(versionId));
 									}
 								}
 							}
 						}
 					}
-				}
+				});
 			}
 
 		}

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/IParser.java

@@ -92,19 +92,6 @@ public interface IParser {
 	 */
 	Boolean getStripVersionsFromReferences();
 
-	/**
-	 * If set to <code>true</code> (which is the default), the Bundle.entry.fullUrl will override the Bundle.entry.resource's
-	 * resource id if the fullUrl is defined. This behavior happens when parsing the source data into a Bundle object. Set this
-	 * to <code>false</code> if this is not the desired behavior (e.g. the client code wishes to perform additional
-	 * validation checks between the fullUrl and the resource id).
-	 *
-	 * @return Returns the parser instance's configuration setting for overriding resource ids with Bundle.entry.fullUrl when
-	 *         parsing the source data into a Bundle object. This method will return <code>null</code> if no value is set, in
-	 *         which case the value from the {@link ParserOptions} will be used (default is <code>true</code>)
-	 * @see ParserOptions
-	 */
-	Boolean getOverrideResourceIdWithBundleEntryFullUrl();
-
 	/**
 	 * Is the parser in "summary mode"? See {@link #setSummaryMode(boolean)} for information
 	 * 

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/BundleUtil.java

@@ -168,6 +168,8 @@ public class BundleUtil {
 
 		BaseRuntimeElementCompositeDefinition<?> entryChildContentsDef = (BaseRuntimeElementCompositeDefinition<?>) entryChildDef.getChildByName("entry");
 
+		BaseRuntimeChildDefinition fullUrlChildDef = entryChildContentsDef.getChildByName("fullUrl");
+
 		BaseRuntimeChildDefinition resourceChildDef = entryChildContentsDef.getChildByName("resource");
 		BaseRuntimeChildDefinition requestChildDef = entryChildContentsDef.getChildByName("request");
 		BaseRuntimeElementCompositeDefinition<?> requestChildContentsDef = (BaseRuntimeElementCompositeDefinition<?>) requestChildDef.getChildByName("request");
@@ -180,6 +182,11 @@ public class BundleUtil {
 			String url = null;
 			RequestTypeEnum requestType = null;
 			String conditionalUrl = null;
+			String fullUrl = fullUrlChildDef
+				.getAccessor()
+				.getFirstValueOrNull(nextEntry)
+				.map(t->((IPrimitiveType<?>)t).getValueAsString())
+				.orElse(null);
 
 			for (IBase nextResource : resourceChildDef.getAccessor().getValues(nextEntry)) {
 				resource = (IBaseResource) nextResource;
@@ -217,7 +224,7 @@ public class BundleUtil {
 			 * order in the original bundle.
 			 */
 			BundleEntryMutator mutator = new BundleEntryMutator(nextEntry, requestChildDef, requestChildContentsDef);
-			ModifiableBundleEntry entry = new ModifiableBundleEntry(new BundleEntryParts(requestType, url, resource, conditionalUrl), mutator);
+			ModifiableBundleEntry entry = new ModifiableBundleEntry(new BundleEntryParts(fullUrl, requestType, url, resource, conditionalUrl), mutator);
 			theProcessor.accept(entry);
 		}
 	}

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/bundle/BundleEntryParts.java

@@ -28,15 +28,21 @@ public class BundleEntryParts {
 	private final IBaseResource myResource;
 	private final String myUrl;
 	private final String myConditionalUrl;
+	private final String myFullUrl;
 
-	public BundleEntryParts(RequestTypeEnum theRequestType, String theUrl, IBaseResource theResource, String theConditionalUrl) {
+	public BundleEntryParts(String theFullUrl, RequestTypeEnum theRequestType, String theUrl, IBaseResource theResource, String theConditionalUrl) {
 		super();
+		myFullUrl = theFullUrl;
 		myRequestType = theRequestType;
 		myUrl = theUrl;
 		myResource = theResource;
 		myConditionalUrl = theConditionalUrl;
 	}
 
+	public String getFullUrl() {
+		return myFullUrl;
+	}
+
 	public RequestTypeEnum getRequestType() {
 		return myRequestType;
 	}

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/bundle/ModifiableBundleEntry.java

@@ -21,6 +21,7 @@ package ca.uhn.fhir.util.bundle;
  */
 
 import ca.uhn.fhir.context.FhirContext;
+import org.hl7.fhir.instance.model.api.IBaseResource;
 
 public class ModifiableBundleEntry {
 	private final BundleEntryParts myBundleEntryParts;
@@ -39,7 +40,15 @@ public class ModifiableBundleEntry {
 		myBundleEntryMutator.setRequestUrl(theFhirContext, theRequestUrl);
 	}
 
+	public String getFullUrl() {
+		return myBundleEntryParts.getFullUrl();
+	}
+
 	public String getRequestUrl() {
 		return myBundleEntryParts.getUrl();
 	}
+
+	public IBaseResource getResource() {
+		return myBundleEntryParts.getResource();
+	}
 }

hapi-fhir-client/src/main/java/ca/uhn/fhir/rest/client/method/ConditionalParamBinder.java

@@ -44,7 +44,7 @@ class ConditionalParamBinder implements IParameter {
 	@Override
 	public void initializeTypes(Method theMethod, Class<? extends Collection<?>> theOuterCollectionType, Class<? extends Collection<?>> theInnerCollectionType, Class<?> theParameterType) {
 		if (theOuterCollectionType != null || theInnerCollectionType != null || theParameterType.equals(String.class) == false) {
-			throw new ConfigurationException("Parameters annotated with @" + ConditionalUrlParam.class.getSimpleName()  + " must be of type String, found incorrect parameteter in method \"" + theMethod + "\"");
+			throw new ConfigurationException("Parameters annotated with @" + ConditionalUrlParam.class.getSimpleName()  + " must be of type String, found incorrect parameter in method \"" + theMethod + "\"");
 		}
 	}
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetConceptAccumulator.java

@@ -31,7 +31,7 @@ public interface IValueSetConceptAccumulator {
 
 	void includeConcept(String theSystem, String theCode, String theDisplay);
 
-	void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, Collection<TermConceptDesignation> theDesignations);
+	void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, @Nullable Collection<TermConceptDesignation> theDesignations);
 
 	void excludeConcept(String theSystem, String theCode);
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java

@@ -34,7 +34,9 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 
-import static org.apache.commons.lang3.StringUtils.*;
+import static org.apache.commons.lang3.StringUtils.isAnyBlank;
+import static org.apache.commons.lang3.StringUtils.isNoneBlank;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ValueSetConceptAccumulator.class);
@@ -71,8 +73,10 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
 	@Override
 	public void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, Collection<TermConceptDesignation> theDesignations) {
 		TermValueSetConcept concept = saveConcept(theSystem, theCode, theDisplay);
-		for (TermConceptDesignation designation : theDesignations) {
+		if (theDesignations != null) {
-			saveConceptDesignation(concept, designation);
+			for (TermConceptDesignation designation : theDesignations) {
+				saveConceptDesignation(concept, designation);
+			}
 		}
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java

@@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.dao.r4;
 
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
 import ca.uhn.fhir.jpa.dao.DaoConfig;
+import ca.uhn.fhir.jpa.dao.DaoMethodOutcome;
 import ca.uhn.fhir.jpa.entity.Search;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamNumber;
@@ -2154,6 +2155,27 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
 
 	}
 
+	@Test
+	public void testSearchResourceLinkOnCanonical() {
+
+		Questionnaire q = new Questionnaire();
+		q.setId("Q");
+		myQuestionnaireDao.update(q);
+
+		QuestionnaireResponse qr = new QuestionnaireResponse();
+		qr.setId("QR");
+		qr.setQuestionnaire("Questionnaire/Q");
+		String qrId = myQuestionnaireResponseDao.update(qr).getId().toUnqualifiedVersionless().getValue();
+
+		List<QuestionnaireResponse> result = toList(myQuestionnaireResponseDao
+			.search(new SearchParameterMap().setLoadSynchronous(true).add(QuestionnaireResponse.SP_QUESTIONNAIRE, new ReferenceParam("Questionnaire/Q"))));
+		assertEquals(1, result.size());
+		assertEquals(qrId, result.get(0).getIdElement().toUnqualifiedVersionless().getValue());
+
+
+	}
+
+
 	@Test
 	public void testSearchResourceLinkWithChain() {
 		Patient patient = new Patient();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithLuceneDisabledTest.java

@@ -10,6 +10,8 @@ import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
+import ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl;
+import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
 import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.rest.api.EncodingEnum;
 import ca.uhn.fhir.rest.api.server.IBundleProvider;
@@ -116,6 +118,8 @@ public class FhirResourceDaoR4SearchWithLuceneDisabledTest extends BaseJpaTest {
 	private IResourceReindexingSvc myResourceReindexingSvc;
 	@Autowired
 	private IBulkDataExportSvc myBulkDataExportSvc;
+	@Autowired
+	private ITermReadSvc myTermSvc;
 
 	@Before
 	@Transactional()
@@ -190,6 +194,36 @@ public class FhirResourceDaoR4SearchWithLuceneDisabledTest extends BaseJpaTest {
 		}
 	}
 
+	@Test
+		public void testExpandValueSet() {
+		CodeSystem cs = new CodeSystem();
+		cs.setUrl("http://fooCS");
+		cs.setContent(CodeSystem.CodeSystemContentMode.COMPLETE);
+		cs.addConcept().setCode("CODEA");
+		cs.addConcept().setCode("CODEB");
+		myCodeSystemDao.create(cs);
+
+		ValueSet vs = new ValueSet();
+		vs.setUrl("http://fooVS");
+		vs.getCompose()
+			.addInclude()
+			.setSystem("http://fooCS")
+			.addConcept(new ValueSet.ConceptReferenceComponent().setCode("CODEA"));
+
+		// Explicit expand
+		ValueSet outcome = myValueSetDao.expand(vs, null);
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
+		assertEquals("CODEA", outcome.getExpansion().getContains().get(0).getCode());
+
+		// Deferred expand
+		IIdType id = myValueSetDao.create(vs).getId().toUnqualifiedVersionless();
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+		outcome = myValueSetDao.expand(id, null, mySrd);
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
+		assertEquals("CODEA", outcome.getExpansion().getContains().get(0).getCode());
+	}
+
+
 	@Test
 	public void testSearchByCodeIn() {
 		CodeSystem cs = new CodeSystem();
@@ -207,6 +241,7 @@ public class FhirResourceDaoR4SearchWithLuceneDisabledTest extends BaseJpaTest {
 			.addConcept(new ValueSet.ConceptReferenceComponent().setCode("CODEA"));
 		myValueSetDao.create(vs);
 
+
 		Observation obs = new Observation();
 		obs.getCode().addCoding().setSystem("http://fooCS").setCode("CODEA");
 		String obs1id = myObservationDao.create(obs).getId().toUnqualifiedVersionless().getValue();

hapi-fhir-jpaserver-base/src/test/resources/david_big_bundle.json

@@ -2666,7 +2666,7 @@
 			"fullUrl": "http://fhirtest.uhn.ca/baseDstu3/Practitioner/A45515",
 			"resource": {
 				"resourceType": "Practitioner",
-				"id": "45515",
+				"id": "A45515",
 				"meta": {
 					"versionId": "1",
 					"lastUpdated": "2016-05-07T11:45:53.918-04:00"

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java

@@ -51,6 +51,7 @@ import org.hl7.fhir.instance.model.api.IBaseReference;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
+import org.hl7.fhir.r4.model.IdType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
 
@@ -151,7 +152,31 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
 
 			String nextType = toRootTypeName(value);
 			switch (nextType) {
+				case "uri":
 				case "canonical":
+					String typeName = toTypeName(value);
+
+					// Canonical has a root type of "uri"
+					if ("canonical".equals(typeName)) {
+						IPrimitiveType<?> valuePrimitive = (IPrimitiveType<?>) value;
+						IBaseReference fakeReference = (IBaseReference) myContext.getElementDefinition("Reference").newInstance();
+						fakeReference.setReference(valuePrimitive.getValueAsString());
+
+						/*
+						 * See #1583
+						 * Technically canonical fields should not allow local references (e.g.
+						 * Questionnaire/123) but it seems reasonable for us to interpret a canonical
+						 * containing a local reference for what it is, and allow people to seaerch
+						 * based on that.
+						 */
+						IIdType parsed = fakeReference.getReferenceElement();
+						if (parsed.hasIdPart() && parsed.hasResourceType() && !parsed.isAbsolute()) {
+							PathAndRef ref = new PathAndRef(searchParam.getName(), path, fakeReference);
+							params.add(ref);
+							break;
+						}
+					}
+
 					params.addWarning("Ignoring canonical reference (indexing canonical is not yet supported)");
 					break;
 				case "reference":
@@ -789,6 +814,11 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
 		return rootParentDefinition.getName();
 	}
 
+	private String toTypeName(IBase nextObject) {
+		BaseRuntimeElementDefinition<?> elementDefinition = getContext().getElementDefinition(nextObject.getClass());
+		return elementDefinition.getName();
+	}
+
 	@SuppressWarnings("unchecked")
 	private void addDateTimeTypes(String theResourceType, Set<ResourceIndexedSearchParamDate> theParams, RuntimeSearchParam theSearchParam, IBase theValue) {
 		IPrimitiveType<Date> nextBaseDateTime = (IPrimitiveType<Date>) theValue;

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/method/ConditionalParamBinder.java

@@ -47,7 +47,7 @@ class ConditionalParamBinder implements IParameter {
 	public void initializeTypes(Method theMethod, Class<? extends Collection<?>> theOuterCollectionType, Class<? extends Collection<?>> theInnerCollectionType, Class<?> theParameterType) {
 		if (theOuterCollectionType != null || theInnerCollectionType != null || theParameterType.equals(String.class) == false) {
 			throw new ConfigurationException(
-					"Parameters annotated with @" + ConditionalUrlParam.class.getSimpleName() + " must be of type String, found incorrect parameteter in method \"" + theMethod + "\"");
+					"Parameters annotated with @" + ConditionalUrlParam.class.getSimpleName() + " must be of type String, found incorrect parameter in method \"" + theMethod + "\"");
 		}
 	}
 

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/method/OperationMethodBinding.java

@@ -28,6 +28,8 @@ import ca.uhn.fhir.parser.DataFormatException;
 import ca.uhn.fhir.rest.annotation.IdParam;
 import ca.uhn.fhir.rest.annotation.Operation;
 import ca.uhn.fhir.rest.annotation.OperationParam;
+import ca.uhn.fhir.rest.annotation.OptionalParam;
+import ca.uhn.fhir.rest.annotation.RequiredParam;
 import ca.uhn.fhir.rest.api.RequestTypeEnum;
 import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
 import ca.uhn.fhir.rest.api.server.IBundleProvider;
@@ -91,6 +93,14 @@ public class OperationMethodBinding extends BaseResourceReturningMethodBinding {
 			myDescription = null;
 		}
 
+		for (Annotation[] nextParamAnnotations : theMethod.getParameterAnnotations()) {
+			for (Annotation nextParam : nextParamAnnotations) {
+				if (nextParam instanceof OptionalParam || nextParam instanceof RequiredParam) {
+					throw new ConfigurationException("Illegal method parameter annotation @" + nextParam.annotationType().getSimpleName() + " on method: " + theMethod.toString());
+				}
+			}
+		}
+
 		if (isBlank(theOperationName)) {
 			throw new ConfigurationException("Method '" + theMethod.getName() + "' on type " + theMethod.getDeclaringClass().getName() + " is annotated with @" + Operation.class.getSimpleName()
 				+ " but this annotation has no name defined");

hapi-fhir-structures-dstu3/pom.xml

@@ -150,7 +150,7 @@
 		<dependency>
 			<groupId>javax.servlet</groupId>
 			<artifactId>javax.servlet-api</artifactId>
-			<version>3.0.1</version>
+			<version>3.1.0</version>
 			<optional>true</optional>
 		</dependency>
 

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/parser/JsonParserR4Test.java

@@ -23,6 +23,7 @@ import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.stringContainsInOrder;
 import static org.hamcrest.core.IsNot.not;
 import static org.junit.Assert.*;
 import static org.mockito.Mockito.mock;
@@ -174,6 +175,34 @@ public class JsonParserR4Test extends BaseTest {
 		assertEquals("<div xmlns=\"http://www.w3.org/1999/xhtml\">Copy © 1999</div>", p.getText().getDivAsString());
 	}
 
+	@Test
+	public void testEncodeAndParseBundleWithFullUrlAndResourceIdMismatch() {
+
+		MessageHeader header = new MessageHeader();
+		header.setId("1.1.1.1");
+		header.setDefinition("Hello");
+
+		Bundle input = new Bundle();
+		input
+			.addEntry()
+			.setFullUrl("urn:uuid:0.0.0.0")
+			.setResource(header);
+
+		String encoded = ourCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(input);
+
+		ourLog.info("Encoded: {}", encoded);
+		assertThat(encoded, stringContainsInOrder(
+			"\"fullUrl\": \"urn:uuid:0.0.0.0\"",
+			"\"id\": \"1.1.1.1\""
+		));
+
+		input = ourCtx.newJsonParser().parseResource(Bundle.class, encoded);
+		assertEquals("urn:uuid:0.0.0.0", input.getEntry().get(0).getFullUrl());
+		assertEquals("MessageHeader/1.1.1.1", input.getEntry().get(0).getResource().getId());
+
+	}
+
+
 	@Test
 	public void testEncodeBinary() {
 		Binary b = new Binary();

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/parser/XmlParserR4Test.java

@@ -1,8 +1,13 @@
 package ca.uhn.fhir.parser;
 
+import static org.hamcrest.Matchers.stringContainsInOrder;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThat;
 
+import org.hl7.fhir.r4.model.Bundle;
 import org.hl7.fhir.r4.model.Composition;
+import org.hl7.fhir.r4.model.MessageHeader;
 import org.hl7.fhir.r4.model.Narrative;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -41,5 +46,33 @@ public class XmlParserR4Test {
 		int idx = encoded.indexOf(sectionText);
 		assertNotEquals(-1, idx);
 	}
-	
+
+	@Test
+	public void testEncodeAndParseBundleWithFullUrlAndResourceIdMismatch() {
+
+		MessageHeader header = new MessageHeader();
+		header.setId("1.1.1.1");
+		header.setDefinition("Hello");
+
+		Bundle input = new Bundle();
+		input
+			.addEntry()
+			.setFullUrl("urn:uuid:0.0.0.0")
+			.setResource(header);
+
+		String encoded = ourCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(input);
+
+		ourLog.info("Encoded: {}", encoded);
+		assertThat(encoded, stringContainsInOrder(
+			"<fullUrl value=\"urn:uuid:0.0.0.0\"/>",
+			"<id value=\"1.1.1.1\"/>"
+		));
+
+		input = ourCtx.newXmlParser().parseResource(Bundle.class, encoded);
+		assertEquals("urn:uuid:0.0.0.0", input.getEntry().get(0).getFullUrl());
+		assertEquals("MessageHeader/1.1.1.1", input.getEntry().get(0).getResource().getId());
+
+	}
+
+
 }

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/BlockingContentR4Test.java

@@ -0,0 +1,113 @@
+package ca.uhn.fhir.rest.server;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.rest.annotation.Transaction;
+import ca.uhn.fhir.rest.annotation.TransactionParam;
+import ca.uhn.fhir.test.utilities.server.ResourceProviderRule;
+import ca.uhn.fhir.test.utilities.server.RestfulServerRule;
+import com.google.common.base.Charsets;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.hl7.fhir.r4.model.Bundle;
+import org.hl7.fhir.r4.model.Patient;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+public class BlockingContentR4Test {
+
+	private static final Logger ourLog = LoggerFactory.getLogger(BlockingContentR4Test.class);
+	private static FhirContext ourCtx = FhirContext.forR4();
+	@ClassRule
+	public static RestfulServerRule ourServerRule = new RestfulServerRule(ourCtx);
+	@Rule
+	public ResourceProviderRule myPatientProviderRule = new ResourceProviderRule(ourServerRule, new SystemProvider());
+
+	@Test
+	public void testCreateWith100Continue() throws Exception {
+		Patient patient = new Patient();
+		patient.setActive(true);
+
+		Bundle input = new Bundle();
+		input.setType(Bundle.BundleType.TRANSACTION);
+		input.addEntry().setResource(patient).setFullUrl("Patient/").getRequest().setMethod(Bundle.HTTPVerb.POST);
+
+		RequestConfig config = RequestConfig.custom().setExpectContinueEnabled(true).build();
+		PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
+		HttpClientBuilder builder = HttpClientBuilder.create();
+		builder.setConnectionManager(connectionManager);
+		builder.setDefaultRequestConfig(config);
+		try (CloseableHttpClient client = builder.build()) {
+
+			String resourceAsString = ourCtx.newJsonParser().encodeResourceToString(input);
+
+//			InputStream inputStream = new BlockingInputStream(resourceAsString.getBytes(Charsets.UTF_8));
+			byte[] bytes = resourceAsString.getBytes(Charsets.UTF_8);
+			InputStream inputStream = new ByteArrayInputStream(bytes);
+			HttpEntity entity = new InputStreamEntity(inputStream, ContentType.parse("application/fhir+json")){
+				@Override
+				public long getContentLength() {
+					return bytes.length + 100;
+				}
+			};
+
+			HttpPost post = new HttpPost("http://localhost:" + ourServerRule.getPort() + "/");
+			post.setEntity(entity);
+			try (CloseableHttpResponse resp = client.execute(post)) {
+				ourLog.info(Arrays.asList(resp.getAllHeaders()).toString().replace(", ", "\n"));
+				ourLog.info(resp.toString());
+				ourLog.info(IOUtils.toString(resp.getEntity().getContent(), Charsets.UTF_8));
+			}
+
+		}
+	}
+
+	private static class BlockingInputStream extends InputStream {
+		private final ByteArrayInputStream myWrap;
+		private int myByteCount = 0;
+
+		public BlockingInputStream(byte[] theBytes) {
+			myWrap = new ByteArrayInputStream(theBytes);
+		}
+
+		@Override
+		public int read() throws IOException {
+			if (myByteCount++ == 10) {
+				ourLog.info("About to block...");
+				try {
+					Thread.sleep(30000);
+				} catch (InterruptedException e) {
+					ourLog.warn("Interrupted", e);
+				}
+			}
+			return myWrap.read();
+		}
+	}
+
+
+	public static class SystemProvider {
+
+		@Transaction
+		public Bundle transaction(@TransactionParam Bundle theInput) {
+			return theInput;
+		}
+
+	}
+
+}

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/ServerInvalidDefinitionR4Test.java

@@ -1,30 +1,30 @@
 package ca.uhn.fhir.rest.server;
 
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.rest.annotation.ConditionalUrlParam;
+import ca.uhn.fhir.rest.annotation.Operation;
+import ca.uhn.fhir.rest.annotation.OptionalParam;
+import ca.uhn.fhir.rest.annotation.ResourceParam;
+import ca.uhn.fhir.rest.annotation.Update;
+import ca.uhn.fhir.rest.annotation.Validate;
+import ca.uhn.fhir.rest.api.MethodOutcome;
+import ca.uhn.fhir.rest.param.TokenParam;
+import ca.uhn.fhir.util.TestUtil;
+import org.hamcrest.core.StringContains;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.r4.model.Patient;
+import org.hl7.fhir.r4.model.StringType;
+import org.junit.AfterClass;
+import org.junit.Test;
+
+import javax.servlet.ServletException;
+
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
 
-import javax.servlet.ServletException;
+public class ServerInvalidDefinitionR4Test {
 
-import org.hamcrest.core.StringContains;
+	private static FhirContext ourCtx = FhirContext.forR4();
-import org.hl7.fhir.instance.model.api.IBaseResource;
-import org.junit.AfterClass;
-import org.junit.Test;
-
-import ca.uhn.fhir.context.FhirContext;
-import ca.uhn.fhir.model.dstu2.resource.Patient;
-import ca.uhn.fhir.rest.annotation.*;
-import ca.uhn.fhir.rest.api.MethodOutcome;
-import ca.uhn.fhir.rest.param.TokenParam;
-import ca.uhn.fhir.util.TestUtil;
-
-public class ServerInvalidDefinitionDstu2Test {
-
-	private static FhirContext ourCtx = FhirContext.forDstu2();
-
-	@AfterClass
-	public static void afterClassClearContext() {
-		TestUtil.clearAllStaticFieldsForUnitTest();
-	}
 
 	@Test
 	public void testWrongConditionalUrlType() {
@@ -38,7 +38,7 @@ public class ServerInvalidDefinitionDstu2Test {
 		} catch (ServletException e) {
 			assertThat(e.getCause().toString(), StringContains.containsString("ConfigurationException"));
 			assertThat(e.getCause().toString(), StringContains.containsString(
-					"Parameters annotated with @ConditionalUrlParam must be of type String, found incorrect parameteter in method \"public ca.uhn.fhir.rest.api.MethodOutcome ca.uhn.fhir.rest.server.ServerInvalidDefinitionDstu2Test$UpdateWithWrongConditionalUrlType.update(ca.uhn.fhir.rest.param.TokenParam,ca.uhn.fhir.model.dstu2.resource.Patient)"));
+				"Parameters annotated with @ConditionalUrlParam must be of type String, found incorrect parameter in method \"public ca.uhn.fhir.rest.api.MethodOutcome ca.uhn.fhir.rest.server.ServerInvalidDefinitionR4Test$UpdateWithWrongConditionalUrlType.update(ca.uhn.fhir.rest.param.TokenParam,org.hl7.fhir.r4.model.Patient)"));
 		}
 	}
 
@@ -54,7 +54,7 @@ public class ServerInvalidDefinitionDstu2Test {
 		} catch (ServletException e) {
 			assertThat(e.getCause().toString(), StringContains.containsString("ConfigurationException"));
 			assertThat(e.getCause().toString(), StringContains
-					.containsString("Method 'update' is annotated with @ResourceParam but has a type that is not an implemtation of org.hl7.fhir.instance.model.api.IBaseResource or String or byte[]"));
+				.containsString("Method 'update' is annotated with @ResourceParam but has a type that is not an implemtation of org.hl7.fhir.instance.model.api.IBaseResource or String or byte[]"));
 		}
 	}
 
@@ -88,6 +88,29 @@ public class ServerInvalidDefinitionDstu2Test {
 		}
 	}
 
+	@Test
+	public void testWrongParameterAnnotationOnOperation() {
+		class MyProvider {
+
+			@Operation(name = "foo")
+			public MethodOutcome update(@OptionalParam(name = "foo") StringType theFoo) {
+				return null;
+			}
+
+		}
+
+		RestfulServer srv = new RestfulServer(ourCtx);
+		srv.setFhirContext(ourCtx);
+		srv.registerProvider(new MyProvider());
+
+		try {
+			srv.init();
+			fail();
+		} catch (ServletException e) {
+			assertThat(e.getCause().toString(), StringContains.containsString("Failure scanning class MyProvider: Illegal method parameter annotation @OptionalParam on method: public ca.uhn.fhir.rest.api.MethodOutcome ca.uhn.fhir.rest.server.ServerInvalidDefinitionR4Test$1MyProvider.update(org.hl7.fhir.r4.model.StringType)"));
+		}
+	}
+
 	public static class UpdateWithWrongConditionalUrlType implements IResourceProvider {
 
 		@Override
@@ -144,4 +167,9 @@ public class ServerInvalidDefinitionDstu2Test {
 
 	}
 
+	@AfterClass
+	public static void afterClassClearContext() {
+		TestUtil.clearAllStaticFieldsForUnitTest();
+	}
+
 }

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/ServerMimetypeR4Test.java

@@ -1,18 +1,17 @@
 package ca.uhn.fhir.rest.server;
 
-import static org.hamcrest.Matchers.contains;
+import ca.uhn.fhir.context.FhirContext;
-import static org.hamcrest.Matchers.containsString;
+import ca.uhn.fhir.rest.annotation.Create;
-import static org.hamcrest.Matchers.not;
+import ca.uhn.fhir.rest.annotation.IdParam;
-import static org.junit.Assert.assertEquals;
+import ca.uhn.fhir.rest.annotation.Read;
-import static org.junit.Assert.assertThat;
+import ca.uhn.fhir.rest.annotation.ResourceParam;
-
+import ca.uhn.fhir.rest.annotation.Search;
-import java.io.IOException;
+import ca.uhn.fhir.rest.api.Constants;
-import java.net.URI;
+import ca.uhn.fhir.rest.api.EncodingEnum;
-import java.nio.charset.StandardCharsets;
+import ca.uhn.fhir.rest.api.MethodOutcome;
-import java.util.ArrayList;
+import ca.uhn.fhir.rest.client.MyPatientWithExtensions;
-import java.util.List;
+import ca.uhn.fhir.test.utilities.JettyUtil;
-import java.util.concurrent.TimeUnit;
+import ca.uhn.fhir.util.TestUtil;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -40,18 +39,18 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import ca.uhn.fhir.context.FhirContext;
+import java.io.IOException;
-import ca.uhn.fhir.rest.annotation.Create;
+import java.net.URI;
-import ca.uhn.fhir.rest.annotation.IdParam;
+import java.nio.charset.StandardCharsets;
-import ca.uhn.fhir.rest.annotation.Read;
+import java.util.ArrayList;
-import ca.uhn.fhir.rest.annotation.ResourceParam;
+import java.util.List;
-import ca.uhn.fhir.rest.annotation.Search;
+import java.util.concurrent.TimeUnit;
-import ca.uhn.fhir.rest.api.Constants;
+
-import ca.uhn.fhir.rest.api.EncodingEnum;
+import static org.hamcrest.Matchers.contains;
-import ca.uhn.fhir.rest.api.MethodOutcome;
+import static org.hamcrest.Matchers.containsString;
-import ca.uhn.fhir.rest.client.MyPatientWithExtensions;
+import static org.hamcrest.Matchers.not;
-import ca.uhn.fhir.test.utilities.JettyUtil;
+import static org.junit.Assert.assertEquals;
-import ca.uhn.fhir.util.TestUtil;
+import static org.junit.Assert.assertThat;
 
 public class ServerMimetypeR4Test {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ServerMimetypeR4Test.class);
@@ -406,45 +405,16 @@ public class ServerMimetypeR4Test {
 		return retVal;
 	}
 
-	@AfterClass
-	public static void afterClassClearContext() throws Exception {
-		JettyUtil.closeServer(ourServer);
-		TestUtil.clearAllStaticFieldsForUnitTest();
-	}
-
-	@BeforeClass
-	public static void beforeClass() throws Exception {
-		ourServer = new Server(0);
-
-		PatientProvider patientProvider = new PatientProvider();
-
-		ServletHandler proxyHandler = new ServletHandler();
-		ourServlet = new RestfulServer(ourCtx);
-
-		ourServlet.setResourceProviders(patientProvider);
-		ServletHolder servletHolder = new ServletHolder(ourServlet);
-		proxyHandler.addServletWithMapping(servletHolder, "/*");
-		ourServer.setHandler(proxyHandler);
-		JettyUtil.startServer(ourServer);
-        ourPort = JettyUtil.getPortForStartedServer(ourServer);
-
-		PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
-		HttpClientBuilder builder = HttpClientBuilder.create();
-		builder.setConnectionManager(connectionManager);
-		ourClient = builder.build();
-
-	}
-
 	public static class PatientProvider implements IResourceProvider {
 
 		@Create()
 		public MethodOutcome create(@ResourceParam Patient theIdParam) {
 			OperationOutcome oo = new OperationOutcome();
 			oo.addIssue().setDiagnostics(theIdParam.getNameFirstRep().getFamily());
-			
+
 			theIdParam.setId("1");
 			theIdParam.getMeta().setVersionId("1");
-			
+
 			return new MethodOutcome(new IdType("Patient", "1"), true).setOperationOutcome(oo).setResource(theIdParam);
 		}
 
@@ -480,4 +450,33 @@ public class ServerMimetypeR4Test {
 
 	}
 
+	@AfterClass
+	public static void afterClassClearContext() throws Exception {
+		JettyUtil.closeServer(ourServer);
+		TestUtil.clearAllStaticFieldsForUnitTest();
+	}
+
+	@BeforeClass
+	public static void beforeClass() throws Exception {
+		ourServer = new Server(0);
+
+		PatientProvider patientProvider = new PatientProvider();
+
+		ServletHandler proxyHandler = new ServletHandler();
+		ourServlet = new RestfulServer(ourCtx);
+
+		ourServlet.setResourceProviders(patientProvider);
+		ServletHolder servletHolder = new ServletHolder(ourServlet);
+		proxyHandler.addServletWithMapping(servletHolder, "/*");
+		ourServer.setHandler(proxyHandler);
+		JettyUtil.startServer(ourServer);
+		ourPort = JettyUtil.getPortForStartedServer(ourServer);
+
+		PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
+		HttpClientBuilder builder = HttpClientBuilder.create();
+		builder.setConnectionManager(connectionManager);
+		ourClient = builder.build();
+
+	}
+
 }

hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/server/ResourceProviderRule.java

@@ -0,0 +1,55 @@
+package ca.uhn.fhir.test.utilities.server;
+
+/*-
+ * #%L
+ * HAPI FHIR Test Utilities
+ * %%
+ * Copyright (C) 2014 - 2019 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+public class ResourceProviderRule implements TestRule {
+
+	private final RestfulServerRule myRestfulServerRule;
+	private Object myProvider;
+
+	/**
+	 * Constructor
+	 */
+	public ResourceProviderRule(RestfulServerRule theRestfulServerRule, Object theProvider) {
+		myRestfulServerRule = theRestfulServerRule;
+		myProvider = theProvider;
+	}
+
+	@Override
+	public Statement apply(Statement base, Description description) {
+		return new Statement() {
+			@Override
+			public void evaluate() throws Throwable {
+				myRestfulServerRule.getRestfulServer().registerProvider(myProvider);
+				try {
+					base.evaluate();
+				} finally {
+					myRestfulServerRule.getRestfulServer().unregisterProvider(myProvider);
+				}
+			}
+		};
+	}
+
+}

hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/server/RestfulServerRule.java

@@ -112,4 +112,8 @@ public class RestfulServerRule implements TestRule {
 	public RestfulServer getRestfulServer() {
 		return myServlet;
 	}
+
+	public int getPort() {
+		return myPort;
+	}
 }

pom.xml

@@ -615,7 +615,7 @@
 		<jaxb_runtime_version>2.3.1</jaxb_runtime_version>
 		<jersey_version>2.25.1</jersey_version>
 		<!-- 9.4.17 seems to have issues -->
-		<jetty_version>9.4.14.v20181114</jetty_version>
+		<jetty_version>9.4.23.v20191118</jetty_version>
 		<jsr305_version>3.0.2</jsr305_version>
 		<!--<hibernate_version>5.2.10.Final</hibernate_version>-->
 		<hibernate_version>5.4.6.Final</hibernate_version>

src/changes/changes.xml

@@ -6,6 +6,38 @@
 		<title>HAPI FHIR Changelog</title>
 	</properties>
 	<body>
+		<release version="4.2.0" date="TBD">
+			<action type="add">
+				The version of a few dependencies have been bumped to the
+				latest versions (dependent HAPI modules listed in brackets):
+				<![CDATA[
+				<ul>
+					<li>Jetty (CLI): 9.4.14.v20181114 -&gt; 9.4.23.v20191118</li>
+				</ul>
+				]]>
+			</action>
+			<action type="fix" issue="1583">
+				As of FHIR R4, some fields that were previously of type reference are now of type canonical.
+				One example is QuestionnaireResponse.questionnaire. Technically this means that this field
+				should no longer contain a relative reference, but as they are sometimes used that way, HAPI
+				FHIR will now try to be permissive and will index relative link canonical fields 
+				such as (Questionnaire/123) as though it actually was a local relative link. Thanks to 
+				Dean Atchley for reporting and providing a test case!
+			</action>
+			<action type="fix">
+				ValueSet PreCalculation did not successfully expand valuesets when Lucene was not enabled
+				in the JPA server. This has been corrected.
+			</action>
+			<action type="change">
+				When parsing Bundle resources containing other resources, XML/JSON parsers have an option called 
+				"override resource ID with bundle entry fullUrl". This option previously caused any value
+				found in Bundle.entry.fullUrl to override any value found in 
+				Bundle.entry.resource.id (meaning that the parsed resource would take its ID from
+				the fullUrl even if that ID disagreed with the ID found in the resource itself. As of
+				HAPI FHIR 4.1.0 the value in Bundle.entry.fullUrl will only be used to set the parsed resource
+				ID if the resource has no ID present.
+			</action>
+		</release>
 		<release version="4.1.0" date="2019-11-13" description="Jitterbug">
 			<action type="add">
 				The version of a few dependencies have been bumped to the