Test fixes

hapi-fhir-base/src/main/java/ca/uhn/fhir/model/api/BaseElement.java

@@ -20,31 +20,29 @@ package ca.uhn.fhir.model.api;
  * #L%
  */
 
-import java.util.ArrayList;
+import ca.uhn.fhir.model.api.annotation.Child;
-import java.util.Collections;
+import ca.uhn.fhir.model.api.annotation.Description;
-import java.util.List;
-
 import org.apache.commons.lang3.Validate;
 import org.hl7.fhir.instance.model.api.IBaseDatatype;
 
-import ca.uhn.fhir.model.api.annotation.Child;
+import java.util.*;
-import ca.uhn.fhir.model.api.annotation.Description;
 
 public abstract class BaseElement implements /*IElement, */ISupportsUndeclaredExtensions {
 
 	private static final long serialVersionUID = -3092659584634499332L;
 	private List<String> myFormatCommentsPost;
 	private List<String> myFormatCommentsPre;
+	private Map<String, Object> userData;
 
-   @Child(name = "extension", type = {ExtensionDt.class}, order=0, min=0, max=Child.MAX_UNLIMITED, modifier=false, summary=false)
+	@Child(name = "extension", type = {ExtensionDt.class}, order = 0, min = 0, max = Child.MAX_UNLIMITED, modifier = false, summary = false)
-   @Description(shortDefinition="Additional Content defined by implementations", formalDefinition="May be used to represent additional information that is not part of the basic definition of the resource. In order to make the use of extensions safe and manageable, there is a strict set of governance  applied to the definition and use of extensions. Though any implementer is allowed to define an extension, there is a set of requirements that SHALL be met as part of the definition of the extension." )
+	@Description(shortDefinition = "Additional Content defined by implementations", formalDefinition = "May be used to represent additional information that is not part of the basic definition of the resource. In order to make the use of extensions safe and manageable, there is a strict set of governance  applied to the definition and use of extensions. Though any implementer is allowed to define an extension, there is a set of requirements that SHALL be met as part of the definition of the extension.")
 	private List<ExtensionDt> myUndeclaredExtensions;
 
-   /**
+	/**
-    * May be used to represent additional information that is not part of the basic definition of the resource, and that modifies the understanding of the element that contains it. Usually modifier elements provide negation or qualification. In order to make the use of extensions safe and manageable, there is a strict set of governance applied to the definition and use of extensions. Though any implementer is allowed to define an extension, there is a set of requirements that SHALL be met as part of the definition of the extension. Applications processing a resource are required to check for modifier extensions.
+	 * May be used to represent additional information that is not part of the basic definition of the resource, and that modifies the understanding of the element that contains it. Usually modifier elements provide negation or qualification. In order to make the use of extensions safe and manageable, there is a strict set of governance applied to the definition and use of extensions. Though any implementer is allowed to define an extension, there is a set of requirements that SHALL be met as part of the definition of the extension. Applications processing a resource are required to check for modifier extensions.
-    */
+	 */
-   @Child(name = "modifierExtension", type = {ExtensionDt.class}, order=1, min=0, max=Child.MAX_UNLIMITED, modifier=true, summary=false)
+	@Child(name = "modifierExtension", type = {ExtensionDt.class}, order = 1, min = 0, max = Child.MAX_UNLIMITED, modifier = true, summary = false)
-   @Description(shortDefinition="Extensions that cannot be ignored", formalDefinition="May be used to represent additional information that is not part of the basic definition of the resource, and that modifies the understanding of the element that contains it. Usually modifier elements provide negation or qualification. In order to make the use of extensions safe and manageable, there is a strict set of governance applied to the definition and use of extensions. Though any implementer is allowed to define an extension, there is a set of requirements that SHALL be met as part of the definition of the extension. Applications processing a resource are required to check for modifier extensions." )
+	@Description(shortDefinition = "Extensions that cannot be ignored", formalDefinition = "May be used to represent additional information that is not part of the basic definition of the resource, and that modifies the understanding of the element that contains it. Usually modifier elements provide negation or qualification. In order to make the use of extensions safe and manageable, there is a strict set of governance applied to the definition and use of extensions. Though any implementer is allowed to define an extension, there is a set of requirements that SHALL be met as part of the definition of the extension. Applications processing a resource are required to check for modifier extensions.")
 	private List<ExtensionDt> myUndeclaredModifierExtensions;
 
 	@Override
@@ -148,6 +146,21 @@ public abstract class BaseElement implements /*IElement, */ISupportsUndeclaredEx
 		return (myFormatCommentsPre != null && !myFormatCommentsPre.isEmpty()) || (myFormatCommentsPost != null && !myFormatCommentsPost.isEmpty());
 	}
 
+	@Override
+	public Object getUserData(String name) {
+		if (userData == null)
+			return null;
+		return userData.get(name);
+	}
+
+	@Override
+	public void setUserData(String name, Object value) {
+		if (userData == null) {
+			userData = new HashMap<>();
+		}
+		userData.put(name, value);
+	}
+
 	/**
 	 * Intended to be called by extending classes {@link #isEmpty()} implementations, returns <code>true</code> if all
 	 * content in this superclass instance is empty per the semantics of {@link #isEmpty()}.

hapi-fhir-base/src/main/java/ca/uhn/fhir/model/api/IElement.java

@@ -26,4 +26,5 @@ import org.hl7.fhir.instance.model.api.IBase;
 
 public interface IElement extends IBase {
 
+
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/model/api/TagList.java

@@ -282,4 +282,14 @@ public class TagList implements Set<Tag>, Serializable, IBase {
 		throw new UnsupportedOperationException();
 	}
 
+	@Override
+	public Object getUserData(String theName) {
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public void setUserData(String theName, Object theValue) {
+		throw new UnsupportedOperationException();
+	}
+
 }

hapi-fhir-base/src/main/java/org/hl7/fhir/instance/model/api/IBase.java

@@ -61,4 +61,14 @@ public interface IBase extends Serializable {
 	 */
 	default String fhirType() { return null; }
 
+	/**
+	 * Retrieves any user suplied data in this element
+	 */
+	Object getUserData(String theName);
+
+	/**
+	 * Sets a user supplied data value in this element
+	 */
+	void setUserData(String theName, Object theValue);
+
 }

hapi-fhir-base/src/main/java/org/hl7/fhir/instance/model/api/IBaseElement.java

@@ -26,4 +26,8 @@ public interface IBaseElement {
 
 	String getId();
 
+	Object getUserData(String theName);
+
+	void setUserData(String theName, Object theValue);
+
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/binstore/BinaryAccessProvider.java

@@ -201,6 +201,7 @@ public class BinaryAccessProvider {
 
 			IBaseExtension<?, ?> ext = target.getTarget().addExtension();
 			ext.setUrl(JpaConstants.EXT_EXTERNALIZED_BINARY_ID);
+			ext.setUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED, Boolean.TRUE);
 			IPrimitiveType<String> blobIdString = (IPrimitiveType<String>) myCtx.getElementDefinition("string").newInstance();
 			blobIdString.setValueAsString(blobId);
 			ext.setValue(blobIdString);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/binstore/MemoryBinaryStorageSvcImpl.java

@@ -85,6 +85,7 @@ public class MemoryBinaryStorageSvcImpl extends BaseBinaryStorageSvcImpl impleme
 	public void expungeBlob(IIdType theResourceId, String theBlobId) {
 		String key = toKey(theResourceId, theBlobId);
 		myDataMap.remove(key);
+		myDetailsMap.remove(key);
 	}
 
 	private String toKey(IIdType theResourceId, String theBlobId) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java

@@ -173,7 +173,6 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 	private SearchBuilderFactory mySearchBuilderFactory;
 	private FhirContext myContext;
 	private ApplicationContext myApplicationContext;
-	private Class<? extends IPrimitiveType<byte[]>> myBase64Type;
 
 	@Override
 	public void setApplicationContext(ApplicationContext theApplicationContext) throws BeansException {
@@ -1448,21 +1447,6 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 
 		validateMetaCount(totalMetaCount);
 
-		List<? extends IPrimitiveType<byte[]>> base64fields = myContext.newTerser().getAllPopulatedChildElementsOfType(theResource, myBase64Type);
-		for (IPrimitiveType<byte[]> nextBase64 : base64fields) {
-			if (nextBase64 instanceof IBaseHasExtensions) {
-				boolean hasExternalizedBinaryReference = ((IBaseHasExtensions) nextBase64)
-					.getExtension()
-					.stream()
-					.anyMatch(t-> t.getUrl().equals(EXT_EXTERNALIZED_BINARY_ID));
-				if (hasExternalizedBinaryReference) {
-					String msg = getContext().getLocalizer().getMessage(BaseHapiFhirDao.class, "externalizedBinaryStorageExtensionFoundInRequestBody", EXT_EXTERNALIZED_BINARY_ID);
-					throw new InvalidRequestException(msg);
-				}
-			}
-		}
-
-
 	}
 
 	@Override
@@ -1470,10 +1454,9 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 		return mySearchParamRegistry;
 	}
 
-	@SuppressWarnings("unchecked")
 	@PostConstruct
 	public void start() {
-		myBase64Type = (Class<? extends IPrimitiveType<byte[]>>) myContext.getElementDefinition("base64Binary").getImplementingClass();
+		// nothing yet
 	}
 
 	@SuppressWarnings("unchecked")

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java

@@ -29,6 +29,7 @@ import ca.uhn.fhir.interceptor.api.Pointcut;
 import ca.uhn.fhir.jpa.delete.DeleteConflictList;
 import ca.uhn.fhir.jpa.model.entity.*;
 import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
 import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider;
 import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
@@ -75,6 +76,7 @@ import javax.validation.constraints.NotNull;
 import java.io.IOException;
 import java.util.*;
 
+import static ca.uhn.fhir.jpa.model.util.JpaConstants.EXT_EXTERNALIZED_BINARY_ID;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 @Transactional(propagation = Propagation.REQUIRED)
@@ -96,6 +98,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 	private String myResourceName;
 	private Class<T> myResourceType;
 	private String mySecondaryPrimaryKeyParamName;
+	private Class<? extends IPrimitiveType<byte[]>> myBase64Type;
 
 	@Override
 	public void addTag(IIdType theId, TagTypeEnum theTagType, String theScheme, String theTerm, String theLabel, RequestDetails theRequest) {
@@ -871,6 +874,25 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 			}
 		}
 
+		/*
+		 * Don't allow clients to submit resources with binary storage attachments present.
+		 */
+		List<? extends IPrimitiveType<byte[]>> base64fields = getContext().newTerser().getAllPopulatedChildElementsOfType(theResource, myBase64Type);
+		for (IPrimitiveType<byte[]> nextBase64 : base64fields) {
+			if (nextBase64 instanceof IBaseHasExtensions) {
+				boolean hasExternalizedBinaryReference = ((IBaseHasExtensions) nextBase64)
+					.getExtension()
+					.stream()
+					.filter(t-> t.getUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED) == null)
+					.anyMatch(t-> t.getUrl().equals(EXT_EXTERNALIZED_BINARY_ID));
+				if (hasExternalizedBinaryReference) {
+					String msg = getContext().getLocalizer().getMessage(BaseHapiFhirDao.class, "externalizedBinaryStorageExtensionFoundInRequestBody", EXT_EXTERNALIZED_BINARY_ID);
+					throw new InvalidRequestException(msg);
+				}
+			}
+		}
+
+
 	}
 
 	@Override
@@ -1178,6 +1200,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 	public void start() {
 		super.start();
 		ourLog.debug("Starting resource DAO for type: {}", getResourceName());
+		myBase64Type = (Class<? extends IPrimitiveType<byte[]>>) getContext().getElementDefinition("base64Binary").getImplementingClass();
 	}
 
 	protected <MT extends IBaseMetaType> MT toMetaDt(Class<MT> theType, Collection<TagDefinition> tagDefinitions) {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/BinaryAccessProviderR4Test.java

@@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.provider.r4;
 import ca.uhn.fhir.interceptor.api.HookParams;
 import ca.uhn.fhir.interceptor.api.IAnonymousInterceptor;
 import ca.uhn.fhir.interceptor.api.Pointcut;
+import ca.uhn.fhir.jpa.binstore.IBinaryStorageSvc;
 import ca.uhn.fhir.jpa.binstore.MemoryBinaryStorageSvcImpl;
 import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.model.util.JpaConstants;
@@ -41,6 +42,8 @@ public class BinaryAccessProviderR4Test extends BaseResourceProviderR4Test {
 
 	@Autowired
 	private MemoryBinaryStorageSvcImpl myStorageSvc;
+	@Autowired
+	private IBinaryStorageSvc myBinaryStorageSvc;
 
 	@Override
 	@Before
@@ -165,14 +168,38 @@ public class BinaryAccessProviderR4Test extends BaseResourceProviderR4Test {
 	public void testReadUnknownBlobId() throws IOException {
 		IIdType id = createDocumentReference(false);
 
-		DocumentReference dr = ourClient.read().resource(DocumentReference.class).withId(id).execute();
+		// Write a binary using the operation
-		dr.getContentFirstRep()
-			.getAttachment()
-			.getDataElement()
-			.addExtension(JpaConstants.EXT_EXTERNALIZED_BINARY_ID, new StringType("AAAAA"));
-		ourClient.update().resource(dr).execute();
 
 		String path = ourServerBase +
+			"/DocumentReference/" + id.getIdPart() + "/" +
+			JpaConstants.OPERATION_BINARY_ACCESS_WRITE +
+			"?path=DocumentReference.content.attachment";
+		HttpPost post = new HttpPost(path);
+		post.setEntity(new ByteArrayEntity(SOME_BYTES, ContentType.IMAGE_JPEG));
+		post.addHeader("Accept", "application/fhir+json; _pretty=true");
+		String attachmentId;
+		try (CloseableHttpResponse resp = ourHttpClient.execute(post)) {
+
+			assertEquals(200, resp.getStatusLine().getStatusCode());
+			assertThat(resp.getEntity().getContentType().getValue(), containsString("application/fhir+json"));
+			String response = IOUtils.toString(resp.getEntity().getContent(), Constants.CHARSET_UTF8);
+			ourLog.info("Response: {}", response);
+
+			DocumentReference ref = myFhirCtx.newJsonParser().parseResource(DocumentReference.class, response);
+
+			Attachment attachment = ref.getContentFirstRep().getAttachment();
+			assertEquals(ContentType.IMAGE_JPEG.getMimeType(), attachment.getContentType());
+			assertEquals(15, attachment.getSize());
+			assertEquals(null, attachment.getData());
+			assertEquals("2", ref.getMeta().getVersionId());
+			attachmentId = attachment.getDataElement().getExtensionString(JpaConstants.EXT_EXTERNALIZED_BINARY_ID);
+			assertThat(attachmentId, matchesPattern("[a-zA-Z0-9]{100}"));
+		}
+
+
+		myBinaryStorageSvc.expungeBlob(id, attachmentId);
+
+		path = ourServerBase +
 			"/DocumentReference/" + id.getIdPart() + "/" +
 			JpaConstants.OPERATION_BINARY_ACCESS_READ +
 			"?path=DocumentReference.content.attachment";

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java

@@ -238,6 +238,11 @@ public class JpaConstants {
 	 */
 	public static final String EXT_EXTERNALIZED_BINARY_ID = "http://hapifhir.io/fhir/StructureDefinition/externalized-binary-id";
 
+	/**
+	 * Placed in system-generated extensions
+	 */
+	public static final String EXTENSION_EXT_SYSTEMDEFINED = JpaConstants.class.getName() + "_EXTENSION_EXT_SYSTEMDEFINED";
+
 	/**
 	 * <p>
 	 * This extension represents the equivalent of the

hapi-fhir-structures-dstu2/src/main/java/ca/uhn/fhir/model/dstu2/composite/ContainedDt.java

@@ -51,4 +51,14 @@ public class ContainedDt extends BaseContainedDt {
 		return myContainedResources == null || myContainedResources.size() == 0;
 	}
 
+	@Override
+	public Object getUserData(String theName) {
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public void setUserData(String theName, Object theValue) {
+		throw new UnsupportedOperationException();
+	}
+
 }

hapi-fhir-structures-dstu2/src/main/java/ca/uhn/fhir/model/dstu2/resource/BaseResource.java

@@ -168,6 +168,16 @@ public abstract class BaseResource extends BaseElement implements IResource {
 				return Collections.emptyList();
 			}
 
+			@Override
+			public Object getUserData(String theName) {
+				throw new UnsupportedOperationException();
+			}
+
+			@Override
+			public void setUserData(String theName, Object theValue) {
+				throw new UnsupportedOperationException();
+			}
+
 			@Override
 			public List<String> getFormatCommentsPre() {
 				return Collections.emptyList();

hapi-fhir-structures-dstu2/src/test/java/ca/uhn/fhir/rest/client/GenericClientDstu2Test.java

@@ -1193,6 +1193,16 @@ public class GenericClientDstu2Test {
 				return null;
 			}
 
+			@Override
+			public Object getUserData(String theName) {
+				throw new UnsupportedOperationException();
+			}
+
+			@Override
+			public void setUserData(String theName, Object theValue) {
+				throw new UnsupportedOperationException();
+			}
+
 			@Override
 			public List<String> getFormatCommentsPre() {
 				return null;

src/changes/changes.xml

@@ -579,6 +579,14 @@
 				can now be used to upload custom vocabulary that has been converted into a standard file format
 				defined by HAPI FHIR. This is useful for uploading large organizational code systems.
 			</action>
+			<action type="change">
+				Two new operations,
+				<![CDATA[<code>$apply-codesystem-delta-add</code>]]>
+				and
+				<![CDATA[<code>$apply-codesystem-delta-remove</code>]]>
+				have been added to the terminology server. These methods allow
+				codes to be dynamically added and removed from external (notpresent) codesystems.
+			</action>
 			<action type="fix" issue="1404">
 				In the JAX-RS server, the resource type history and instance vread
 				operations had ambiguous paths that could lead to the wrong method