Improve parallel model scanning (#3046)

* Improve parallel model scanning

* Add changelog

* Resolve fixme

* Test fix

hapi-fhir-base/src/main/java/ca/uhn/fhir/context/BaseRuntimeElementCompositeDefinition.java

@@ -1,57 +1,5 @@
 package ca.uhn.fhir.context;
 
-import static org.apache.commons.lang3.StringUtils.isNotBlank;
-
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-
-/*
- * #%L
- * HAPI FHIR - Core Library
- * %%
- * Copyright (C) 2014 - 2021 Smile CDR, Inc.
- * %%
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * #L%
- */
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.hl7.fhir.instance.model.api.IAnyResource;
-import org.hl7.fhir.instance.model.api.IBase;
-import org.hl7.fhir.instance.model.api.IBaseBackboneElement;
-import org.hl7.fhir.instance.model.api.IBaseDatatype;
-import org.hl7.fhir.instance.model.api.IBaseDatatypeElement;
-import org.hl7.fhir.instance.model.api.IBaseEnumeration;
-import org.hl7.fhir.instance.model.api.IBaseExtension;
-import org.hl7.fhir.instance.model.api.IBaseReference;
-import org.hl7.fhir.instance.model.api.IBaseResource;
-import org.hl7.fhir.instance.model.api.ICompositeType;
-import org.hl7.fhir.instance.model.api.INarrative;
-import org.hl7.fhir.instance.model.api.IPrimitiveType;
-
 import ca.uhn.fhir.model.api.IBoundCodeableConcept;
 import ca.uhn.fhir.model.api.IDatatype;
 import ca.uhn.fhir.model.api.IElement;
@@ -69,18 +17,47 @@ import ca.uhn.fhir.model.base.composite.BaseResourceReferenceDt;
 import ca.uhn.fhir.model.primitive.BoundCodeDt;
 import ca.uhn.fhir.parser.DataFormatException;
 import ca.uhn.fhir.util.ReflectionUtil;
+import org.hl7.fhir.instance.model.api.IAnyResource;
+import org.hl7.fhir.instance.model.api.IBase;
+import org.hl7.fhir.instance.model.api.IBaseBackboneElement;
+import org.hl7.fhir.instance.model.api.IBaseDatatype;
+import org.hl7.fhir.instance.model.api.IBaseDatatypeElement;
+import org.hl7.fhir.instance.model.api.IBaseEnumeration;
+import org.hl7.fhir.instance.model.api.IBaseExtension;
+import org.hl7.fhir.instance.model.api.IBaseReference;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.instance.model.api.ICompositeType;
+import org.hl7.fhir.instance.model.api.INarrative;
+import org.hl7.fhir.instance.model.api.IPrimitiveType;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> extends BaseRuntimeElementDefinition<T> {
 
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseRuntimeElementCompositeDefinition.class);
+	private final FhirContext myContext;
 	private Map<String, Integer> forcedOrder = null;
 	private List<BaseRuntimeChildDefinition> myChildren = new ArrayList<>();
 	private List<BaseRuntimeChildDefinition> myChildrenAndExtensions;
 	private Map<Class<? extends IBase>, BaseRuntimeElementDefinition<?>> myClassToElementDefinitions;
-	private final FhirContext myContext;
 	private Map<String, BaseRuntimeChildDefinition> myNameToChild = new HashMap<>();
 	private List<ScannedField> myScannedFields = new ArrayList<>();
-	private volatile boolean mySealed;
+	private volatile SealingStateEnum mySealed = SealingStateEnum.NOT_SEALED;
 
 	@SuppressWarnings("unchecked")
 	public BaseRuntimeElementCompositeDefinition(String theName, Class<? extends T> theImplementingClass, boolean theStandardType, FhirContext theContext, Map<Class<? extends IBase>, BaseRuntimeElementDefinition<?>> theClassToElementDefinitions) {
@@ -108,7 +85,7 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
 					}
 				}
 			}
-			classes .push(current);
+			classes.push(current);
 			if (IBase.class.isAssignableFrom(current.getSuperclass())) {
 				current = (Class<? extends IBase>) current.getSuperclass();
 			} else {
@@ -143,7 +120,7 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
 	}
 
 	@Override
-	public BaseRuntimeChildDefinition getChildByName(String theName){
+	public BaseRuntimeChildDefinition getChildByName(String theName) {
 		validateSealed();
 		return myNameToChild.get(theName);
 	}
@@ -174,7 +151,7 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
 	 * Has this class been sealed
 	 */
 	public boolean isSealed() {
-		return mySealed;
+		return mySealed == SealingStateEnum.SEALED;
 	}
 
 	@SuppressWarnings("unchecked")
@@ -384,7 +361,7 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
 					def = new RuntimeChildDeclaredExtensionDefinition(nextField, childAnnotation, descriptionAnnotation, extensionAttr, elementName, extensionAttr.url(), et, binder);
 
 					if (IBaseEnumeration.class.isAssignableFrom(nextElementType)) {
-						((RuntimeChildDeclaredExtensionDefinition)def).setEnumerationType(ReflectionUtil.getGenericCollectionTypeOfFieldWithSecondOrderForList(nextField));
+						((RuntimeChildDeclaredExtensionDefinition) def).setEnumerationType(ReflectionUtil.getGenericCollectionTypeOfFieldWithSecondOrderForList(nextField));
 					}
 				} else if (BaseResourceReferenceDt.class.isAssignableFrom(nextElementType) || IBaseReference.class.isAssignableFrom(nextElementType)) {
 					/*
@@ -459,12 +436,18 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
 			elementNames.add(elementName);
 		}
 	}
+
 	@Override
 	void sealAndInitialize(FhirContext theContext, Map<Class<? extends IBase>, BaseRuntimeElementDefinition<?>> theClassToElementDefinitions) {
-		if (mySealed) {
+		if (mySealed == SealingStateEnum.SEALED) {
 			return;
 		}
-		mySealed = true;
+
+		synchronized (myContext) {
+			if (mySealed == SealingStateEnum.SEALED || mySealed == SealingStateEnum.SEALING) {
+				return;
+			}
+			mySealed = SealingStateEnum.SEALING;
 
 			scanCompositeElementForChildren();
 
@@ -477,7 +460,7 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
 			myNameToChild = new HashMap<>();
 			for (BaseRuntimeChildDefinition next : myChildren) {
 				if (next instanceof RuntimeChildChoiceDefinition) {
-				String key = next.getElementName()+"[x]";
+					String key = next.getElementName() + "[x]";
 					myNameToChild.put(key, next);
 				}
 				for (String nextName : next.getValidChildNames()) {
@@ -530,30 +513,27 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
 				children.addAll(findIndex(children, "modifierExtension", true), getExtensionsModifier());
 			}
 
-		myChildrenAndExtensions=Collections.unmodifiableList(children);
+			myChildrenAndExtensions = Collections.unmodifiableList(children);
+			mySealed = SealingStateEnum.SEALED;
+		}
 	}
 
 
 	@Override
 	protected void validateSealed() {
-		if (!mySealed) {
+		if (mySealed != SealingStateEnum.SEALED) {
-			synchronized(myContext) {
+			synchronized (myContext) {
-				if(!mySealed) {
+				if (mySealed == SealingStateEnum.NOT_SEALED) {
 					sealAndInitialize(myContext, myClassToElementDefinitions);
 				}
 			}
 		}
 	}
 
-	private static int findIndex(List<BaseRuntimeChildDefinition> theChildren, String theName, boolean theDefaultAtEnd) {
+	private enum SealingStateEnum {
-		int index = theDefaultAtEnd ? theChildren.size() : -1;
+		NOT_SEALED,
-		for (ListIterator<BaseRuntimeChildDefinition> iter = theChildren.listIterator(); iter.hasNext(); ) {
+		SEALING,
-			if (iter.next().getElementName().equals(theName)) {
+		SEALED
-				index = iter.previousIndex();
-				break;
-			}
-		}
-		return index;
 	}
 
 	private static class ScannedField {
@@ -563,6 +543,7 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
 		private Class<?> myElementType;
 		private Field myField;
 		private boolean myFirstFieldInNewClass;
+
 		ScannedField(Field theField, Class<?> theClass, boolean theFirstFieldInNewClass) {
 			myField = theField;
 			myFirstFieldInNewClass = theFirstFieldInNewClass;
@@ -609,4 +590,15 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
 		}
 	}
 
+	private static int findIndex(List<BaseRuntimeChildDefinition> theChildren, String theName, boolean theDefaultAtEnd) {
+		int index = theDefaultAtEnd ? theChildren.size() : -1;
+		for (ListIterator<BaseRuntimeChildDefinition> iter = theChildren.listIterator(); iter.hasNext(); ) {
+			if (iter.next().getElementName().equals(theName)) {
+				index = iter.previousIndex();
+				break;
+			}
+		}
+		return index;
+	}
+
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/context/BaseRuntimeElementDefinition.java

@@ -224,9 +224,8 @@ public abstract class BaseRuntimeElementDefinition<T extends IBase> {
 		/*
 		 * this does nothing, but BaseRuntimeElementCompositeDefinition
 		 * overrides this method to provide functionality because that class
-		 * defers the dealing process
+		 * defers the sealing process
 		 */
-
 	}
 
 	public BaseRuntimeChildDefinition getChildByName(String theChildName) {

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/FhirTerser.java

@@ -988,7 +988,8 @@ public class FhirTerser {
 			case RESOURCE_BLOCK:
 			case COMPOSITE_DATATYPE: {
 				BaseRuntimeElementCompositeDefinition<?> childDef = (BaseRuntimeElementCompositeDefinition<?>) def;
-				for (BaseRuntimeChildDefinition nextChild : childDef.getChildrenAndExtension()) {
+				List<BaseRuntimeChildDefinition> childrenAndExtensionDefs = childDef.getChildrenAndExtension();
+				for (BaseRuntimeChildDefinition nextChild : childrenAndExtensionDefs) {
 
 					List<?> values = nextChild.getAccessor().getValues(theElement);
 

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_6_0/3046-avoid-crash-on-parallel-scanning.yaml

@@ -0,0 +1,4 @@
+---
+type: fix
+issue: 3046
+title: When using deferred model scanning in highly parallelized environments, a crash could sometimes occur during parse/serialize operations.

hapi-fhir-structures-dstu3/src/test/java/ca/uhn/fhir/rest/server/InterceptorDstu3Test.java

@@ -58,6 +58,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.either;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -228,7 +230,7 @@ public class InterceptorDstu3Test {
 		httpPost.setEntity(new StringEntity(input, ContentType.create(Constants.CT_FHIR_JSON, "UTF-8")));
 		HttpResponse status = ourClient.execute(httpPost);
 		try {
-			assertEquals(201, status.getStatusLine().getStatusCode());
+			assertThat(status.getStatusLine().getStatusCode(), either(equalTo(200)).or(equalTo(201)));
 		} finally {
 			IOUtils.closeQuietly(status.getEntity().getContent());
 		}

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/parser/JsonParserR4Test.java

@@ -2,6 +2,7 @@ package ca.uhn.fhir.parser;
 
 import ca.uhn.fhir.context.ConfigurationException;
 import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.context.PerformanceOptionsEnum;
 import ca.uhn.fhir.model.api.annotation.DatatypeDef;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.test.BaseTest;
@@ -43,9 +44,15 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -270,6 +277,50 @@ public class JsonParserR4Test extends BaseTest {
 
 	}
 
+	/**
+	 * Make sure we can perform parallel parse/encodes against the same
+	 * FhirContext instance when deferred model scanning is enabled without
+	 * running into threading issues.
+	 */
+	@Test
+	public void testEncodeAndDecodeMultithreadedWithDeferredModelScanning() throws IOException, ExecutionException, InterruptedException {
+		String input = loadResource("/multi-thread-parsing-issue-bundle.json");
+
+		ExecutorService executor = Executors.newFixedThreadPool(10);
+		for (int pass = 0; pass < 10; pass++) {
+			ourLog.info("Starting pass: {}", pass);
+
+			FhirContext parseCtx = FhirContext.forR4();
+			parseCtx.setPerformanceOptions(PerformanceOptionsEnum.DEFERRED_MODEL_SCANNING);
+			List<Future<Bundle>> bundleFutures = new ArrayList<>();
+			for (int readIdx = 0; readIdx < 10; readIdx++) {
+				bundleFutures.add(executor.submit(()->parseCtx.newJsonParser().parseResource(Bundle.class, input)));
+			}
+
+			List<Bundle> parsedBundles = new ArrayList<>();
+			for (Future<Bundle> nextFuture : bundleFutures) {
+				Bundle nextBundle = nextFuture.get();
+				parsedBundles.add(nextBundle);
+			}
+
+			FhirContext encodeCtx = FhirContext.forR4();
+			encodeCtx.setPerformanceOptions(PerformanceOptionsEnum.DEFERRED_MODEL_SCANNING);
+			List<Future<String>> encodeFutures = new ArrayList<>();
+			for (Bundle nextBundle : parsedBundles) {
+				encodeFutures.add(executor.submit(()->encodeCtx.newJsonParser().encodeResourceToString(nextBundle)));
+			}
+
+			List<String> outputs = new ArrayList<>();
+			for (Future<String> nextFuture : encodeFutures) {
+				outputs.add(nextFuture.get());
+			}
+
+			assertEquals(outputs.get(0), outputs.get(1));
+		}
+	}
+
+
+
 	@Test
 	public void testEncodeAndParseUnicodeCharacterInNarrative() {
 		Patient p = new Patient();

hapi-fhir-structures-r4/src/test/resources/multi-thread-parsing-issue-bundle.json

@@ -0,0 +1,471 @@
+{
+	"resourceType": "Bundle",
+	"type": "transaction",
+	"entry": [
+		{
+			"fullUrl": "5b6a6591-ae0f-3bfc-987e-5f144a43a036",
+			"resource": {
+				"resourceType": "Condition",
+				"id": "5b6a6591-ae0f-3bfc-987e-5f144a43a036",
+				"meta": {
+					"profile": [
+						"http://hl7.org/fhir/us/core/StructureDefinition/us-core-condition"
+					]
+				},
+				"identifier": [
+					{
+						"system": "http://bluecrossnc.com/fhir/conditionIdentifier",
+						"value": "13028683"
+					}
+				],
+				"clinicalStatus": {
+					"coding": [
+						{
+							"system": "http://terminology.hl7.org/CodeSystem/condition-clinical",
+							"code": "active"
+						}
+					]
+				},
+				"verificationStatus": {
+					"coding": [
+						{
+							"system": "http://terminology.hl7.org/CodeSystem/condition-ver-status",
+							"code": "unconfirmed"
+						}
+					]
+				},
+				"category": [
+					{
+						"coding": [
+							{
+								"system": "http://terminology.hl7.org/CodeSystem/condition-category",
+								"code": "encounter-diagnosis"
+							}
+						]
+					}
+				],
+				"code": {
+					"coding": [
+						{
+							"system": "http://hl7.org/fhir/sid/icd-10-cm",
+							"code": "Z80.3"
+						}
+					]
+				},
+				"subject": {
+					"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=111111111",
+					"identifier": {
+						"type": {
+							"coding": [
+								{
+									"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
+									"code": "MB"
+								}
+							]
+						},
+						"value": "111111111"
+					}
+				}
+			},
+			"request": {
+				"method": "PUT",
+				"url": "Condition/5b6a6591-ae0f-3bfc-987e-5f144a43a036"
+			}
+		},
+		{
+			"fullUrl": "a621019a-fadb-37dd-964a-01c21926e9b4",
+			"resource": {
+				"resourceType": "Practitioner",
+				"id": "a621019a-fadb-37dd-964a-01c21926e9b4",
+				"meta": {
+					"profile": [
+						"http://hl7.org/fhir/us/core/StructureDefinition/us-core-practitioner",
+						"http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Practitioner"
+					]
+				},
+				"identifier": [
+					{
+						"type": {
+							"coding": [
+								{
+									"system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType",
+									"code": "npi"
+								}
+							]
+						},
+						"system": "http://hl7.org/fhir/sid/us-npi",
+						"value": "1134567787"
+					}
+				],
+				"active": true,
+				"name": [
+					{
+						"family": "BRIMMAGE",
+						"given": [
+							"BRAIN BRIMMAGE"
+						]
+					}
+				]
+			},
+			"request": {
+				"method": "PUT",
+				"url": "Practitioner/a621019a-fadb-37dd-964a-01c21926e9b4"
+			}
+		},
+		{
+			"fullUrl": "fb1490ab-c62e-33ca-8414-b6437b0fa578",
+			"resource": {
+				"resourceType": "Practitioner",
+				"id": "fb1490ab-c62e-33ca-8414-b6437b0fa578",
+				"meta": {
+					"profile": [
+						"http://hl7.org/fhir/us/core/StructureDefinition/us-core-practitioner",
+						"http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Practitioner"
+					]
+				},
+				"identifier": [
+					{
+						"type": {
+							"coding": [
+								{
+									"system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType",
+									"code": "npi"
+								}
+							]
+						},
+						"system": "http://hl7.org/fhir/sid/us-npi",
+						"value": "1386735827"
+					}
+				],
+				"active": true,
+				"name": [
+					{
+						"family": "NIEVES-ARRIBA",
+						"given": [
+							"NIEVES-ARRIBA, LUCYBETH"
+						]
+					}
+				]
+			},
+			"request": {
+				"method": "PUT",
+				"url": "Practitioner/fb1490ab-c62e-33ca-8414-b6437b0fa578"
+			}
+		},
+		{
+			"fullUrl": "77409d95-a9e3-3868-8706-069ee7074d90",
+			"resource": {
+				"resourceType": "Location",
+				"id": "77409d95-a9e3-3868-8706-069ee7074d90",
+				"name": "LabCorp",
+				"address": {
+					"line": [
+						"231 MAPLE AVENUE",
+						"BURLINGTON"
+					],
+					"city": "NC",
+					"state": "27215"
+				}
+			},
+			"request": {
+				"method": "PUT",
+				"url": "Location/77409d95-a9e3-3868-8706-069ee7074d90"
+			}
+		},
+		{
+			"fullUrl": "6ee4bf07-3948-3093-b0ed-c84d0e511036",
+			"resource": {
+				"resourceType": "Encounter",
+				"id": "6ee4bf07-3948-3093-b0ed-c84d0e511036",
+				"meta": {
+					"profile": [
+						"http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"
+					]
+				},
+				"identifier": [
+					{
+						"system": "http://bluecrossnc.com/fhir/encounterIdentifier",
+						"value": "13028683"
+					}
+				],
+				"status": "finished",
+				"class": {
+					"system": "http://terminology.hl7.org/CodeSystem/v3-ActCode",
+					"code": "AMB"
+				},
+				"type": [
+					{
+						"text": "Authorization"
+					},
+					{
+						"coding": [
+							{
+								"system": "http://www.ama-assn.org/go/cpt",
+								"code": "81163"
+							}
+						]
+					}
+				],
+				"subject": {
+					"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=111111111",
+					"identifier": {
+						"type": {
+							"coding": [
+								{
+									"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
+									"code": "MB"
+								}
+							]
+						},
+						"value": "111111111"
+					}
+				},
+				"participant": [
+					{
+						"type": [
+							{
+								"coding": [
+									{
+										"system": "http://terminology.hl7.org/CodeSystem/v3-ParticipationType",
+										"code": "PART"
+									}
+								]
+							}
+						],
+						"period": {
+							"start": "2021-08-27",
+							"end": "2021-09-26"
+						},
+						"individual": {
+							"reference": "Practitioner/a621019a-fadb-37dd-964a-01c21926e9b4"
+						}
+					},
+					{
+						"type": [
+							{
+								"coding": [
+									{
+										"system": "http://terminology.hl7.org/CodeSystem/v3-ParticipationType",
+										"code": "PART"
+									}
+								]
+							}
+						],
+						"period": {
+							"start": "2019-04-15",
+							"end": "2019-05-15"
+						},
+						"individual": {
+							"reference": "Practitioner/fb1490ab-c62e-33ca-8414-b6437b0fa578"
+						}
+					}
+				],
+				"period": {
+					"start": "2019-04-15",
+					"end": "2019-05-15"
+				},
+				"reasonCode": [
+					{
+						"text": "Diagnostic Medical"
+					}
+				],
+				"location": [
+					{
+						"location": {
+							"reference": "Location/77409d95-a9e3-3868-8706-069ee7074d90"
+						}
+					}
+				]
+			},
+			"request": {
+				"method": "PUT",
+				"url": "Encounter/6ee4bf07-3948-3093-b0ed-c84d0e511036"
+			}
+		},
+		{
+			"fullUrl": "aa00fd16-222f-36e6-ad0d-df8cf9abb325",
+			"resource": {
+				"resourceType": "DocumentReference",
+				"id": "aa00fd16-222f-36e6-ad0d-df8cf9abb325",
+				"meta": {
+					"profile": [
+						"http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"
+					]
+				},
+				"identifier": [
+					{
+						"system": "http://bluecrossnc.com/fhir/documentreferenceIdentifier",
+						"value": "127598"
+					}
+				],
+				"status": "current",
+				"type": {
+					"coding": [
+						{
+							"system": "http://loinc.org",
+							"code": "11506-3"
+						}
+					]
+				},
+				"category": [
+					{
+						"coding": [
+							{
+								"system": "http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category",
+								"code": "clinical-note"
+							}
+						]
+					}
+				],
+				"subject": {
+					"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=10215619000",
+					"identifier": {
+						"type": {
+							"coding": [
+								{
+									"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
+									"code": "MB"
+								}
+							]
+						},
+						"value": "10215619000"
+					}
+				},
+				"date": "2021-09-16T20:00:00.000-04:00",
+				"author": [
+					{
+						"reference": "Organization/BCBSNC"
+					}
+				],
+				"content": [
+					{
+						"attachment": {
+							"contentType": "text/plain",
+							"data": "VkdWemRDQlFhSGx6YVdOcFlXNGdVbUYwYVc5dVlXeGw="
+						}
+					}
+				],
+				"context": {
+					"encounter": [
+						{
+							"display": "13028683"
+						}
+					],
+					"period": {
+						"start": "2021-08-26T20:00:00-04:00",
+						"end": "2021-09-25T20:00:00-04:00"
+					}
+				}
+			},
+			"request": {
+				"method": "PUT",
+				"url": "DocumentReference/aa00fd16-222f-36e6-ad0d-df8cf9abb325"
+			}
+		},
+		{
+			"fullUrl": "89a97fee-729a-3953-9482-caeb82ee8db5",
+			"resource": {
+				"resourceType": "Provenance",
+				"id": "89a97fee-729a-3953-9482-caeb82ee8db5",
+				"meta": {
+					"profile": [
+						"http://hl7.org/fhir/us/core/StructureDefinition/us-core-provenance"
+					]
+				},
+				"target": [
+					{
+						"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=10215619000",
+						"identifier": {
+							"type": {
+								"coding": [
+									{
+										"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
+										"code": "MB"
+									}
+								]
+							},
+							"value": "10215619000"
+						}
+					},
+					{
+						"reference": "Condition/5b6a6591-ae0f-3bfc-987e-5f144a43a036"
+					},
+					{
+						"reference": "DocumentReference/aa00fd16-222f-36e6-ad0d-df8cf9abb325"
+					}
+				],
+				"recorded": "2021-08-27T00:00:00.000-04:00",
+				"agent": [
+					{
+						"type": {
+							"coding": [
+								{
+									"system": "http://terminology.hl7.org/CodeSystem/provenance-participant-type",
+									"code": "author"
+								}
+							]
+						},
+						"who": {
+							"reference": "Organization/BCBSNC"
+						}
+					}
+				]
+			},
+			"request": {
+				"method": "PUT",
+				"url": "Provenance/89a97fee-729a-3953-9482-caeb82ee8db5"
+			}
+		},
+		{
+			"fullUrl": "69d1f8f0-87e6-3cff-bd41-c3e01a5c479d",
+			"resource": {
+				"resourceType": "Provenance",
+				"id": "69d1f8f0-87e6-3cff-bd41-c3e01a5c479d",
+				"meta": {
+					"profile": [
+						"http://hl7.org/fhir/us/core/StructureDefinition/us-core-provenance"
+					]
+				},
+				"target": [
+					{
+						"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=10215619000",
+						"identifier": {
+							"type": {
+								"coding": [
+									{
+										"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
+										"code": "MB"
+									}
+								]
+							},
+							"value": "10215619000"
+						}
+					},
+					{
+						"reference": "Encounter/6ee4bf07-3948-3093-b0ed-c84d0e511036"
+					}
+				],
+				"recorded": "2019-04-15T00:00:00.000-04:00",
+				"agent": [
+					{
+						"type": {
+							"coding": [
+								{
+									"system": "http://terminology.hl7.org/CodeSystem/provenance-participant-type",
+									"code": "author"
+								}
+							]
+						},
+						"who": {
+							"reference": "Organization/BCBSNC"
+						}
+					}
+				]
+			},
+			"request": {
+				"method": "PUT",
+				"url": "Provenance/69d1f8f0-87e6-3cff-bd41-c3e01a5c479d"
+			}
+		}
+	]
+}