Improve parallel model scanning (#3046)

* Improve parallel model scanning

* Add changelog

* Resolve fixme

* Test fix
This commit is contained in:
James Agnew 2021-10-03 19:40:24 -04:00 committed by GitHub
parent 53a2d3c1ac
commit 95e853bdf7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 679 additions and 159 deletions

View File

@ -1,57 +1,5 @@
package ca.uhn.fhir.context;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
/*
* #%L
* HAPI FHIR - Core Library
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBase;
import org.hl7.fhir.instance.model.api.IBaseBackboneElement;
import org.hl7.fhir.instance.model.api.IBaseDatatype;
import org.hl7.fhir.instance.model.api.IBaseDatatypeElement;
import org.hl7.fhir.instance.model.api.IBaseEnumeration;
import org.hl7.fhir.instance.model.api.IBaseExtension;
import org.hl7.fhir.instance.model.api.IBaseReference;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.ICompositeType;
import org.hl7.fhir.instance.model.api.INarrative;
import org.hl7.fhir.instance.model.api.IPrimitiveType;
import ca.uhn.fhir.model.api.IBoundCodeableConcept;
import ca.uhn.fhir.model.api.IDatatype;
import ca.uhn.fhir.model.api.IElement;
@ -69,18 +17,47 @@ import ca.uhn.fhir.model.base.composite.BaseResourceReferenceDt;
import ca.uhn.fhir.model.primitive.BoundCodeDt;
import ca.uhn.fhir.parser.DataFormatException;
import ca.uhn.fhir.util.ReflectionUtil;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBase;
import org.hl7.fhir.instance.model.api.IBaseBackboneElement;
import org.hl7.fhir.instance.model.api.IBaseDatatype;
import org.hl7.fhir.instance.model.api.IBaseDatatypeElement;
import org.hl7.fhir.instance.model.api.IBaseEnumeration;
import org.hl7.fhir.instance.model.api.IBaseExtension;
import org.hl7.fhir.instance.model.api.IBaseReference;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.ICompositeType;
import org.hl7.fhir.instance.model.api.INarrative;
import org.hl7.fhir.instance.model.api.IPrimitiveType;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> extends BaseRuntimeElementDefinition<T> {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseRuntimeElementCompositeDefinition.class);
private final FhirContext myContext;
private Map<String, Integer> forcedOrder = null;
private List<BaseRuntimeChildDefinition> myChildren = new ArrayList<>();
private List<BaseRuntimeChildDefinition> myChildrenAndExtensions;
private Map<Class<? extends IBase>, BaseRuntimeElementDefinition<?>> myClassToElementDefinitions;
private final FhirContext myContext;
private Map<String, BaseRuntimeChildDefinition> myNameToChild = new HashMap<>();
private List<ScannedField> myScannedFields = new ArrayList<>();
private volatile boolean mySealed;
private volatile SealingStateEnum mySealed = SealingStateEnum.NOT_SEALED;
@SuppressWarnings("unchecked")
public BaseRuntimeElementCompositeDefinition(String theName, Class<? extends T> theImplementingClass, boolean theStandardType, FhirContext theContext, Map<Class<? extends IBase>, BaseRuntimeElementDefinition<?>> theClassToElementDefinitions) {
@ -174,7 +151,7 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
* Has this class been sealed
*/
public boolean isSealed() {
return mySealed;
return mySealed == SealingStateEnum.SEALED;
}
@SuppressWarnings("unchecked")
@ -459,12 +436,18 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
elementNames.add(elementName);
}
}
@Override
void sealAndInitialize(FhirContext theContext, Map<Class<? extends IBase>, BaseRuntimeElementDefinition<?>> theClassToElementDefinitions) {
if (mySealed) {
if (mySealed == SealingStateEnum.SEALED) {
return;
}
mySealed = true;
synchronized (myContext) {
if (mySealed == SealingStateEnum.SEALED || mySealed == SealingStateEnum.SEALING) {
return;
}
mySealed = SealingStateEnum.SEALING;
scanCompositeElementForChildren();
@ -531,29 +514,26 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
}
myChildrenAndExtensions = Collections.unmodifiableList(children);
mySealed = SealingStateEnum.SEALED;
}
}
@Override
protected void validateSealed() {
if (!mySealed) {
if (mySealed != SealingStateEnum.SEALED) {
synchronized (myContext) {
if(!mySealed) {
if (mySealed == SealingStateEnum.NOT_SEALED) {
sealAndInitialize(myContext, myClassToElementDefinitions);
}
}
}
}
private static int findIndex(List<BaseRuntimeChildDefinition> theChildren, String theName, boolean theDefaultAtEnd) {
int index = theDefaultAtEnd ? theChildren.size() : -1;
for (ListIterator<BaseRuntimeChildDefinition> iter = theChildren.listIterator(); iter.hasNext(); ) {
if (iter.next().getElementName().equals(theName)) {
index = iter.previousIndex();
break;
}
}
return index;
private enum SealingStateEnum {
NOT_SEALED,
SEALING,
SEALED
}
private static class ScannedField {
@ -563,6 +543,7 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
private Class<?> myElementType;
private Field myField;
private boolean myFirstFieldInNewClass;
ScannedField(Field theField, Class<?> theClass, boolean theFirstFieldInNewClass) {
myField = theField;
myFirstFieldInNewClass = theFirstFieldInNewClass;
@ -609,4 +590,15 @@ public abstract class BaseRuntimeElementCompositeDefinition<T extends IBase> ext
}
}
private static int findIndex(List<BaseRuntimeChildDefinition> theChildren, String theName, boolean theDefaultAtEnd) {
int index = theDefaultAtEnd ? theChildren.size() : -1;
for (ListIterator<BaseRuntimeChildDefinition> iter = theChildren.listIterator(); iter.hasNext(); ) {
if (iter.next().getElementName().equals(theName)) {
index = iter.previousIndex();
break;
}
}
return index;
}
}

View File

@ -224,9 +224,8 @@ public abstract class BaseRuntimeElementDefinition<T extends IBase> {
/*
* this does nothing, but BaseRuntimeElementCompositeDefinition
* overrides this method to provide functionality because that class
* defers the dealing process
* defers the sealing process
*/
}
public BaseRuntimeChildDefinition getChildByName(String theChildName) {

View File

@ -988,7 +988,8 @@ public class FhirTerser {
case RESOURCE_BLOCK:
case COMPOSITE_DATATYPE: {
BaseRuntimeElementCompositeDefinition<?> childDef = (BaseRuntimeElementCompositeDefinition<?>) def;
for (BaseRuntimeChildDefinition nextChild : childDef.getChildrenAndExtension()) {
List<BaseRuntimeChildDefinition> childrenAndExtensionDefs = childDef.getChildrenAndExtension();
for (BaseRuntimeChildDefinition nextChild : childrenAndExtensionDefs) {
List<?> values = nextChild.getAccessor().getValues(theElement);

View File

@ -0,0 +1,4 @@
---
type: fix
issue: 3046
title: When using deferred model scanning in highly parallelized environments, a crash could sometimes occur during parse/serialize operations.

View File

@ -58,6 +58,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.either;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@ -228,7 +230,7 @@ public class InterceptorDstu3Test {
httpPost.setEntity(new StringEntity(input, ContentType.create(Constants.CT_FHIR_JSON, "UTF-8")));
HttpResponse status = ourClient.execute(httpPost);
try {
assertEquals(201, status.getStatusLine().getStatusCode());
assertThat(status.getStatusLine().getStatusCode(), either(equalTo(200)).or(equalTo(201)));
} finally {
IOUtils.closeQuietly(status.getEntity().getContent());
}

View File

@ -2,6 +2,7 @@ package ca.uhn.fhir.parser;
import ca.uhn.fhir.context.ConfigurationException;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.PerformanceOptionsEnum;
import ca.uhn.fhir.model.api.annotation.DatatypeDef;
import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.test.BaseTest;
@ -43,9 +44,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.MatcherAssert.assertThat;
@ -270,6 +277,50 @@ public class JsonParserR4Test extends BaseTest {
}
/**
* Make sure we can perform parallel parse/encodes against the same
* FhirContext instance when deferred model scanning is enabled without
* running into threading issues.
*/
@Test
public void testEncodeAndDecodeMultithreadedWithDeferredModelScanning() throws IOException, ExecutionException, InterruptedException {
String input = loadResource("/multi-thread-parsing-issue-bundle.json");
ExecutorService executor = Executors.newFixedThreadPool(10);
for (int pass = 0; pass < 10; pass++) {
ourLog.info("Starting pass: {}", pass);
FhirContext parseCtx = FhirContext.forR4();
parseCtx.setPerformanceOptions(PerformanceOptionsEnum.DEFERRED_MODEL_SCANNING);
List<Future<Bundle>> bundleFutures = new ArrayList<>();
for (int readIdx = 0; readIdx < 10; readIdx++) {
bundleFutures.add(executor.submit(()->parseCtx.newJsonParser().parseResource(Bundle.class, input)));
}
List<Bundle> parsedBundles = new ArrayList<>();
for (Future<Bundle> nextFuture : bundleFutures) {
Bundle nextBundle = nextFuture.get();
parsedBundles.add(nextBundle);
}
FhirContext encodeCtx = FhirContext.forR4();
encodeCtx.setPerformanceOptions(PerformanceOptionsEnum.DEFERRED_MODEL_SCANNING);
List<Future<String>> encodeFutures = new ArrayList<>();
for (Bundle nextBundle : parsedBundles) {
encodeFutures.add(executor.submit(()->encodeCtx.newJsonParser().encodeResourceToString(nextBundle)));
}
List<String> outputs = new ArrayList<>();
for (Future<String> nextFuture : encodeFutures) {
outputs.add(nextFuture.get());
}
assertEquals(outputs.get(0), outputs.get(1));
}
}
@Test
public void testEncodeAndParseUnicodeCharacterInNarrative() {
Patient p = new Patient();

View File

@ -0,0 +1,471 @@
{
"resourceType": "Bundle",
"type": "transaction",
"entry": [
{
"fullUrl": "5b6a6591-ae0f-3bfc-987e-5f144a43a036",
"resource": {
"resourceType": "Condition",
"id": "5b6a6591-ae0f-3bfc-987e-5f144a43a036",
"meta": {
"profile": [
"http://hl7.org/fhir/us/core/StructureDefinition/us-core-condition"
]
},
"identifier": [
{
"system": "http://bluecrossnc.com/fhir/conditionIdentifier",
"value": "13028683"
}
],
"clinicalStatus": {
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/condition-clinical",
"code": "active"
}
]
},
"verificationStatus": {
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/condition-ver-status",
"code": "unconfirmed"
}
]
},
"category": [
{
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/condition-category",
"code": "encounter-diagnosis"
}
]
}
],
"code": {
"coding": [
{
"system": "http://hl7.org/fhir/sid/icd-10-cm",
"code": "Z80.3"
}
]
},
"subject": {
"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=111111111",
"identifier": {
"type": {
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
"code": "MB"
}
]
},
"value": "111111111"
}
}
},
"request": {
"method": "PUT",
"url": "Condition/5b6a6591-ae0f-3bfc-987e-5f144a43a036"
}
},
{
"fullUrl": "a621019a-fadb-37dd-964a-01c21926e9b4",
"resource": {
"resourceType": "Practitioner",
"id": "a621019a-fadb-37dd-964a-01c21926e9b4",
"meta": {
"profile": [
"http://hl7.org/fhir/us/core/StructureDefinition/us-core-practitioner",
"http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Practitioner"
]
},
"identifier": [
{
"type": {
"coding": [
{
"system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType",
"code": "npi"
}
]
},
"system": "http://hl7.org/fhir/sid/us-npi",
"value": "1134567787"
}
],
"active": true,
"name": [
{
"family": "BRIMMAGE",
"given": [
"BRAIN BRIMMAGE"
]
}
]
},
"request": {
"method": "PUT",
"url": "Practitioner/a621019a-fadb-37dd-964a-01c21926e9b4"
}
},
{
"fullUrl": "fb1490ab-c62e-33ca-8414-b6437b0fa578",
"resource": {
"resourceType": "Practitioner",
"id": "fb1490ab-c62e-33ca-8414-b6437b0fa578",
"meta": {
"profile": [
"http://hl7.org/fhir/us/core/StructureDefinition/us-core-practitioner",
"http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Practitioner"
]
},
"identifier": [
{
"type": {
"coding": [
{
"system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType",
"code": "npi"
}
]
},
"system": "http://hl7.org/fhir/sid/us-npi",
"value": "1386735827"
}
],
"active": true,
"name": [
{
"family": "NIEVES-ARRIBA",
"given": [
"NIEVES-ARRIBA, LUCYBETH"
]
}
]
},
"request": {
"method": "PUT",
"url": "Practitioner/fb1490ab-c62e-33ca-8414-b6437b0fa578"
}
},
{
"fullUrl": "77409d95-a9e3-3868-8706-069ee7074d90",
"resource": {
"resourceType": "Location",
"id": "77409d95-a9e3-3868-8706-069ee7074d90",
"name": "LabCorp",
"address": {
"line": [
"231 MAPLE AVENUE",
"BURLINGTON"
],
"city": "NC",
"state": "27215"
}
},
"request": {
"method": "PUT",
"url": "Location/77409d95-a9e3-3868-8706-069ee7074d90"
}
},
{
"fullUrl": "6ee4bf07-3948-3093-b0ed-c84d0e511036",
"resource": {
"resourceType": "Encounter",
"id": "6ee4bf07-3948-3093-b0ed-c84d0e511036",
"meta": {
"profile": [
"http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"
]
},
"identifier": [
{
"system": "http://bluecrossnc.com/fhir/encounterIdentifier",
"value": "13028683"
}
],
"status": "finished",
"class": {
"system": "http://terminology.hl7.org/CodeSystem/v3-ActCode",
"code": "AMB"
},
"type": [
{
"text": "Authorization"
},
{
"coding": [
{
"system": "http://www.ama-assn.org/go/cpt",
"code": "81163"
}
]
}
],
"subject": {
"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=111111111",
"identifier": {
"type": {
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
"code": "MB"
}
]
},
"value": "111111111"
}
},
"participant": [
{
"type": [
{
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/v3-ParticipationType",
"code": "PART"
}
]
}
],
"period": {
"start": "2021-08-27",
"end": "2021-09-26"
},
"individual": {
"reference": "Practitioner/a621019a-fadb-37dd-964a-01c21926e9b4"
}
},
{
"type": [
{
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/v3-ParticipationType",
"code": "PART"
}
]
}
],
"period": {
"start": "2019-04-15",
"end": "2019-05-15"
},
"individual": {
"reference": "Practitioner/fb1490ab-c62e-33ca-8414-b6437b0fa578"
}
}
],
"period": {
"start": "2019-04-15",
"end": "2019-05-15"
},
"reasonCode": [
{
"text": "Diagnostic Medical"
}
],
"location": [
{
"location": {
"reference": "Location/77409d95-a9e3-3868-8706-069ee7074d90"
}
}
]
},
"request": {
"method": "PUT",
"url": "Encounter/6ee4bf07-3948-3093-b0ed-c84d0e511036"
}
},
{
"fullUrl": "aa00fd16-222f-36e6-ad0d-df8cf9abb325",
"resource": {
"resourceType": "DocumentReference",
"id": "aa00fd16-222f-36e6-ad0d-df8cf9abb325",
"meta": {
"profile": [
"http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"
]
},
"identifier": [
{
"system": "http://bluecrossnc.com/fhir/documentreferenceIdentifier",
"value": "127598"
}
],
"status": "current",
"type": {
"coding": [
{
"system": "http://loinc.org",
"code": "11506-3"
}
]
},
"category": [
{
"coding": [
{
"system": "http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category",
"code": "clinical-note"
}
]
}
],
"subject": {
"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=10215619000",
"identifier": {
"type": {
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
"code": "MB"
}
]
},
"value": "10215619000"
}
},
"date": "2021-09-16T20:00:00.000-04:00",
"author": [
{
"reference": "Organization/BCBSNC"
}
],
"content": [
{
"attachment": {
"contentType": "text/plain",
"data": "VkdWemRDQlFhSGx6YVdOcFlXNGdVbUYwYVc5dVlXeGw="
}
}
],
"context": {
"encounter": [
{
"display": "13028683"
}
],
"period": {
"start": "2021-08-26T20:00:00-04:00",
"end": "2021-09-25T20:00:00-04:00"
}
}
},
"request": {
"method": "PUT",
"url": "DocumentReference/aa00fd16-222f-36e6-ad0d-df8cf9abb325"
}
},
{
"fullUrl": "89a97fee-729a-3953-9482-caeb82ee8db5",
"resource": {
"resourceType": "Provenance",
"id": "89a97fee-729a-3953-9482-caeb82ee8db5",
"meta": {
"profile": [
"http://hl7.org/fhir/us/core/StructureDefinition/us-core-provenance"
]
},
"target": [
{
"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=10215619000",
"identifier": {
"type": {
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
"code": "MB"
}
]
},
"value": "10215619000"
}
},
{
"reference": "Condition/5b6a6591-ae0f-3bfc-987e-5f144a43a036"
},
{
"reference": "DocumentReference/aa00fd16-222f-36e6-ad0d-df8cf9abb325"
}
],
"recorded": "2021-08-27T00:00:00.000-04:00",
"agent": [
{
"type": {
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/provenance-participant-type",
"code": "author"
}
]
},
"who": {
"reference": "Organization/BCBSNC"
}
}
]
},
"request": {
"method": "PUT",
"url": "Provenance/89a97fee-729a-3953-9482-caeb82ee8db5"
}
},
{
"fullUrl": "69d1f8f0-87e6-3cff-bd41-c3e01a5c479d",
"resource": {
"resourceType": "Provenance",
"id": "69d1f8f0-87e6-3cff-bd41-c3e01a5c479d",
"meta": {
"profile": [
"http://hl7.org/fhir/us/core/StructureDefinition/us-core-provenance"
]
},
"target": [
{
"reference": "Patient?_type=http://terminology.hl7.org/CodeSystem/v2-0203|MB&identifier=10215619000",
"identifier": {
"type": {
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
"code": "MB"
}
]
},
"value": "10215619000"
}
},
{
"reference": "Encounter/6ee4bf07-3948-3093-b0ed-c84d0e511036"
}
],
"recorded": "2019-04-15T00:00:00.000-04:00",
"agent": [
{
"type": {
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/provenance-participant-type",
"code": "author"
}
]
},
"who": {
"reference": "Organization/BCBSNC"
}
}
]
},
"request": {
"method": "PUT",
"url": "Provenance/69d1f8f0-87e6-3cff-bd41-c3e01a5c479d"
}
}
]
}