Survivorship 6

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md

@@ -318,6 +318,14 @@ This operation takes the following parameters:
                 The id of the Golden Resource to merge data into.
             </td>
         </tr>
+        <tr>
+            <td>resource</td>
+            <td>Resource</td>
+            <td>0..1</td>
+            <td>
+                Optional manually merged Golden Resource. All values except for the metadata, PID and identifiers will be copied from this resource, if it is present. If no value is specified, all fields from the resource pointed to by "fromGoldenResourceId" will be copied instead.
+            </td>
+        </tr>
     </tbody>
 </table>
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/IdHelperService.java

@@ -437,7 +437,6 @@ public class IdHelperService {
 	@Nonnull
 	public Long getPidOrThrowException(IAnyResource theResource) {
 		Long retVal = (Long) theResource.getUserData(RESOURCE_PID);
-		// FIXME NG Is this the intent?
 		if (retVal == null) {
 			throw new IllegalStateException(
 				String.format("Unable to find %s in the user data for %s with ID %s", RESOURCE_PID, theResource, theResource.getId())

hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/GoldenResourceMergerSvcImpl.java

@@ -60,13 +60,16 @@ public class GoldenResourceMergerSvcImpl implements IGoldenResourceMergerSvc {
 
 	@Override
 	@Transactional
-	public IAnyResource mergeGoldenResources(IAnyResource theFromGoldenResource, IAnyResource theToGoldenResource, MdmTransactionContext theMdmTransactionContext) {
+	public IAnyResource mergeGoldenResources(IAnyResource theFromGoldenResource, IAnyResource theMergedResource, IAnyResource theToGoldenResource, MdmTransactionContext theMdmTransactionContext) {
 		Long fromGoldenResourcePid = myIdHelperService.getPidOrThrowException(theFromGoldenResource);
 		Long toGoldenResourcePid = myIdHelperService.getPidOrThrowException(theToGoldenResource);
 		String resourceType = theMdmTransactionContext.getResourceType();
 
-		//Merge attributes, to be determined when survivorship is solved.
+		// Merge attributes, to be determined when survivorship is solved.
-		myGoldenResourceHelper.mergeFields(theFromGoldenResource, theToGoldenResource, theMdmTransactionContext);
+		myGoldenResourceHelper.mergeIndentifierFields(theFromGoldenResource, theToGoldenResource, theMdmTransactionContext);
+
+		IAnyResource mergeSource = ( theMergedResource == null ) ? theFromGoldenResource : theMergedResource;
+		myGoldenResourceHelper.mergeNonIdentiferFields(mergeSource, theToGoldenResource, theMdmTransactionContext);
 
 		//Merge the links from the FROM to the TO resource. Clean up dangling links.
 		mergeGoldenResourceLinks(theFromGoldenResource, theToGoldenResource, toGoldenResourcePid, theMdmTransactionContext);

hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java

@@ -44,6 +44,7 @@ import java.util.stream.Stream;
  */
 @Service
 public class MdmControllerSvcImpl implements IMdmControllerSvc {
+
 	@Autowired
 	MdmControllerHelper myMdmControllerHelper;
 	@Autowired
@@ -54,22 +55,14 @@ public class MdmControllerSvcImpl implements IMdmControllerSvc {
 	IMdmLinkUpdaterSvc myIMdmLinkUpdaterSvc;
 
 	@Override
-	public IAnyResource mergeGoldenResources(String theFromGoldenResourceId, String theToGoldenResourceId, IAnyResource theFromGoldenResource, MdmTransactionContext theMdmTransactionContext) {
+	public IAnyResource mergeGoldenResources(String theFromGoldenResourceId, String theToGoldenResourceId, IAnyResource theManuallyMergedResource, MdmTransactionContext theMdmTransactionContext) {
-		boolean isOverwritingGoldenResource = (theFromGoldenResource != null);
+		IAnyResource fromGoldenResource = myMdmControllerHelper.getLatestGoldenResourceFromIdOrThrowException(ProviderConstants.MDM_MERGE_GR_FROM_GOLDEN_RESOURCE_ID, theFromGoldenResourceId);
-
-		IAnyResource fromGoldenResource;
-		if (theFromGoldenResource == null) {
-			fromGoldenResource = myMdmControllerHelper.getLatestGoldenResourceFromIdOrThrowException(ProviderConstants.MDM_MERGE_GR_FROM_GOLDEN_RESOURCE_ID, theFromGoldenResourceId);
-		} else {
-			fromGoldenResource = theFromGoldenResource;
-		}
-
 		IAnyResource toGoldenResource = myMdmControllerHelper.getLatestGoldenResourceFromIdOrThrowException(ProviderConstants.MDM_MERGE_GR_TO_GOLDEN_RESOURCE_ID, theToGoldenResourceId);
 		myMdmControllerHelper.validateMergeResources(fromGoldenResource, toGoldenResource);
 		myMdmControllerHelper.validateSameVersion(fromGoldenResource, theFromGoldenResourceId);
 		myMdmControllerHelper.validateSameVersion(toGoldenResource, theToGoldenResourceId);
 
-		return myGoldenResourceMergerSvc.mergeGoldenResources(fromGoldenResource, toGoldenResource, theMdmTransactionContext);
+		return myGoldenResourceMergerSvc.mergeGoldenResources(fromGoldenResource, theManuallyMergedResource, toGoldenResource, theMdmTransactionContext);
 	}
 
 	@Override

hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmSurvivorshipSvcImpl.java

@@ -34,30 +34,12 @@ public class MdmSurvivorshipSvcImpl implements IMdmSurvivorshipService {
 	private FhirContext myFhirContext;
 
 	/**
-	 * Survivorship rules may include the following data consolidation methods:
+	 * Merges two golden resources by overwriting all field values on theGoldenResource param for all REST operation methods
+	 * except MERGE_GOLDEN_RESOURCES. In case of MERGE_GOLDEN_RESOURCES, it will attempt to copy field values from
+	 * theTargetResource that do not exist in theGoldenResource. PID, indentifiers and meta values are not affected by
+	 * this operation.
 	 *
-	 * <ul>
+	 * @param theTargetResource        Target resource to retrieve fields from
-	 *  <li>
-	 *  Length of field - apply the field value containing most or least number of characters - e.g. longest name
-	 *  </li>
-	 *  <li>
-	 *  Date time - all the field value from the oldest or the newest recrod - e.g. use the most recent phone number
-	 *  </li>
-	 *  <li>
-	 *  Frequency - use the most or least frequent number of occurrence - e.g. most common phone number
-	 *  </li>
-	 *  <li>
-	 *  Integer - number functions (largest, sum, avg) - e.g. number of patient encounters
-	 *  </li>
-	 *  <li>
-	 *  Quality of data - best quality data - e.g. data coming from a certain system is considered trusted and overrides all other values
-	 *  </li>
-	 *  <li>
-	 *  A hybrid approach combining all methods listed above as best fits
-	 *  </li>
-	 * </ul>
-	 *
-	 * @param theTargetResource        Target resource to merge fields from
 	 * @param theGoldenResource        Golden resource to merge fields into
 	 * @param theMdmTransactionContext Current transaction context
 	 * @param <T>
@@ -66,14 +48,10 @@ public class MdmSurvivorshipSvcImpl implements IMdmSurvivorshipService {
 	public <T extends IBase> void applySurvivorshipRulesToGoldenResource(T theTargetResource, T theGoldenResource, MdmTransactionContext theMdmTransactionContext) {
 		switch (theMdmTransactionContext.getRestOperation()) {
 			case MERGE_GOLDEN_RESOURCES:
-				if (theMdmTransactionContext.isForceResourceUpdate()) {
-					TerserUtil.overwriteFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource, TerserUtil.EXCLUDE_IDS_AND_META);
-					break;
-				}
 				TerserUtil.mergeFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource, TerserUtil.EXCLUDE_IDS_AND_META);
 				break;
 			default:
-				TerserUtil.overwriteFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource, TerserUtil.EXCLUDE_IDS_AND_META);
+				TerserUtil.replaceFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource, TerserUtil.EXCLUDE_IDS_AND_META);
 				break;
 		}
 	}

hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmGoldenResourceMergerSvcTest.java

@@ -101,11 +101,13 @@ public class MdmGoldenResourceMergerSvcTest extends BaseMdmR4Test {
 
 	private Patient mergeGoldenPatients() {
 		assertEquals(0, redirectLinkCount());
-		Patient retval = (Patient) myGoldenResourceMergerSvc.mergeGoldenResources(myFromGoldenPatient, myToGoldenPatient, createMdmContext());
+		Patient retval = (Patient) myGoldenResourceMergerSvc.mergeGoldenResources(myFromGoldenPatient, null, myToGoldenPatient, createMdmContext());
 		assertEquals(1, redirectLinkCount());
 		return retval;
 	}
 
+	// TODO NG - add a test with a manually merged golden patient
+
 	private int redirectLinkCount() {
 		MdmLink mdmLink = new MdmLink().setMatchResult(MdmMatchResultEnum.REDIRECT);
 		Example<MdmLink> example = Example.of(mdmLink);

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/NormalizedQuantitySearchLevel.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.model.entity;
 
+/*-
+ * #%L
+ * HAPI FHIR Model
+ * %%
+ * Copyright (C) 2014 - 2021 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 /**
  * Support different UCUM services level for FHIR Quantity data type.
  * 

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamQuantity.java

@@ -197,4 +197,4 @@ public class ResourceIndexedSearchParamQuantity extends ResourceIndexedSearchPar
 		return retval;
 	}
 
-}
+}

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IGoldenResourceMergerSvc.java

@@ -29,8 +29,9 @@ public interface IGoldenResourceMergerSvc {
 	 * Merge all Golden Resource fields subject to survivorship rules.
 	 *
 	 * @param theFromGoldenResource the golden resource we are merging from
+	 * @param theManuallyMergedResource an optional golden resource that was manually merged
 	 * @param theToGoldenResource the golden resource we are merging to
 	 * @return updated theToGoldenResource with the merged fields and links.
 	 */
-	IAnyResource mergeGoldenResources(IAnyResource theFromGoldenResource, IAnyResource theToGoldenResource, MdmTransactionContext theMdmTransactionContext);
+	IAnyResource mergeGoldenResources(IAnyResource theFromGoldenResource, IAnyResource theManuallyMergedResource, IAnyResource theToGoldenResource, MdmTransactionContext theMdmTransactionContext);
 }

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmSurvivorshipService.java

@@ -29,7 +29,30 @@ import org.hl7.fhir.instance.model.api.IBase;
 public interface IMdmSurvivorshipService {
 
 	/**
-	 * Applies survivorship rules to merge fields from the specified target resource to the golden resource
+	 * Applies survivorship rules to merge fields from the specified target resource to the golden resource. Survivorship
+	 * rules may include, but not limited to the following data consolidation methods:
+	 *
+	 * <ul>
+	 *  <li>
+	 *  Length of field - apply the field value containing most or least number of characters - e.g. longest name
+	 *  </li>
+	 *  <li>
+	 *  Date time - all the field value from the oldest or the newest recrod - e.g. use the most recent phone number
+	 *  </li>
+	 *  <li>
+	 *  Frequency - use the most or least frequent number of occurrence - e.g. most common phone number
+	 *  </li>
+	 *  <li>
+	 *  Integer - number functions (largest, sum, avg) - e.g. number of patient encounters
+	 *  </li>
+	 *  <li>
+	 *  Quality of data - best quality data - e.g. data coming from a certain system is considered trusted and overrides
+	 *  all other values
+	 *  </li>
+	 *  <li>
+	 *  A hybrid approach combining all methods listed above as best fits
+	 *  </li>
+	 * </ul>
 	 *
 	 * @param theTargetResource        Target resource to merge fields from
 	 * @param theGoldenResource        Golden resource to merge fields into

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/model/MdmTransactionContext.java

@@ -32,7 +32,8 @@ public class MdmTransactionContext {
 		UPDATE_LINK,
 		DUPLICATE_GOLDEN_RESOURCES,
 		NOT_DUPLICATE,
-		MERGE_GOLDEN_RESOURCES
+		MERGE_GOLDEN_RESOURCES,
+		MANUAL_MERGE_GOLDEN_RESOURCES
 	}
 
 	/**
@@ -44,8 +45,6 @@ public class MdmTransactionContext {
 
 	private String myResourceType;
 
-	private boolean myForceResourceUpdate;
-
 	public TransactionLogMessages getTransactionLogMessages() {
 		return myTransactionLogMessages;
 	}
@@ -89,12 +88,4 @@ public class MdmTransactionContext {
 	public void setResourceType(String myResourceType) {
 		this.myResourceType = myResourceType;
 	}
-
-	public boolean isForceResourceUpdate() {
-		return myForceResourceUpdate;
-	}
-
-	public void setForceResourceUpdate(boolean theForceResourceUpdate) {
-		myForceResourceUpdate = theForceResourceUpdate;
-	}
 }

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/BaseMdmProvider.java

@@ -55,6 +55,8 @@ public abstract class BaseMdmProvider {
 		if (theFromGoldenResource == null) {
 			return;
 		}
+
+		// FIXME NG - ID is proveded as a param anyways - can always retrived it there
 		validateNotNull(ProviderConstants.MDM_MERGE_RESOURCE_ID, theFromGoldenResource.getIdElement());
 		if (theFromGoldenResource.getIdElement().getValue().equals(theToGoldenResourceId.getValue())) {
 			throw new InvalidRequestException("resource must be different from the one with toGoldenResourceId");

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java

@@ -149,9 +149,10 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider {
 		validateMergeParameters(theFromGoldenResourceId, theToGoldenResourceId);
 		validateOptionalMergeResource(theMergedResource, theToGoldenResourceId);
 
-		MdmTransactionContext txContext = createMdmContext(theRequestDetails, MdmTransactionContext.OperationType.MERGE_GOLDEN_RESOURCES,
+		MdmTransactionContext.OperationType operationType = (theMergedResource == null) ?
+			MdmTransactionContext.OperationType.MERGE_GOLDEN_RESOURCES : MdmTransactionContext.OperationType.MANUAL_MERGE_GOLDEN_RESOURCES;
+		MdmTransactionContext txContext = createMdmContext(theRequestDetails, operationType,
 			getResourceType(ProviderConstants.MDM_MERGE_GR_FROM_GOLDEN_RESOURCE_ID, theFromGoldenResourceId));
-		txContext.setForceResourceUpdate(theMergedResource != null);
 		return myMdmControllerSvc.mergeGoldenResources(theFromGoldenResourceId.getValueAsString(), theToGoldenResourceId.getValueAsString(), theMergedResource, txContext);
 	}
 

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/GoldenResourceHelper.java

@@ -241,8 +241,11 @@ public class GoldenResourceHelper {
 		}
 	}
 
-	public void mergeFields(IBaseResource theFromGoldenResource, IBaseResource theToGoldenResource, MdmTransactionContext theMdmTransactionContext) {
+	public void mergeIndentifierFields(IBaseResource theFromGoldenResource, IBaseResource theToGoldenResource, MdmTransactionContext theMdmTransactionContext) {
 		TerserUtil.cloneCompositeField(myFhirContext, theFromGoldenResource, theToGoldenResource, FIELD_NAME_IDENTIFIER);
+	}
+
+	public void mergeNonIdentiferFields(IBaseResource theFromGoldenResource, IBaseResource theToGoldenResource, MdmTransactionContext theMdmTransactionContext) {
 		myMdmSurvivorshipService.applySurvivorshipRulesToGoldenResource(theFromGoldenResource, theToGoldenResource, theMdmTransactionContext);
 	}
 

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/TerserUtil.java

@@ -42,7 +42,7 @@ import static ca.uhn.fhir.mdm.util.GoldenResourceHelper.FIELD_NAME_IDENTIFIER;
 public final class TerserUtil {
 
 	public static final Collection<String> IDS_AND_META_EXCLUDES =
-		Collections.unmodifiableSet(Stream.of("id", "identifier", "meta", "active").collect(Collectors.toSet()));
+		Collections.unmodifiableSet(Stream.of("id", "identifier", "meta").collect(Collectors.toSet()));
 
 	public static final Predicate<String> EXCLUDE_IDS_AND_META = new Predicate<String>() {
 		@Override
@@ -151,7 +151,7 @@ public final class TerserUtil {
 		mergeFields(theFhirContext, theFrom, theTo, INCLUDE_ALL);
 	}
 
-	public static void overwriteFields(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo, Predicate<String> inclusionStrategy) {
+	public static void replaceFields(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo, Predicate<String> inclusionStrategy) {
 		FhirTerser terser = theFhirContext.newTerser();
 
 		RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
@@ -160,13 +160,25 @@ public final class TerserUtil {
 				continue;
 			}
 
-			childDefinition.getAccessor().getFirstValueOrNull(theFrom).ifPresent( v -> {
+			replaceField(theFrom, theTo, childDefinition);
-					childDefinition.getMutator().setValue(theTo, v);
-				}
-			);
 		}
 	}
 
+	public static void replaceField(FhirContext theFhirContext, FhirTerser theTerser, String theFieldName, IBaseResource theFrom, IBaseResource theTo) {
+		FhirTerser terser = theFhirContext.newTerser();
+
+		RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
+		BaseRuntimeChildDefinition childDefinition = definition.getChildByName(theFieldName);
+		replaceField(theFrom, theTo, childDefinition);
+	}
+
+	private static void replaceField(IBaseResource theFrom, IBaseResource theTo, BaseRuntimeChildDefinition childDefinition) {
+		childDefinition.getAccessor().getFirstValueOrNull(theFrom).ifPresent(v -> {
+				childDefinition.getMutator().setValue(theTo, v);
+			}
+		);
+	}
+
 	public static void mergeFieldsExceptIdAndMeta(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo) {
 		mergeFields(theFhirContext, theFrom, theTo, EXCLUDE_IDS_AND_META);
 	}
@@ -182,21 +194,45 @@ public final class TerserUtil {
 
 			List<IBase> theFromFieldValues = childDefinition.getAccessor().getValues(theFrom);
 			List<IBase> theToFieldValues = childDefinition.getAccessor().getValues(theTo);
-			
-			for (IBase theFromFieldValue : theFromFieldValues) {
-				if (contains(theFromFieldValue, theToFieldValues)) {
-					continue;
-				}
 
-				IBase newFieldValue = childDefinition.getChildByName(childDefinition.getElementName()).newInstance();
+			mergeFields(terser, theTo, childDefinition, theFromFieldValues, theToFieldValues);
-				terser.cloneInto(theFromFieldValue, newFieldValue, true);
+		}
+	}
 
-				try {
+	/**
-					theToFieldValues.add(newFieldValue);
+	 * Merges value of the specified field from theFrom resource to theTo resource. Fields values are compared via
-				} catch (UnsupportedOperationException e) {
+	 * the equalsDeep method, or via object identity if this method is not available.
-					childDefinition.getMutator().setValue(theTo, newFieldValue);
+	 *
-					break;
+	 * @param theFhirContext
-				}
+	 * @param theTerser
+	 * @param theFieldName
+	 * @param theFrom
+	 * @param theTo
+	 */
+	public static void mergeField(FhirContext theFhirContext, FhirTerser theTerser, String theFieldName, IBaseResource theFrom, IBaseResource theTo) {
+		RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
+		BaseRuntimeChildDefinition childDefinition = definition.getChildByName(theFieldName);
+
+		List<IBase> theFromFieldValues = childDefinition.getAccessor().getValues(theFrom);
+		List<IBase> theToFieldValues = childDefinition.getAccessor().getValues(theTo);
+
+		mergeFields(theTerser, theTo, childDefinition, theFromFieldValues, theToFieldValues);
+	}
+
+	private static void mergeFields(FhirTerser theTerser, IBaseResource theTo, BaseRuntimeChildDefinition childDefinition, List<IBase> theFromFieldValues, List<IBase> theToFieldValues) {
+		for (IBase theFromFieldValue : theFromFieldValues) {
+			if (contains(theFromFieldValue, theToFieldValues)) {
+				continue;
+			}
+
+			IBase newFieldValue = childDefinition.getChildByName(childDefinition.getElementName()).newInstance();
+			theTerser.cloneInto(theFromFieldValue, newFieldValue, true);
+
+			try {
+				theToFieldValues.add(newFieldValue);
+			} catch (UnsupportedOperationException e) {
+				childDefinition.getMutator().setValue(theTo, newFieldValue);
+				break;
 			}
 		}
 	}