merge master

NOTICE.txt

@@ -10,3 +10,4 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+

hapi-fhir-base/src/main/java/ca/uhn/fhir/context/support/IValidationSupport.java

@@ -1071,8 +1071,9 @@ public interface IValidationSupport {
 			}
 		}
 
-		public void setErrorMessage(String theErrorMessage) {
+		public LookupCodeResult setErrorMessage(String theErrorMessage) {
 			myErrorMessage = theErrorMessage;
+			return this;
 		}
 
 		public String getErrorMessage() {

hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java

@@ -2515,6 +2515,22 @@ public enum Pointcut implements IPointcut {
 	MDM_SUBMIT(
 			void.class, "ca.uhn.fhir.rest.api.server.RequestDetails", "ca.uhn.fhir.mdm.model.mdmevents.MdmSubmitEvent"),
 
+	/**
+	 * <b>MDM_SUBMIT_PRE_MESSAGE_DELIVERY Hook:</b>
+	 * Invoked immediately before the delivery of a MESSAGE to the broker.
+	 * <p>
+	 * Hooks can make changes to the delivery payload.
+	 * Furthermore, modification can be made to the outgoing message,
+	 * for example adding headers or changing message key,
+	 * which will be used for the subsequent processing.
+	 * </p>
+	 * Hooks should accept the following parameters:
+	 * <ul>
+	 * <li>ca.uhn.fhir.jpa.subscription.model.ResourceModifiedJsonMessage</li>
+	 * </ul>
+	 */
+	MDM_SUBMIT_PRE_MESSAGE_DELIVERY(void.class, "ca.uhn.fhir.jpa.subscription.model.ResourceModifiedJsonMessage"),
+
 	/**
 	 * <b>JPA Hook:</b>
 	 * This hook is invoked when a cross-partition reference is about to be

hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/model/RequestPartitionId.java

@@ -40,6 +40,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
 
@@ -98,6 +99,33 @@ public class RequestPartitionId implements IModelJson {
 		myAllPartitions = true;
 	}
 
+	/**
+	 * Creates a new RequestPartitionId which includes all partition IDs from
+	 * this {@link RequestPartitionId} but also includes all IDs from the given
+	 * {@link RequestPartitionId}. Any duplicates are only included once, and
+	 * partition names and dates are ignored and not returned. This {@link RequestPartitionId}
+	 * and {@literal theOther} are not modified.
+	 *
+	 * @since 7.4.0
+	 */
+	public RequestPartitionId mergeIds(RequestPartitionId theOther) {
+		if (isAllPartitions() || theOther.isAllPartitions()) {
+			return RequestPartitionId.allPartitions();
+		}
+
+		// don't know why this is required - otherwise PartitionedStrictTransactionR4Test fails
+		if (this.equals(theOther)) {
+			return this;
+		}
+
+		List<Integer> thisPartitionIds = getPartitionIds();
+		List<Integer> otherPartitionIds = theOther.getPartitionIds();
+		List<Integer> newPartitionIds = Stream.concat(thisPartitionIds.stream(), otherPartitionIds.stream())
+				.distinct()
+				.collect(Collectors.toList());
+		return RequestPartitionId.fromPartitionIds(newPartitionIds);
+	}
+
 	public static RequestPartitionId fromJson(String theJson) throws JsonProcessingException {
 		return ourObjectMapper.readValue(theJson, RequestPartitionId.class);
 	}
@@ -332,6 +360,14 @@ public class RequestPartitionId implements IModelJson {
 		return new RequestPartitionId(thePartitionNames, thePartitionIds, thePartitionDate);
 	}
 
+	public static boolean isDefaultPartition(@Nullable RequestPartitionId thePartitionId) {
+		if (thePartitionId == null) {
+			return false;
+		}
+
+		return thePartitionId.isDefaultPartition();
+	}
+
 	/**
 	 * Create a string representation suitable for use as a cache key. Null aware.
 	 * <p>

hapi-fhir-base/src/main/java/ca/uhn/fhir/repository/Repository.java

@@ -1,3 +1,22 @@
+/*-
+ * #%L
+ * HAPI FHIR - Core Library
+ * %%
+ * Copyright (C) 2014 - 2024 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
 package ca.uhn.fhir.repository;
 
 import ca.uhn.fhir.context.FhirContext;

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/client/api/IRestfulClientFactory.java

@@ -37,6 +37,11 @@ public interface IRestfulClientFactory {
 	 */
 	public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT = 10000;
 
+	/**
+	 * Default value for {@link #getConnectionTimeToLive()}
+	 */
+	public static final int DEFAULT_CONNECTION_TTL = 5000;
+
 	/**
 	 * Default value for {@link #getServerValidationModeEnum()}
 	 */
@@ -75,6 +80,16 @@ public interface IRestfulClientFactory {
 	 */
 	int getConnectTimeout();
 
+	/**
+	 * Gets the connection time to live, in milliseconds. This is the amount of time to keep connections alive for reuse.
+	 * <p>
+	 * The default value for this setting is defined by {@link #DEFAULT_CONNECTION_TTL}
+	 * </p>
+	 */
+	default int getConnectionTimeToLive() {
+		return DEFAULT_CONNECTION_TTL;
+	}
+
 	/**
 	 * Returns the HTTP client instance. This method will not return null.
 	 * @param theUrl
@@ -179,6 +194,14 @@ public interface IRestfulClientFactory {
 	 */
 	void setConnectTimeout(int theConnectTimeout);
 
+	/**
+	 * Sets the connection time to live, in milliseconds. This is the amount of time to keep connections alive for reuse.
+	 * <p>
+	 * The default value for this setting is defined by {@link #DEFAULT_CONNECTION_TTL}
+	 * </p>
+	 */
+	default void setConnectionTimeToLive(int theConnectionTimeToLive) {}
+
 	/**
 	 * Sets the Apache HTTP client instance to be used by any new restful clients created by this factory. If set to
 	 * <code>null</code>, a new HTTP client with default settings will be created.

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/FhirTerser.java

@@ -101,7 +101,7 @@ public class FhirTerser {
 		return newList;
 	}
 
-	private ExtensionDt createEmptyExtensionDt(IBaseExtension theBaseExtension, String theUrl) {
+	private ExtensionDt createEmptyExtensionDt(IBaseExtension<?, ?> theBaseExtension, String theUrl) {
 		return createEmptyExtensionDt(theBaseExtension, false, theUrl);
 	}
 
@@ -122,13 +122,13 @@ public class FhirTerser {
 		return theSupportsUndeclaredExtensions.addUndeclaredExtension(theIsModifier, theUrl);
 	}
 
-	private IBaseExtension createEmptyExtension(IBaseHasExtensions theBaseHasExtensions, String theUrl) {
-		return (IBaseExtension) theBaseHasExtensions.addExtension().setUrl(theUrl);
+	private IBaseExtension<?, ?> createEmptyExtension(IBaseHasExtensions theBaseHasExtensions, String theUrl) {
+		return (IBaseExtension<?, ?>) theBaseHasExtensions.addExtension().setUrl(theUrl);
 	}
 
-	private IBaseExtension createEmptyModifierExtension(
+	private IBaseExtension<?, ?> createEmptyModifierExtension(
 			IBaseHasModifierExtensions theBaseHasModifierExtensions, String theUrl) {
-		return (IBaseExtension)
+		return (IBaseExtension<?, ?>)
 				theBaseHasModifierExtensions.addModifierExtension().setUrl(theUrl);
 	}
 
@@ -407,7 +407,7 @@ public class FhirTerser {
 
 	public String getSinglePrimitiveValueOrNull(IBase theTarget, String thePath) {
 		return getSingleValue(theTarget, thePath, IPrimitiveType.class)
-				.map(t -> t.getValueAsString())
+				.map(IPrimitiveType::getValueAsString)
 				.orElse(null);
 	}
 
@@ -487,7 +487,7 @@ public class FhirTerser {
 			} else {
 				// DSTU3+
 				final String extensionUrlForLambda = extensionUrl;
-				List<IBaseExtension> extensions = Collections.emptyList();
+				List<IBaseExtension<?, ?>> extensions = Collections.emptyList();
 				if (theCurrentObj instanceof IBaseHasExtensions) {
 					extensions = ((IBaseHasExtensions) theCurrentObj)
 							.getExtension().stream()
@@ -505,7 +505,7 @@ public class FhirTerser {
 					}
 				}
 
-				for (IBaseExtension next : extensions) {
+				for (IBaseExtension<?, ?> next : extensions) {
 					if (theWantedClass.isAssignableFrom(next.getClass())) {
 						retVal.add((T) next);
 					}
@@ -581,7 +581,7 @@ public class FhirTerser {
 			} else {
 				// DSTU3+
 				final String extensionUrlForLambda = extensionUrl;
-				List<IBaseExtension> extensions = Collections.emptyList();
+				List<IBaseExtension<?, ?>> extensions = Collections.emptyList();
 
 				if (theCurrentObj instanceof IBaseHasModifierExtensions) {
 					extensions = ((IBaseHasModifierExtensions) theCurrentObj)
@@ -602,7 +602,7 @@ public class FhirTerser {
 					}
 				}
 
-				for (IBaseExtension next : extensions) {
+				for (IBaseExtension<?, ?> next : extensions) {
 					if (theWantedClass.isAssignableFrom(next.getClass())) {
 						retVal.add((T) next);
 					}
@@ -1203,7 +1203,6 @@ public class FhirTerser {
 	public void visit(IBase theElement, IModelVisitor2 theVisitor) {
 		BaseRuntimeElementDefinition<?> def = myContext.getElementDefinition(theElement.getClass());
 		if (def instanceof BaseRuntimeElementCompositeDefinition) {
-			BaseRuntimeElementCompositeDefinition<?> defComposite = (BaseRuntimeElementCompositeDefinition<?>) def;
 			visit(theElement, null, def, theVisitor, new ArrayList<>(), new ArrayList<>(), new ArrayList<>());
 		} else if (theElement instanceof IBaseExtension) {
 			theVisitor.acceptUndeclaredExtension(
@@ -1562,7 +1561,7 @@ public class FhirTerser {
 				throw new DataFormatException(Msg.code(1796) + "Invalid path " + thePath + ": Element of type "
 						+ def.getName() + " has no child named " + nextPart + ". Valid names: "
 						+ def.getChildrenAndExtension().stream()
-								.map(t -> t.getElementName())
+								.map(BaseRuntimeChildDefinition::getElementName)
 								.sorted()
 								.collect(Collectors.joining(", ")));
 			}
@@ -1817,7 +1816,18 @@ public class FhirTerser {
 			if (getResourceToIdMap() == null) {
 				return null;
 			}
-			return getResourceToIdMap().get(theNext);
+
+			var idFromMap = getResourceToIdMap().get(theNext);
+			if (idFromMap != null) {
+				return idFromMap;
+			} else if (theNext.getIdElement().getIdPart() != null) {
+				return getResourceToIdMap().values().stream()
+						.filter(id -> theNext.getIdElement().getIdPart().equals(id.getIdPart()))
+						.findAny()
+						.orElse(null);
+			} else {
+				return null;
+			}
 		}
 
 		private List<IBaseResource> getOrCreateResourceList() {

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/SubscriptionUtil.java

@@ -65,7 +65,7 @@ public class SubscriptionUtil {
 		populatePrimitiveValue(theContext, theSubscription, "status", theStatus);
 	}
 
-	public static boolean isCrossPartition(IBaseResource theSubscription) {
+	public static boolean isDefinedAsCrossPartitionSubcription(IBaseResource theSubscription) {
 		if (theSubscription instanceof IBaseHasExtensions) {
 			IBaseExtension extension = ExtensionUtil.getExtensionByUrl(
 					theSubscription, HapiExtensions.EXTENSION_SUBSCRIPTION_CROSS_PARTITION);

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/TaskChunker.java

@@ -20,10 +20,12 @@ package ca.uhn.fhir.util;
  * #L%
  */
 
+import com.google.common.collect.Streams;
 import jakarta.annotation.Nonnull;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 import java.util.function.Consumer;
 import java.util.stream.Stream;
@@ -57,4 +59,9 @@ public class TaskChunker<T> {
 	public <T> Stream<List<T>> chunk(Stream<T> theStream, int theChunkSize) {
 		return StreamUtil.partition(theStream, theChunkSize);
 	}
+
+	@Nonnull
+	public void chunk(Iterator<T> theIterator, int theChunkSize, Consumer<List<T>> theListConsumer) {
+		chunk(Streams.stream(theIterator), theChunkSize).forEach(theListConsumer);
+	}
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/VersionEnum.java

@@ -151,9 +151,13 @@ public enum VersionEnum {
 	V7_0_0,
 	V7_0_1,
 	V7_0_2,
+	V7_0_3,
 
 	V7_1_0,
 	V7_2_0,
+	V7_2_1,
+	V7_2_2,
+	V7_2_3,
 
 	V7_3_0,
 	V7_4_0,

hapi-fhir-base/src/main/java/org/hl7/fhir/instance/model/api/IAnyResource.java

@@ -20,29 +20,115 @@
 package org.hl7.fhir.instance.model.api;
 
 import ca.uhn.fhir.model.api.annotation.SearchParamDefinition;
+import ca.uhn.fhir.rest.gclient.DateClientParam;
 import ca.uhn.fhir.rest.gclient.TokenClientParam;
+import ca.uhn.fhir.rest.gclient.UriClientParam;
 
 /**
  * An IBaseResource that has a FHIR version of DSTU3 or higher
  */
 public interface IAnyResource extends IBaseResource {
 
+	String SP_RES_ID = "_id";
 	/**
 	 * Search parameter constant for <b>_id</b>
 	 */
-	@SearchParamDefinition(name = "_id", path = "", description = "The ID of the resource", type = "token")
-	String SP_RES_ID = "_id";
+	@SearchParamDefinition(
+			name = SP_RES_ID,
+			path = "Resource.id",
+			description = "The ID of the resource",
+			type = "token")
 
 	/**
 	 * <b>Fluent Client</b> search parameter constant for <b>_id</b>
 	 * <p>
 	 * Description: <b>the _id of a resource</b><br>
 	 * Type: <b>string</b><br>
-	 * Path: <b>Resource._id</b><br>
+	 * Path: <b>Resource.id</b><br>
 	 * </p>
 	 */
 	TokenClientParam RES_ID = new TokenClientParam(IAnyResource.SP_RES_ID);
 
+	String SP_RES_LAST_UPDATED = "_lastUpdated";
+	/**
+	 * Search parameter constant for <b>_lastUpdated</b>
+	 */
+	@SearchParamDefinition(
+			name = SP_RES_LAST_UPDATED,
+			path = "Resource.meta.lastUpdated",
+			description = "Only return resources which were last updated as specified by the given range",
+			type = "date")
+
+	/**
+	 * <b>Fluent Client</b> search parameter constant for <b>_lastUpdated</b>
+	 * <p>
+	 * Description: <b>The last updated date of a resource</b><br>
+	 * Type: <b>date</b><br>
+	 * Path: <b>Resource.meta.lastUpdated</b><br>
+	 * </p>
+	 */
+	DateClientParam RES_LAST_UPDATED = new DateClientParam(IAnyResource.SP_RES_LAST_UPDATED);
+
+	String SP_RES_TAG = "_tag";
+	/**
+	 * Search parameter constant for <b>_tag</b>
+	 */
+	@SearchParamDefinition(
+			name = SP_RES_TAG,
+			path = "Resource.meta.tag",
+			description = "The tag of the resource",
+			type = "token")
+
+	/**
+	 * <b>Fluent Client</b> search parameter constant for <b>_tag</b>
+	 * <p>
+	 * Description: <b>The tag of a resource</b><br>
+	 * Type: <b>token</b><br>
+	 * Path: <b>Resource.meta.tag</b><br>
+	 * </p>
+	 */
+	TokenClientParam RES_TAG = new TokenClientParam(IAnyResource.SP_RES_TAG);
+
+	String SP_RES_PROFILE = "_profile";
+	/**
+	 * Search parameter constant for <b>_profile</b>
+	 */
+	@SearchParamDefinition(
+			name = SP_RES_PROFILE,
+			path = "Resource.meta.profile",
+			description = "The profile of the resource",
+			type = "uri")
+
+	/**
+	 * <b>Fluent Client</b> search parameter constant for <b>_profile</b>
+	 * <p>
+	 * Description: <b>The profile of a resource</b><br>
+	 * Type: <b>uri</b><br>
+	 * Path: <b>Resource.meta.profile</b><br>
+	 * </p>
+	 */
+	UriClientParam RES_PROFILE = new UriClientParam(IAnyResource.SP_RES_PROFILE);
+
+	String SP_RES_SECURITY = "_security";
+	/**
+	 * Search parameter constant for <b>_security</b>
+	 */
+	@SearchParamDefinition(
+			name = SP_RES_SECURITY,
+			path = "Resource.meta.security",
+			description = "The security of the resource",
+			type = "token")
+
+	/**
+	 * <b>Fluent Client</b> search parameter constant for <b>_security</b>
+	 * <p>
+	 * Description: <b>The security of a resource</b><br>
+	 * Type: <b>token</b><br>
+	 * Path: <b>Resource.meta.security</b><br>
+	 * </p>
+	 */
+	TokenClientParam RES_SECURITY = new TokenClientParam(IAnyResource.SP_RES_SECURITY);
+
 	String getId();
 
 	IIdType getIdElement();

hapi-fhir-base/src/main/java/org/hl7/fhir/instance/model/api/IIdType.java

@@ -120,6 +120,13 @@ public interface IIdType extends IPrimitiveType<String> {
 	 */
 	boolean isVersionIdPartValidLong();
 
+	/**
+	 * @return true if the id begins with "urn:uuid:"
+	 */
+	default boolean isUuid() {
+		return getValue() != null && getValue().startsWith("urn:uuid:");
+	}
+
 	@Override
 	IIdType setValue(String theString);
 

hapi-fhir-base/src/main/resources/ca/uhn/fhir/i18n/hapi-messages.properties

@@ -6,6 +6,9 @@ org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService.
 org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService.mismatchCodeSystem=Inappropriate CodeSystem URL "{0}" for ValueSet: {1}
 org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService.codeNotFoundInValueSet=Code "{0}" is not in valueset: {1}
 
+org.hl7.fhir.common.hapi.validation.support.RemoteTerminologyServiceValidationSupport.unknownCodeInSystem=Unknown code "{0}#{1}". The Remote Terminology server {2} returned {3}
+org.hl7.fhir.common.hapi.validation.support.RemoteTerminologyServiceValidationSupport.unknownCodeInValueSet=Unknown code "{0}#{1}" for ValueSet with URL "{2}". The Remote Terminology server {3} returned {4}
+
 ca.uhn.fhir.jpa.term.TermReadSvcImpl.expansionRefersToUnknownCs=Unknown CodeSystem URI "{0}" referenced from ValueSet
 ca.uhn.fhir.jpa.term.TermReadSvcImpl.valueSetNotYetExpanded=ValueSet "{0}" has not yet been pre-expanded. Performing in-memory expansion without parameters. Current status: {1} | {2}
 ca.uhn.fhir.jpa.term.TermReadSvcImpl.valueSetNotYetExpanded_OffsetNotAllowed=ValueSet expansion can not combine "offset" with "ValueSet.compose.exclude" unless the ValueSet has been pre-expanded. ValueSet "{0}" must be pre-expanded for this operation to work.
@@ -91,6 +94,7 @@ ca.uhn.fhir.jpa.dao.BaseStorageDao.inlineMatchNotSupported=Inline match URLs are
 ca.uhn.fhir.jpa.dao.BaseStorageDao.transactionOperationWithMultipleMatchFailure=Failed to {0} resource with match URL "{1}" because this search matched {2} resources
 ca.uhn.fhir.jpa.dao.BaseStorageDao.deleteByUrlThresholdExceeded=Failed to DELETE resources with match URL "{0}" because the resolved number of resources: {1} exceeds the threshold of {2}
 ca.uhn.fhir.jpa.dao.BaseStorageDao.transactionOperationWithIdNotMatchFailure=Failed to {0} resource with match URL "{1}" because the matching resource does not match the provided ID
+ca.uhn.fhir.jpa.dao.BaseTransactionProcessor.multiplePartitionAccesses=Can not process transaction with {0} entries: Entries require access to multiple/conflicting partitions
 ca.uhn.fhir.jpa.dao.BaseHapiFhirDao.transactionOperationFailedNoId=Failed to {0} resource in transaction because no ID was provided
 ca.uhn.fhir.jpa.dao.BaseHapiFhirDao.transactionOperationFailedUnknownId=Failed to {0} resource in transaction because no resource could be found with ID {1}
 ca.uhn.fhir.jpa.dao.BaseHapiFhirDao.uniqueIndexConflictFailure=Can not create resource of type {0} as it would create a duplicate unique index matching query: {1} (existing index belongs to {2}, new unique index created by {3})

hapi-fhir-base/src/test/java/ca/uhn/fhir/interceptor/model/RequestPartitionIdTest.java

@@ -41,6 +41,50 @@ public class RequestPartitionIdTest {
 		assertFalse(RequestPartitionId.forPartitionIdsAndNames(null, Lists.newArrayList(1, 2), null).isDefaultPartition());
 	}
 
+	@Test
+	public void testMergeIds() {
+		RequestPartitionId input0 = RequestPartitionId.fromPartitionIds(1, 2, 3);
+		RequestPartitionId input1 = RequestPartitionId.fromPartitionIds(1, 2, 4);
+
+		RequestPartitionId actual = input0.mergeIds(input1);
+		RequestPartitionId expected = RequestPartitionId.fromPartitionIds(1, 2, 3, 4);
+		assertEquals(expected, actual);
+
+	}
+
+	@Test
+	public void testMergeIds_ThisAllPartitions() {
+		RequestPartitionId input0 = RequestPartitionId.allPartitions();
+		RequestPartitionId input1 = RequestPartitionId.fromPartitionIds(1, 2, 4);
+
+		RequestPartitionId actual = input0.mergeIds(input1);
+		RequestPartitionId expected = RequestPartitionId.allPartitions();
+		assertEquals(expected, actual);
+
+	}
+
+	@Test
+	public void testMergeIds_OtherAllPartitions() {
+		RequestPartitionId input0 = RequestPartitionId.fromPartitionIds(1, 2, 3);
+		RequestPartitionId input1 = RequestPartitionId.allPartitions();
+
+		RequestPartitionId actual = input0.mergeIds(input1);
+		RequestPartitionId expected = RequestPartitionId.allPartitions();
+		assertEquals(expected, actual);
+
+	}
+
+	@Test
+	public void testMergeIds_IncludesDefault() {
+		RequestPartitionId input0 = RequestPartitionId.fromPartitionIds(1, 2, 3);
+		RequestPartitionId input1 = RequestPartitionId.defaultPartition();
+
+		RequestPartitionId actual = input0.mergeIds(input1);
+		RequestPartitionId expected = RequestPartitionId.fromPartitionIds(1, 2, 3, null);
+		assertEquals(expected, actual);
+
+	}
+
 	@Test
 	public void testSerDeserSer() throws JsonProcessingException {
 		{

hapi-fhir-base/src/test/java/ca/uhn/fhir/util/TaskChunkerTest.java

@@ -3,14 +3,21 @@ package ca.uhn.fhir.util;
 import jakarta.annotation.Nonnull;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Captor;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.function.Consumer;
 import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.times;
@@ -43,8 +50,32 @@ public class TaskChunkerTest {
 
 	@Nonnull
 	private static List<Integer> newIntRangeList(int startInclusive, int endExclusive) {
-		List<Integer> input = IntStream.range(startInclusive, endExclusive).boxed().toList();
-		return input;
+		return IntStream.range(startInclusive, endExclusive).boxed().toList();
+	}
+
+	@ParameterizedTest
+	@MethodSource("testIteratorChunkArguments")
+	void testIteratorChunk(List<Integer> theListToChunk, List<List<Integer>> theExpectedChunks) {
+	    // given
+		Iterator<Integer> iter = theListToChunk.iterator();
+		ArrayList<List<Integer>> result = new ArrayList<>();
+
+	    // when
+		new TaskChunker<Integer>().chunk(iter, 3, result::add);
+
+	    // then
+	    assertEquals(theExpectedChunks, result);
+	}
+
+	public static Stream<Arguments> testIteratorChunkArguments() {
+		return Stream.of(
+			Arguments.of(Collections.emptyList(), Collections.emptyList()),
+			Arguments.of(List.of(1), List.of(List.of(1))),
+			Arguments.of(List.of(1,2), List.of(List.of(1,2))),
+			Arguments.of(List.of(1,2,3), List.of(List.of(1,2,3))),
+			Arguments.of(List.of(1,2,3,4), List.of(List.of(1,2,3), List.of(4))),
+			Arguments.of(List.of(1,2,3,4,5,6,7,8,9), List.of(List.of(1,2,3), List.of(4,5,6), List.of(7,8,9)))
+		);
 	}
 
 }

hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/client/HapiFhirCliRestfulClientFactory.java

@@ -105,10 +105,11 @@ public class HapiFhirCliRestfulClientFactory extends RestfulClientFactory {
 				Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
 						.register("https", sslConnectionSocketFactory)
 						.build();
-				connectionManager =
-						new PoolingHttpClientConnectionManager(registry, null, null, null, 5000, TimeUnit.MILLISECONDS);
+				connectionManager = new PoolingHttpClientConnectionManager(
+						registry, null, null, null, getConnectionTimeToLive(), TimeUnit.MILLISECONDS);
 			} else {
-				connectionManager = new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
+				connectionManager =
+						new PoolingHttpClientConnectionManager(getConnectionTimeToLive(), TimeUnit.MILLISECONDS);
 			}
 
 			connectionManager.setMaxTotal(getPoolMaxTotal());

hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/ApacheRestfulClientFactoryTest.java

@@ -3,6 +3,7 @@ package ca.uhn.fhir.cli;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.FhirVersionEnum;
 import ca.uhn.fhir.rest.client.apache.ApacheRestfulClientFactory;
+import ca.uhn.fhir.rest.client.api.IRestfulClientFactory;
 import ca.uhn.fhir.test.BaseFhirVersionParameterizedTest;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
@@ -10,6 +11,7 @@ import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.util.EntityUtils;
 import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 
@@ -79,4 +81,13 @@ public class ApacheRestfulClientFactoryTest extends BaseFhirVersionParameterized
 			assertEquals(SSLHandshakeException.class, e.getCause().getCause().getClass());
 		}
 	}
+
+	@Test
+	public void testConnectionTimeToLive() {
+		ApacheRestfulClientFactory clientFactory = new ApacheRestfulClientFactory();
+
+		assertEquals(IRestfulClientFactory.DEFAULT_CONNECTION_TTL, clientFactory.getConnectionTimeToLive());
+		clientFactory.setConnectionTimeToLive(25000);
+		assertEquals(25000, clientFactory.getConnectionTimeToLive());
+	}
 }

hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/client/HapiFhirCliRestfulClientFactoryTest.java

@@ -3,6 +3,7 @@ package ca.uhn.fhir.cli.client;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.FhirVersionEnum;
 import ca.uhn.fhir.i18n.Msg;
+import ca.uhn.fhir.rest.client.api.IRestfulClientFactory;
 import ca.uhn.fhir.test.BaseFhirVersionParameterizedTest;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
@@ -159,4 +160,15 @@ public class HapiFhirCliRestfulClientFactoryTest extends BaseFhirVersionParamete
 		}
 	}
 
+	@ParameterizedTest
+	@MethodSource("baseParamsProvider")
+	public void testConnectionTimeToLive(FhirVersionEnum theFhirVersion) {
+		FhirVersionParams fhirVersionParams = getFhirVersionParams(theFhirVersion);
+		HapiFhirCliRestfulClientFactory clientFactory = new HapiFhirCliRestfulClientFactory(fhirVersionParams.getFhirContext());
+
+		assertEquals(IRestfulClientFactory.DEFAULT_CONNECTION_TTL, clientFactory.getConnectionTimeToLive());
+		clientFactory.setConnectionTimeToLive(25000);
+		assertEquals(25000, clientFactory.getConnectionTimeToLive());
+	}
+
 }

hapi-fhir-client-okhttp/src/main/java/ca/uhn/fhir/okhttp/client/OkHttpRestfulClientFactory.java

@@ -25,6 +25,7 @@ import ca.uhn.fhir.rest.client.api.Header;
 import ca.uhn.fhir.rest.client.api.IHttpClient;
 import ca.uhn.fhir.rest.client.impl.RestfulClientFactory;
 import okhttp3.Call;
+import okhttp3.ConnectionPool;
 import okhttp3.OkHttpClient;
 
 import java.net.InetSocketAddress;
@@ -65,6 +66,7 @@ public class OkHttpRestfulClientFactory extends RestfulClientFactory {
 			myNativeClient = new OkHttpClient()
 					.newBuilder()
 					.connectTimeout(getConnectTimeout(), TimeUnit.MILLISECONDS)
+					.connectionPool(new ConnectionPool(5, getConnectionTimeToLive(), TimeUnit.MILLISECONDS))
 					.readTimeout(getSocketTimeout(), TimeUnit.MILLISECONDS)
 					.writeTimeout(getSocketTimeout(), TimeUnit.MILLISECONDS)
 					.build();

hapi-fhir-client-okhttp/src/test/java/ca/uhn/fhir/okhttp/OkHttpRestfulClientFactoryTest.java

@@ -3,6 +3,7 @@ package ca.uhn.fhir.okhttp;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.FhirVersionEnum;
 import ca.uhn.fhir.okhttp.client.OkHttpRestfulClientFactory;
+import ca.uhn.fhir.rest.client.api.IRestfulClientFactory;
 import ca.uhn.fhir.test.BaseFhirVersionParameterizedTest;
 import okhttp3.Call;
 import okhttp3.OkHttpClient;
@@ -71,6 +72,13 @@ public class OkHttpRestfulClientFactoryTest extends BaseFhirVersionParameterized
 		assertEquals(1516, ((OkHttpClient) clientFactory.getNativeClient()).connectTimeoutMillis());
 	}
 
+	@Test
+	public void testConnectionTimeToLive() {
+		assertEquals(IRestfulClientFactory.DEFAULT_CONNECTION_TTL, clientFactory.getConnectionTimeToLive());
+		clientFactory.setConnectionTimeToLive(25000);
+		assertEquals(25000, clientFactory.getConnectionTimeToLive());
+	}
+
 	@ParameterizedTest
 	@MethodSource("baseParamsProvider")
 	public void testNativeClientHttp(FhirVersionEnum theFhirVersion) throws Exception {

hapi-fhir-client/src/main/java/ca/uhn/fhir/rest/client/apache/ApacheRestfulClientFactory.java

@@ -103,7 +103,7 @@ public class ApacheRestfulClientFactory extends RestfulClientFactory {
 					.disableCookieManagement();
 
 			PoolingHttpClientConnectionManager connectionManager =
-					new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
+					new PoolingHttpClientConnectionManager(getConnectionTimeToLive(), TimeUnit.MILLISECONDS);
 			connectionManager.setMaxTotal(getPoolMaxTotal());
 			connectionManager.setDefaultMaxPerRoute(getPoolMaxPerRoute());
 			builder.setConnectionManager(connectionManager);

hapi-fhir-client/src/main/java/ca/uhn/fhir/rest/client/impl/RestfulClientFactory.java

@@ -57,6 +57,7 @@ public abstract class RestfulClientFactory implements IRestfulClientFactory {
 	private final Set<String> myValidatedServerBaseUrls = Collections.synchronizedSet(new HashSet<>());
 	private int myConnectionRequestTimeout = DEFAULT_CONNECTION_REQUEST_TIMEOUT;
 	private int myConnectTimeout = DEFAULT_CONNECT_TIMEOUT;
+	private int myConnectionTimeToLive = DEFAULT_CONNECTION_TTL;
 	private FhirContext myContext;
 	private final Map<Class<? extends IRestfulClient>, ClientInvocationHandlerFactory> myInvocationHandlers =
 			new HashMap<>();
@@ -91,6 +92,11 @@ public abstract class RestfulClientFactory implements IRestfulClientFactory {
 		return myConnectTimeout;
 	}
 
+	@Override
+	public synchronized int getConnectionTimeToLive() {
+		return myConnectionTimeToLive;
+	}
+
 	/**
 	 * Return the proxy username to authenticate with the HTTP proxy
 	 */
@@ -210,6 +216,12 @@ public abstract class RestfulClientFactory implements IRestfulClientFactory {
 		resetHttpClient();
 	}
 
+	@Override
+	public synchronized void setConnectionTimeToLive(int theConnectionTimeToLive) {
+		myConnectionTimeToLive = theConnectionTimeToLive;
+		resetHttpClient();
+	}
+
 	/**
 	 * Sets the context associated with this client factory. Must not be called more than once.
 	 */

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_0_2/version.yaml

@@ -0,0 +1,3 @@
+---
+release-date: "2024-03-20"
+codename: "Zed"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_0_3/version.yaml

@@ -0,0 +1,3 @@
+---
+release-date: "2024-08-24"
+codename: "Zed"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_1/version.yaml

@@ -0,0 +1,3 @@
+---
+release-date: "2024-05-30"
+codename: "Borealis"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_2/version.yaml

@@ -0,0 +1,3 @@
+---
+release-date: "2024-07-19"
+codename: "Borealis"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_3/version.yaml

@@ -0,0 +1,3 @@
+---
+release-date: "2024-08-25"
+codename: "Borealis"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/4837-fix-stringify-parse-copy-contained-duplicate-bug.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 4837
+title: "In the case where a resource was serialized, deserialized, copied and reserialized it resulted in duplication of
+ contained resources. This has been corrected."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/5960-lucene-chain-sort-silently-not-working.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 5960
+backport: 7.2.1
+title: "Previously, queries with chained would fail to sort correctly with lucene and full text searches enabled.
+        This has been fixed."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6024-full-index-search-with-filter-parameters.yaml

@@ -1,6 +1,7 @@
 ---
 type: fix
 issue: 6024
+backport: 7.2.2
 title: "Fixed a bug in search where requesting a count with HSearch indexing
         and FilterParameter enabled and using the _filter parameter would result
         in inaccurate results being returned.

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6044-fix-issue-with-cache-refresh-infinite-loop.yaml

@@ -1,6 +1,7 @@
 ---
 type: fix
 issue: 6044
+backport: 7.2.2
 title: "Fixed an issue where doing a cache refresh with advanced Hibernate Search
         enabled would result in an infinite loop of cache refresh -> search for
         StructureDefinition -> cache refresh, etc

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-failed-text-and-content-search-in-r5.yaml

@@ -1,4 +1,5 @@
 ---
 type: fix
 issue: 6046
+backport: 7.2.2
 title: "Previously, using `_text` and `_content` searches in Hibernate Search in R5 was not supported. This issue has been fixed."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-text-contains.yaml

@@ -1,5 +1,6 @@
 ---
 type: add
 issue: 6046
+backport: 7.2.2
 title: "Added support for `:contains` parameter qualifier on the `_text` and `_content` Search Parameters. When using Hibernate Search, this will cause 
 the search to perform an substring match on the provided value. Documentation can be found [here](/hapi-fhir/docs/server_jpa/elastic.html#performing-fulltext-search-in-luceneelasticsearch)."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6070-add-offset-support-for-everything-operation-synchronous-search.yaml

@@ -1,5 +1,6 @@
 ---
 type: add
+backport: 7.2.3
 issue: 6070
 jira: SMILE-8503
 title: "Added paging support for `$everything` operation in synchronous search mode."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6099-support-online-migration-azure-sql-server.yaml

@@ -1,4 +1,5 @@
 ---
 type: perf
 issue: 6099
+backport: 7.0.3,7.2.2
 title: "Database migrations that add or drop an index no longer lock tables when running on Azure Sql Server."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6122-validate-operation-with-remote-terminology-service-enabled-returned-bad-request.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 6122
+title: "Previously, executing the '$validate' operation on a resource instance could result in an HTTP 400 Bad Request 
+instead of an HTTP 200 OK response with a list of validation issues. This has been fixed."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6123-search-parameters-missing-in-ianyresource.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 6123
+title: "`IAnyResource` `_id` search parameter was missing `path` property value, which resulted in extractor not 
+    working when standard search parameters were instantiated from defined context. This has been fixed, and also 
+    `_LastUpdated`, `_tag`, `_profile`, and `_security` parameter definitions were added to the class."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6133-make-everything-search-not-rely-on-hibernate-search.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 6083
+backport: 7.2.2
+title: "A bug with $everything operation was discovered when trying to search using hibernate search, this change makes
+all $everything operation rely on database search until hibernate search fully supports the operation."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6134-filesystem-regression.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 6134
+backport: 7.2.2
+title: "Fixed a regression in 7.2.0 which caused systems using `FILESYSTEM` binary storage mode to be unable to read metadata documents
+that had been previously stored on disk."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6148-provide-target-resource-partition-info-in-resourcelink.yaml

@@ -0,0 +1,5 @@
+---
+type: add
+issue: 6148
+jira: SMILE-8613
+title: "Added the target resource partitionId and partitionDate to the resourceLink table."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6150-validate-returns-404-valueset-with-remote-terminology-systems.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 6150
+title: "Previously, the resource $validate operation would return a 404 when the associated profile uses a ValueSet 
+that has multiple includes referencing Remote Terminology CodeSystem resources. 
+This has been fixed to return a 200 with issues instead."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6153-allow-transactional-update-with-conditional-url.yaml

@@ -0,0 +1,11 @@
+---
+type: fix
+issue: 6153
+title: "Previously, if you created a resource with some conditional url,
+        but then submitted a transaction bundle that
+        a) updated the resource to not match the condition anymore and
+        b) create a resource with the (same) condition
+        a unique index violation would result.
+        
+        This has been fixed.
+"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6156-updated-migration-on-idx-idxcmbtqknu-hashc-to-be-concurrent.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 6156
+title: "Index IDX_IDXCMBTOKNU_HASHC on table HFJ_IDX_CMB_TOK_NU's migration
+        is now marked as online (concurrent).
+"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6159-code-system-delta-add-or-remove-fail-without-concept-display.yaml

@@ -0,0 +1,8 @@
+---
+type: fix
+issue: 6159
+jira: SMILE-8604
+title: "Previously, `$apply-codesystem-delta-add` and `$apply-codesystem-delta-remove` operations were failing 
+with a 500 Server Error when invoked with a CodeSystem Resource payload that had a concept without a  
+`display` element. This has now been fixed so that concepts without display field is accepted, as `display` 
+element is not required."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6163-make-transactions-better-partition-aware.yaml

@@ -0,0 +1,7 @@
+---
+type: fix
+jira: SMILE-8652
+title: "When JPA servers are configured to always require a new database
+  transaction when switching partitions, the server will now correctly
+  identify the correct partition for FHIR transaction operations, and
+  fail the operation if multiple partitions would be required."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6179-reindex-job-no-longer-runs-without-urls.yaml

@@ -0,0 +1,7 @@
+
+---
+type: change
+issue: 6179
+title: "The $reindex operation could potentially initiate a reindex job without any urls provided in the parameters.
+We now internally generate a list of urls out of all the supported resource types and attempt to reindex 
+found resources of each type separately. As a result, each reindex (batch2) job chunk will be always associated with a url."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6179-reindex-with-custom-resource-type-based-partitioning-interceptor.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 6179
+title: "Previously, the $reindex operation would fail when using a custom partitioning interceptor which decides the partition 
+based on the resource type in the request. This has been fixed, such that we avoid retrieving the resource type from 
+the request, rather we use the urls provided as parameters to the operation to determine the partitions."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6188-non-cross-partition-subs-match-all-resources.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 6188
+jira: SMILE-8759
+title: "Previously, a Subscription not marked as a cross-partition subscription could listen to incoming resources from
+other partitions.  This issue is fixed."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6208-fix-npe-on-date.yaml

@@ -0,0 +1,4 @@
+---
+type: fix
+issue: 6208
+title: "A regression was temporarily introduced which caused searches by `_lastUpdated` to fail with a NullPointerException when using Lucene as the backing search engine. This has been corrected"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6182-new-pointcut-for-mdm-submit.yaml

@@ -0,0 +1,4 @@
+---
+type: add
+issue: 6182
+title: "A new Pointcut called `MDM_SUBMIT_PRE_MESSAGE_DELIVERY` has been added. If you wish to customize the `ResourceModifiedJsonMessage` sent to the broker, you can do so by implementing this Pointcut, and returning `ResourceModifiedJsonMessage`."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6184-add-configuration-setting-for-ttl-of-http-connections.yaml

@@ -0,0 +1,10 @@
+---
+type: add
+issue: 6184
+title: "Added a configuration setting for the TTL of HTTP connections to IRestfulClientFactory.
+        The following implementations have been updated to respect this new setting:
+        1. ApacheRestfulClientFactory
+        2. OkHttpRestfulClientFactory
+        3. HapiFhirCliRestfulClientFactory
+        Thanks to Alex Kopp and Alex Cote for the contribution!
+"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6203-subscription-validator-interceptor-allows-invalid-urls.yml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 6203
+title: "Previously, the SubscriptionValidatingInterceptor would allow the creation/update of a REST hook subscription 
+where the endpoint URL property is not prefixed with http[s]. This issue is fixed."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6206-close-cursors-during-migration.yaml

@@ -0,0 +1,4 @@
+---
+type: fix
+issue: 6206
+title: "A resource leak during database migration on Oracle could cause a failure `ORA-01000 maximum open cursors for session`. This has been corrected. Thanks to Jonas Beyer for the contribution!"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6216-fulltext-searching-not-returning-expected-results.yaml

@@ -0,0 +1,9 @@
+---
+type: fix
+backport: 7.2.3
+issue: 6216
+jira: SMILE-8806
+title: "Previously, searches combining the `_text` query parameter (using Lucene/Elasticsearch) with query parameters 
+using the database (e.g. `identifier` or `date`) could miss matches when more than 500 results match the `_text` query 
+parameter.  This has been fixed, but may be slow if many results match the `_text` query and must be checked against the 
+database parameters."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6231-patient-id-compartment.yaml

@@ -0,0 +1,14 @@
+---
+type: fix
+issue: 6231
+title: "The PatientIdPartitionInterceptor could on rare occasion select the incorrect 
+partition for a resource. This has been corrected. In order for the wrong partition
+to be selected, the following three things need to be true:
+1) there are multiple values of a patient compartment for a resource (see https://hl7.org/fhir/R4/compartmentdefinition-patient.html)
+2) a patient compartment value is a non-Patient reference 
+3) the search parameter of the incorrect value needs to come alphabetically before the search parameter of the correct
+value.
+
+For example, if a QuestionnaireResponse has subject Patient/123 and author Organization/456,
+then since 'author' appears ahead of 'subject' alphabetically it would incorrectly determine the partition.
+The fix changed the partition selection so that it now only matches on Patient references."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/changes.yaml

@@ -0,0 +1,10 @@
+---
+- item:
+      type: "add"
+      title: "The version of a few dependencies have been bumped to more recent versions
+      (dependent HAPI modules listed in brackets):
+      <ul>
+      <li>Bower/Moment.js (hapi-fhir-testpage-overlay): 2.27.0 -&gt; 2.29.4</li>
+      <li>htmlunit (Base): 3.9.0 -&gt; 3.11.0</li>
+      <li>Elasticsearch (Base): 8.11.1 -&gt; 8.14.3</li>
+      </ul>"

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/Batch2DaoSvcImpl.java

@@ -17,7 +17,7 @@
  * limitations under the License.
  * #L%
  */
-package ca.uhn.fhir.jpa.reindex;
+package ca.uhn.fhir.jpa.batch2;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
@@ -41,8 +41,10 @@ import ca.uhn.fhir.rest.api.SortSpec;
 import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.util.DateRangeUtil;
+import ca.uhn.fhir.util.Logs;
 import jakarta.annotation.Nonnull;
 import jakarta.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 
 import java.util.Date;
@@ -50,7 +52,7 @@ import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 public class Batch2DaoSvcImpl implements IBatch2DaoSvc {
-	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(Batch2DaoSvcImpl.class);
+	private static final org.slf4j.Logger ourLog = Logs.getBatchTroubleshootingLog();
 
 	private final IResourceTableDao myResourceTableDao;
 
@@ -83,7 +85,7 @@ public class Batch2DaoSvcImpl implements IBatch2DaoSvc {
 	@Override
 	public IResourcePidStream fetchResourceIdStream(
 			Date theStart, Date theEnd, RequestPartitionId theRequestPartitionId, String theUrl) {
-		if (theUrl == null) {
+		if (StringUtils.isBlank(theUrl)) {
 			return makeStreamResult(
 					theRequestPartitionId, () -> streamResourceIdsNoUrl(theStart, theEnd, theRequestPartitionId));
 		} else {
@@ -127,6 +129,10 @@ public class Batch2DaoSvcImpl implements IBatch2DaoSvc {
 		return new TypedResourceStream(theRequestPartitionId, streamTemplate);
 	}
 
+	/**
+	 * At the moment there is no use-case for this method.
+	 * This can be cleaned up at a later point in time if there is no use for it.
+	 */
 	@Nonnull
 	private Stream<TypedResourcePid> streamResourceIdsNoUrl(
 			Date theStart, Date theEnd, RequestPartitionId theRequestPartitionId) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java

@@ -19,7 +19,6 @@
  */
 package ca.uhn.fhir.jpa.batch2;
 
-import ca.uhn.fhir.batch2.api.IJobPartitionProvider;
 import ca.uhn.fhir.batch2.api.IJobPersistence;
 import ca.uhn.fhir.batch2.config.BaseBatch2Config;
 import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
@@ -28,8 +27,6 @@ import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository;
 import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkMetadataViewRepository;
 import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository;
 import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService;
-import ca.uhn.fhir.jpa.partition.IPartitionLookupSvc;
-import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
 import jakarta.persistence.EntityManager;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -55,10 +52,4 @@ public class JpaBatch2Config extends BaseBatch2Config {
 				theEntityManager,
 				theInterceptorBroadcaster);
 	}
-
-	@Bean
-	public IJobPartitionProvider jobPartitionProvider(
-			IRequestPartitionHelperSvc theRequestPartitionHelperSvc, IPartitionLookupSvc thePartitionLookupSvc) {
-		return new JpaJobPartitionProvider(theRequestPartitionHelperSvc, thePartitionLookupSvc);
-	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPartitionProvider.java

@@ -19,45 +19,47 @@
  */
 package ca.uhn.fhir.jpa.batch2;
 
-import ca.uhn.fhir.batch2.api.IJobPartitionProvider;
+import ca.uhn.fhir.batch2.coordinator.DefaultJobPartitionProvider;
+import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrl;
+import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.interceptor.model.RequestPartitionId;
 import ca.uhn.fhir.jpa.entity.PartitionEntity;
 import ca.uhn.fhir.jpa.partition.IPartitionLookupSvc;
 import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
-import ca.uhn.fhir.rest.api.server.RequestDetails;
+import ca.uhn.fhir.jpa.searchparam.MatchUrlService;
 
 import java.util.List;
 import java.util.stream.Collectors;
 
 /**
  * The default JPA implementation, which uses {@link IRequestPartitionHelperSvc} and {@link IPartitionLookupSvc}
- * to compute the partition to run a batch2 job.
+ * to compute the {@link PartitionedUrl} list to run a batch2 job.
  * The latter will be used to handle cases when the job is configured to run against all partitions
  * (bulk system operation) and will return the actual list with all the configured partitions.
  */
-public class JpaJobPartitionProvider implements IJobPartitionProvider {
-	protected final IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
+@Deprecated
+public class JpaJobPartitionProvider extends DefaultJobPartitionProvider {
 	private final IPartitionLookupSvc myPartitionLookupSvc;
 
 	public JpaJobPartitionProvider(
 			IRequestPartitionHelperSvc theRequestPartitionHelperSvc, IPartitionLookupSvc thePartitionLookupSvc) {
-		myRequestPartitionHelperSvc = theRequestPartitionHelperSvc;
+		super(theRequestPartitionHelperSvc);
+		myPartitionLookupSvc = thePartitionLookupSvc;
+	}
+
+	public JpaJobPartitionProvider(
+			FhirContext theFhirContext,
+			IRequestPartitionHelperSvc theRequestPartitionHelperSvc,
+			MatchUrlService theMatchUrlService,
+			IPartitionLookupSvc thePartitionLookupSvc) {
+		super(theFhirContext, theRequestPartitionHelperSvc, theMatchUrlService);
 		myPartitionLookupSvc = thePartitionLookupSvc;
 	}
 
 	@Override
-	public List<RequestPartitionId> getPartitions(RequestDetails theRequestDetails, String theOperation) {
-		RequestPartitionId partitionId = myRequestPartitionHelperSvc.determineReadPartitionForRequestForServerOperation(
-				theRequestDetails, theOperation);
-		if (!partitionId.isAllPartitions()) {
-			return List.of(partitionId);
-		}
-		// handle (bulk) system operations that are typically configured with RequestPartitionId.allPartitions()
-		// populate the actual list of all partitions
-		List<RequestPartitionId> partitionIdList = myPartitionLookupSvc.listPartitions().stream()
+	public List<RequestPartitionId> getAllPartitions() {
+		return myPartitionLookupSvc.listPartitions().stream()
 				.map(PartitionEntity::toRequestPartitionId)
 				.collect(Collectors.toList());
-		partitionIdList.add(RequestPartitionId.defaultPartition());
-		return partitionIdList;
 	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/Batch2SupportConfig.java

@@ -25,6 +25,7 @@ import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc;
 import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc;
 import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
+import ca.uhn.fhir.jpa.batch2.Batch2DaoSvcImpl;
 import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
 import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
@@ -32,7 +33,6 @@ import ca.uhn.fhir.jpa.dao.expunge.ResourceTableFKProvider;
 import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService;
 import ca.uhn.fhir.jpa.delete.batch2.DeleteExpungeSqlBuilder;
 import ca.uhn.fhir.jpa.delete.batch2.DeleteExpungeSvcImpl;
-import ca.uhn.fhir.jpa.reindex.Batch2DaoSvcImpl;
 import ca.uhn.fhir.jpa.searchparam.MatchUrlService;
 import jakarta.persistence.EntityManager;
 import org.springframework.beans.factory.annotation.Autowired;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java

@@ -20,7 +20,7 @@
 package ca.uhn.fhir.jpa.dao;
 
 import ca.uhn.fhir.batch2.api.IJobCoordinator;
-import ca.uhn.fhir.batch2.jobs.parameters.UrlPartitioner;
+import ca.uhn.fhir.batch2.api.IJobPartitionProvider;
 import ca.uhn.fhir.batch2.jobs.reindex.ReindexAppCtx;
 import ca.uhn.fhir.batch2.jobs.reindex.ReindexJobParameters;
 import ca.uhn.fhir.batch2.model.JobInstanceStartRequest;
@@ -103,7 +103,6 @@ import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException;
 import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
-import ca.uhn.fhir.rest.server.provider.ProviderConstants;
 import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
 import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
 import ca.uhn.fhir.util.ReflectionUtil;
@@ -193,6 +192,9 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 	@Autowired
 	private IRequestPartitionHelperSvc myRequestPartitionHelperService;
 
+	@Autowired
+	private IJobPartitionProvider myJobPartitionProvider;
+
 	@Autowired
 	private MatchUrlService myMatchUrlService;
 
@@ -214,9 +216,6 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 
 	private TransactionTemplate myTxTemplate;
 
-	@Autowired
-	private UrlPartitioner myUrlPartitioner;
-
 	@Autowired
 	private ResourceSearchUrlSvc myResourceSearchUrlSvc;
 
@@ -1306,14 +1305,12 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 
 			ReindexJobParameters params = new ReindexJobParameters();
 
+			List<String> urls = List.of();
 			if (!isCommonSearchParam(theBase)) {
-				addAllResourcesTypesToReindex(theBase, theRequestDetails, params);
+				urls = theBase.stream().map(t -> t + "?").collect(Collectors.toList());
 			}
 
-			RequestPartitionId requestPartition =
-					myRequestPartitionHelperService.determineReadPartitionForRequestForServerOperation(
-							theRequestDetails, ProviderConstants.OPERATION_REINDEX);
-			params.setRequestPartitionId(requestPartition);
+			myJobPartitionProvider.getPartitionedUrls(theRequestDetails, urls).forEach(params::addPartitionedUrl);
 
 			JobInstanceStartRequest request = new JobInstanceStartRequest();
 			request.setJobDefinitionId(ReindexAppCtx.JOB_REINDEX);
@@ -1334,14 +1331,6 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 		return Boolean.parseBoolean(shouldSkip.toString());
 	}
 
-	private void addAllResourcesTypesToReindex(
-			List<String> theBase, RequestDetails theRequestDetails, ReindexJobParameters params) {
-		theBase.stream()
-				.map(t -> t + "?")
-				.map(url -> myUrlPartitioner.partitionUrl(url, theRequestDetails))
-				.forEach(params::addPartitionedUrl);
-	}
-
 	private boolean isCommonSearchParam(List<String> theBase) {
 		// If the base contains the special resource "Resource", this is a common SP that applies to all resources
 		return theBase.stream().map(String::toLowerCase).anyMatch(BASE_RESOURCE_NAME::equals);
@@ -2457,11 +2446,13 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 			RestOperationTypeEnum theOperationType,
 			TransactionDetails theTransactionDetails) {
 
-		// we stored a resource searchUrl at creation time to prevent resource duplication.  Let's remove the entry on
-		// the
-		// first update but guard against unnecessary trips to the database on subsequent ones.
+		/*
+		 * We stored a resource searchUrl at creation time to prevent resource duplication.
+		 * We'll clear any currently existing urls from the db, otherwise we could hit
+		 * duplicate index violations if we try to add another (after this create/update)
+		 */
 		ResourceTable entity = (ResourceTable) theEntity;
-		if (entity.isSearchUrlPresent() && thePerformIndexing) {
+		if (entity.isSearchUrlPresent()) {
 			myResourceSearchUrlSvc.deleteByResId(
 					(Long) theEntity.getPersistentId().getId());
 			entity.setSearchUrlPresent(false);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java

@@ -32,6 +32,7 @@ import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchResourceProjection;
 import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder;
 import ca.uhn.fhir.jpa.dao.search.IHSearchSortHelper;
 import ca.uhn.fhir.jpa.dao.search.LastNOperation;
+import ca.uhn.fhir.jpa.dao.search.SearchScrollQueryExecutorAdaptor;
 import ca.uhn.fhir.jpa.model.dao.JpaPid;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.model.search.ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams;
@@ -40,6 +41,7 @@ import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
 import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions;
 import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteSearch;
 import ca.uhn.fhir.jpa.search.builder.ISearchQueryExecutor;
+import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
 import ca.uhn.fhir.jpa.search.builder.SearchQueryExecutors;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor;
@@ -183,6 +185,19 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 		return doSearch(theResourceName, theParams, null, theMaxResultsToFetch, theRequestDetails);
 	}
 
+	@Transactional
+	@Override
+	public ISearchQueryExecutor searchScrolled(
+			String theResourceType, SearchParameterMap theParams, RequestDetails theRequestDetails) {
+		validateHibernateSearchIsEnabled();
+
+		SearchQueryOptionsStep<?, Long, SearchLoadingOptionsStep, ?, ?> searchQueryOptionsStep =
+				getSearchQueryOptionsStep(theResourceType, theParams, null);
+		logQuery(searchQueryOptionsStep, theRequestDetails);
+
+		return new SearchScrollQueryExecutorAdaptor(searchQueryOptionsStep.scroll(SearchBuilder.getMaximumPageSize()));
+	}
+
 	// keep this in sync with supportsSomeOf();
 	@SuppressWarnings("rawtypes")
 	private ISearchQueryExecutor doSearch(

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java

@@ -62,6 +62,17 @@ public interface IFulltextSearchSvc {
 			Integer theMaxResultsToFetch,
 			RequestDetails theRequestDetails);
 
+	/**
+	 * Query the index for a complete iterator of ALL results. (scrollable search result).
+	 *
+	 * @param theResourceName      e.g. Patient
+	 * @param theParams            The search query
+	 * @param theRequestDetails The request details
+	 * @return Iterator of result PIDs
+	 */
+	ISearchQueryExecutor searchScrolled(
+			String theResourceName, SearchParameterMap theParams, RequestDetails theRequestDetails);
+
 	/**
 	 * Autocomplete search for NIH $expand contextDirection=existing
 	 * @param theOptions operation options

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/TransactionProcessor.java

@@ -27,7 +27,6 @@ import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
 import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
 import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
 import ca.uhn.fhir.jpa.config.HapiFhirHibernateJpaDialect;
-import ca.uhn.fhir.jpa.model.config.PartitionSettings;
 import ca.uhn.fhir.jpa.model.dao.JpaPid;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
 import ca.uhn.fhir.jpa.model.entity.StorageSettings;
@@ -97,9 +96,6 @@ public class TransactionProcessor extends BaseTransactionProcessor {
 	@Autowired
 	private IIdHelperService<JpaPid> myIdHelperService;
 
-	@Autowired
-	private PartitionSettings myPartitionSettings;
-
 	@Autowired
 	private JpaStorageSettings myStorageSettings;
 
@@ -150,14 +146,9 @@ public class TransactionProcessor extends BaseTransactionProcessor {
 			List<IBase> theEntries,
 			StopWatch theTransactionStopWatch) {
 
-		ITransactionProcessorVersionAdapter versionAdapter = getVersionAdapter();
-		RequestPartitionId requestPartitionId = null;
-		if (!myPartitionSettings.isPartitioningEnabled()) {
-			requestPartitionId = RequestPartitionId.allPartitions();
-		} else {
-			// If all entries in the transaction point to the exact same partition, we'll try and do a pre-fetch
-			requestPartitionId = getSinglePartitionForAllEntriesOrNull(theRequest, theEntries, versionAdapter);
-		}
+		ITransactionProcessorVersionAdapter<?, ?> versionAdapter = getVersionAdapter();
+		RequestPartitionId requestPartitionId =
+				super.determineRequestPartitionIdForWriteEntries(theRequest, theEntries);
 
 		if (requestPartitionId != null) {
 			preFetch(theTransactionDetails, theEntries, versionAdapter, requestPartitionId);
@@ -472,24 +463,6 @@ public class TransactionProcessor extends BaseTransactionProcessor {
 		}
 	}
 
-	private RequestPartitionId getSinglePartitionForAllEntriesOrNull(
-			RequestDetails theRequest, List<IBase> theEntries, ITransactionProcessorVersionAdapter versionAdapter) {
-		RequestPartitionId retVal = null;
-		Set<RequestPartitionId> requestPartitionIdsForAllEntries = new HashSet<>();
-		for (IBase nextEntry : theEntries) {
-			IBaseResource resource = versionAdapter.getResource(nextEntry);
-			if (resource != null) {
-				RequestPartitionId requestPartition = myRequestPartitionSvc.determineCreatePartitionForRequest(
-						theRequest, resource, myFhirContext.getResourceType(resource));
-				requestPartitionIdsForAllEntries.add(requestPartition);
-			}
-		}
-		if (requestPartitionIdsForAllEntries.size() == 1) {
-			retVal = requestPartitionIdsForAllEntries.iterator().next();
-		}
-		return retVal;
-	}
-
 	/**
 	 * Given a token parameter, build the query predicate based on its hash. Uses system and value if both are available, otherwise just value.
 	 * If neither are available, it returns null.
@@ -570,11 +543,6 @@ public class TransactionProcessor extends BaseTransactionProcessor {
 		}
 	}
 
-	@VisibleForTesting
-	public void setPartitionSettingsForUnitTest(PartitionSettings thePartitionSettings) {
-		myPartitionSettings = thePartitionSettings;
-	}
-
 	@VisibleForTesting
 	public void setIdHelperServiceForUnitTest(IIdHelperService theIdHelperService) {
 		myIdHelperService = theIdHelperService;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTableDao.java

@@ -135,7 +135,8 @@ public interface IResourceTableDao
 	 * This method returns a Collection where each row is an element in the collection. Each element in the collection
 	 * is an object array, where the order matters (the array represents columns returned by the query). Be careful if you change this query in any way.
 	 */
-	@Query("SELECT t.myResourceType, t.myId, t.myDeleted FROM ResourceTable t WHERE t.myId IN (:pid)")
+	@Query(
+			"SELECT t.myResourceType, t.myId, t.myDeleted, t.myPartitionIdValue, t.myPartitionDateValue FROM ResourceTable t WHERE t.myId IN (:pid)")
 	Collection<Object[]> findLookupFieldsByResourcePid(@Param("pid") List<Long> thePids);
 
 	/**
@@ -143,7 +144,7 @@ public interface IResourceTableDao
 	 * is an object array, where the order matters (the array represents columns returned by the query). Be careful if you change this query in any way.
 	 */
 	@Query(
-			"SELECT t.myResourceType, t.myId, t.myDeleted FROM ResourceTable t WHERE t.myId IN (:pid) AND t.myPartitionIdValue IN :partition_id")
+			"SELECT t.myResourceType, t.myId, t.myDeleted, t.myPartitionIdValue, t.myPartitionDateValue FROM ResourceTable t WHERE t.myId IN (:pid) AND t.myPartitionIdValue IN :partition_id")
 	Collection<Object[]> findLookupFieldsByResourcePidInPartitionIds(
 			@Param("pid") List<Long> thePids, @Param("partition_id") Collection<Integer> thePartitionId);
 
@@ -152,7 +153,7 @@ public interface IResourceTableDao
 	 * is an object array, where the order matters (the array represents columns returned by the query). Be careful if you change this query in any way.
 	 */
 	@Query(
-			"SELECT t.myResourceType, t.myId, t.myDeleted FROM ResourceTable t WHERE t.myId IN (:pid) AND (t.myPartitionIdValue IS NULL OR t.myPartitionIdValue IN :partition_id)")
+			"SELECT t.myResourceType, t.myId, t.myDeleted, t.myPartitionIdValue, t.myPartitionDateValue FROM ResourceTable t WHERE t.myId IN (:pid) AND (t.myPartitionIdValue IS NULL OR t.myPartitionIdValue IN :partition_id)")
 	Collection<Object[]> findLookupFieldsByResourcePidInPartitionIdsOrNullPartition(
 			@Param("pid") List<Long> thePids, @Param("partition_id") Collection<Integer> thePartitionId);
 
@@ -161,7 +162,7 @@ public interface IResourceTableDao
 	 * is an object array, where the order matters (the array represents columns returned by the query). Be careful if you change this query in any way.
 	 */
 	@Query(
-			"SELECT t.myResourceType, t.myId, t.myDeleted FROM ResourceTable t WHERE t.myId IN (:pid) AND t.myPartitionIdValue IS NULL")
+			"SELECT t.myResourceType, t.myId, t.myDeleted, t.myPartitionIdValue, t.myPartitionDateValue FROM ResourceTable t WHERE t.myId IN (:pid) AND t.myPartitionIdValue IS NULL")
 	Collection<Object[]> findLookupFieldsByResourcePidInPartitionNull(@Param("pid") List<Long> thePids);
 
 	@Query("SELECT t.myVersion FROM ResourceTable t WHERE t.myId = :pid")

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/custom/IResourceTableDaoImpl.java

@@ -56,9 +56,10 @@ public class IResourceTableDaoImpl implements IForcedIdQueries {
 	@Override
 	public Collection<Object[]> findAndResolveByForcedIdWithNoType(
 			String theResourceType, Collection<String> theForcedIds, boolean theExcludeDeleted) {
-		String query = "SELECT t.myResourceType, t.myId, t.myFhirId, t.myDeleted "
-				+ "FROM ResourceTable t "
-				+ "WHERE t.myResourceType = :resource_type AND t.myFhirId IN ( :forced_id )";
+		String query =
+				"SELECT t.myResourceType, t.myId, t.myFhirId, t.myDeleted, t.myPartitionIdValue, t.myPartitionDateValue "
+						+ "FROM ResourceTable t "
+						+ "WHERE t.myResourceType = :resource_type AND t.myFhirId IN ( :forced_id )";
 
 		if (theExcludeDeleted) {
 			query += " AND t.myDeleted IS NULL";
@@ -82,9 +83,10 @@ public class IResourceTableDaoImpl implements IForcedIdQueries {
 			Collection<String> theForcedIds,
 			Collection<Integer> thePartitionId,
 			boolean theExcludeDeleted) {
-		String query = "SELECT t.myResourceType, t.myId, t.myFhirId, t.myDeleted "
-				+ "FROM ResourceTable t "
-				+ "WHERE t.myResourceType = :resource_type AND t.myFhirId IN ( :forced_id ) AND t.myPartitionIdValue IN ( :partition_id )";
+		String query =
+				"SELECT t.myResourceType, t.myId, t.myFhirId, t.myDeleted, t.myPartitionIdValue, t.myPartitionDateValue "
+						+ "FROM ResourceTable t "
+						+ "WHERE t.myResourceType = :resource_type AND t.myFhirId IN ( :forced_id ) AND t.myPartitionIdValue IN ( :partition_id )";
 
 		if (theExcludeDeleted) {
 			query += " AND t.myDeleted IS NULL";
@@ -106,9 +108,11 @@ public class IResourceTableDaoImpl implements IForcedIdQueries {
 	@Override
 	public Collection<Object[]> findAndResolveByForcedIdWithNoTypeInPartitionNull(
 			String theResourceType, Collection<String> theForcedIds, boolean theExcludeDeleted) {
-		String query = "SELECT t.myResourceType, t.myId, t.myFhirId, t.myDeleted "
-				+ "FROM ResourceTable t "
-				+ "WHERE t.myResourceType = :resource_type AND t.myFhirId IN ( :forced_id ) AND t.myPartitionIdValue IS NULL";
+		// we fetch myPartitionIdValue and myPartitionDateValue for resultSet processing consistency
+		String query =
+				"SELECT t.myResourceType, t.myId, t.myFhirId, t.myDeleted, t.myPartitionIdValue, t.myPartitionDateValue "
+						+ "FROM ResourceTable t "
+						+ "WHERE t.myResourceType = :resource_type AND t.myFhirId IN ( :forced_id ) AND t.myPartitionIdValue IS NULL";
 
 		if (theExcludeDeleted) {
 			query += " AND t.myDeleted IS NULL";
@@ -132,9 +136,10 @@ public class IResourceTableDaoImpl implements IForcedIdQueries {
 			Collection<String> theForcedIds,
 			List<Integer> thePartitionIdsWithoutDefault,
 			boolean theExcludeDeleted) {
-		String query = "SELECT t.myResourceType, t.myId, t.myFhirId, t.myDeleted "
-				+ "FROM ResourceTable t "
-				+ "WHERE t.myResourceType = :resource_type AND t.myFhirId IN ( :forced_id ) AND (t.myPartitionIdValue IS NULL OR t.myPartitionIdValue IN ( :partition_id ))";
+		String query =
+				"SELECT t.myResourceType, t.myId, t.myFhirId, t.myDeleted, t.myPartitionIdValue, t.myPartitionDateValue "
+						+ "FROM ResourceTable t "
+						+ "WHERE t.myResourceType = :resource_type AND t.myFhirId IN ( :forced_id ) AND (t.myPartitionIdValue IS NULL OR t.myPartitionIdValue IN ( :partition_id ))";
 
 		if (theExcludeDeleted) {
 			query += " AND t.myDeleted IS NULL";

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/index/IdHelperService.java

@@ -30,6 +30,7 @@ import ca.uhn.fhir.jpa.model.config.PartitionSettings;
 import ca.uhn.fhir.jpa.model.cross.IResourceLookup;
 import ca.uhn.fhir.jpa.model.cross.JpaResourceLookup;
 import ca.uhn.fhir.jpa.model.dao.JpaPid;
+import ca.uhn.fhir.jpa.model.entity.PartitionablePartitionId;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
 import ca.uhn.fhir.jpa.util.MemoryCacheService;
@@ -59,12 +60,11 @@ import org.hl7.fhir.instance.model.api.IAnyResource;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r4.model.IdType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.support.TransactionSynchronizationManager;
 
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -100,7 +100,6 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
  */
 @Service
 public class IdHelperService implements IIdHelperService<JpaPid> {
-	private static final Logger ourLog = LoggerFactory.getLogger(IdHelperService.class);
 	public static final Predicate[] EMPTY_PREDICATE_ARRAY = new Predicate[0];
 	public static final String RESOURCE_PID = "RESOURCE_PID";
 
@@ -523,7 +522,7 @@ public class IdHelperService implements IIdHelperService<JpaPid> {
 		if (myStorageSettings.getResourceClientIdStrategy() != JpaStorageSettings.ClientIdStrategyEnum.ANY) {
 			List<Long> pids = theId.stream()
 					.filter(t -> isValidPid(t))
-					.map(t -> t.getIdPartAsLong())
+					.map(IIdType::getIdPartAsLong)
 					.collect(Collectors.toList());
 			if (!pids.isEmpty()) {
 				resolvePids(requestPartitionId, pids, retVal);
@@ -578,8 +577,14 @@ public class IdHelperService implements IIdHelperService<JpaPid> {
 					Long resourcePid = (Long) next[1];
 					String forcedId = (String) next[2];
 					Date deletedAt = (Date) next[3];
+					Integer partitionId = (Integer) next[4];
+					LocalDate partitionDate = (LocalDate) next[5];
 
-					JpaResourceLookup lookup = new JpaResourceLookup(resourceType, resourcePid, deletedAt);
+					JpaResourceLookup lookup = new JpaResourceLookup(
+							resourceType,
+							resourcePid,
+							deletedAt,
+							PartitionablePartitionId.with(partitionId, partitionDate));
 					retVal.computeIfAbsent(forcedId, id -> new ArrayList<>()).add(lookup);
 
 					if (!myStorageSettings.isDeleteEnabled()) {
@@ -638,7 +643,11 @@ public class IdHelperService implements IIdHelperService<JpaPid> {
 				}
 			}
 			lookup.stream()
-					.map(t -> new JpaResourceLookup((String) t[0], (Long) t[1], (Date) t[2]))
+					.map(t -> new JpaResourceLookup(
+							(String) t[0],
+							(Long) t[1],
+							(Date) t[2],
+							PartitionablePartitionId.with((Integer) t[3], (LocalDate) t[4])))
 					.forEach(t -> {
 						String id = t.getPersistentId().toString();
 						if (!theTargets.containsKey(id)) {
@@ -683,9 +692,8 @@ public class IdHelperService implements IIdHelperService<JpaPid> {
 					MemoryCacheService.CacheEnum.PID_TO_FORCED_ID, nextResourcePid, Optional.empty());
 		}
 		Map<JpaPid, Optional<String>> convertRetVal = new HashMap<>();
-		retVal.forEach((k, v) -> {
-			convertRetVal.put(JpaPid.fromId(k), v);
-		});
+		retVal.forEach((k, v) -> convertRetVal.put(JpaPid.fromId(k), v));
+
 		return new PersistentIdToForcedIdMap<>(convertRetVal);
 	}
 
@@ -716,7 +724,8 @@ public class IdHelperService implements IIdHelperService<JpaPid> {
 		}
 
 		if (!myStorageSettings.isDeleteEnabled()) {
-			JpaResourceLookup lookup = new JpaResourceLookup(theResourceType, theJpaPid.getId(), theDeletedAt);
+			JpaResourceLookup lookup = new JpaResourceLookup(
+					theResourceType, theJpaPid.getId(), theDeletedAt, theJpaPid.getPartitionablePartitionId());
 			String nextKey = theJpaPid.toString();
 			myMemoryCacheService.putAfterCommit(MemoryCacheService.CacheEnum.RESOURCE_LOOKUP, nextKey, lookup);
 		}
@@ -744,8 +753,7 @@ public class IdHelperService implements IIdHelperService<JpaPid> {
 	@Nonnull
 	public List<JpaPid> getPidsOrThrowException(
 			@Nonnull RequestPartitionId theRequestPartitionId, List<IIdType> theIds) {
-		List<JpaPid> resourcePersistentIds = resolveResourcePersistentIdsWithCache(theRequestPartitionId, theIds);
-		return resourcePersistentIds;
+		return resolveResourcePersistentIdsWithCache(theRequestPartitionId, theIds);
 	}
 
 	@Override

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchSearchBuilder.java

@@ -59,7 +59,7 @@ public class ExtendedHSearchSearchBuilder {
 	/**
 	 * These params have complicated semantics, or are best resolved at the JPA layer for now.
 	 */
-	public static final Set<String> ourUnsafeSearchParmeters = Sets.newHashSet("_id", "_meta");
+	public static final Set<String> ourUnsafeSearchParmeters = Sets.newHashSet("_id", "_meta", "_count");
 
 	/**
 	 * Determine if ExtendedHibernateSearchBuilder can support this parameter
@@ -67,20 +67,22 @@ public class ExtendedHSearchSearchBuilder {
 	 * @param theActiveParamsForResourceType active search parameters for the desired resource type
 	 * @return whether or not this search parameter is supported in hibernate
 	 */
-	public boolean supportsSearchParameter(String theParamName, ResourceSearchParams theActiveParamsForResourceType) {
+	public boolean illegalForHibernateSearch(String theParamName, ResourceSearchParams theActiveParamsForResourceType) {
 		if (theActiveParamsForResourceType == null) {
-			return false;
+			return true;
 		}
 		if (ourUnsafeSearchParmeters.contains(theParamName)) {
-			return false;
+			return true;
 		}
 		if (!theActiveParamsForResourceType.containsParamName(theParamName)) {
-			return false;
+			return true;
 		}
-		return true;
+		return false;
 	}
 
 	/**
+	 * By default, do not use Hibernate Search.
+	 * If a Search Parameter is supported by hibernate search,
 	 * Are any of the queries supported by our indexing?
 	 * -
 	 * If not, do not use hibernate, because the results will
@@ -88,12 +90,12 @@ public class ExtendedHSearchSearchBuilder {
 	 */
 	public boolean canUseHibernateSearch(
 			String theResourceType, SearchParameterMap myParams, ISearchParamRegistry theSearchParamRegistry) {
-		boolean canUseHibernate = true;
+		boolean canUseHibernate = false;
 
 		ResourceSearchParams resourceActiveSearchParams = theSearchParamRegistry.getActiveSearchParams(theResourceType);
 		for (String paramName : myParams.keySet()) {
 			// is this parameter supported?
-			if (!supportsSearchParameter(paramName, resourceActiveSearchParams)) {
+			if (illegalForHibernateSearch(paramName, resourceActiveSearchParams)) {
 				canUseHibernate = false;
 			} else {
 				// are the parameter values supported?
@@ -218,7 +220,7 @@ public class ExtendedHSearchSearchBuilder {
 		ArrayList<String> paramNames = compileParamNames(searchParameterMap);
 		ResourceSearchParams activeSearchParams = searchParamRegistry.getActiveSearchParams(resourceType);
 		for (String nextParam : paramNames) {
-			if (!supportsSearchParameter(nextParam, activeSearchParams)) {
+			if (illegalForHibernateSearch(nextParam, activeSearchParams)) {
 				// ignore magic params handled in JPA
 				continue;
 			}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java

@@ -414,6 +414,7 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 			version.onTable("HFJ_IDX_CMB_TOK_NU")
 					.addIndex("20240625.10", "IDX_IDXCMBTOKNU_HASHC")
 					.unique(false)
+					.online(true)
 					.withColumns("HASH_COMPLETE", "RES_ID", "PARTITION_ID");
 			version.onTable("HFJ_IDX_CMP_STRING_UNIQ")
 					.addColumn("20240625.20", "HASH_COMPLETE")
@@ -470,10 +471,26 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 			}
 		}
 
-		version.onTable(Search.HFJ_SEARCH)
-				.modifyColumn("20240722.1", Search.SEARCH_UUID)
-				.nonNullable()
-				.withType(ColumnTypeEnum.STRING, 48);
+		{
+			// Add target resource partition id/date columns to resource link
+			Builder.BuilderWithTableName resourceLinkTable = version.onTable("HFJ_RES_LINK");
+
+			resourceLinkTable
+					.addColumn("20240718.10", "TARGET_RES_PARTITION_ID")
+					.nullable()
+					.type(ColumnTypeEnum.INT);
+			resourceLinkTable
+					.addColumn("20240718.20", "TARGET_RES_PARTITION_DATE")
+					.nullable()
+					.type(ColumnTypeEnum.DATE_ONLY);
+		}
+
+		{
+			version.onTable(Search.HFJ_SEARCH)
+					.modifyColumn("20240722.1", Search.SEARCH_UUID)
+					.nonNullable()
+					.withType(ColumnTypeEnum.STRING, 48);
+		}
 
 		{
 			final Builder.BuilderWithTableName hfjResource = version.onTable("HFJ_RESOURCE");

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/cross/JpaResourceLookup.java

@@ -20,18 +20,26 @@
 package ca.uhn.fhir.jpa.model.cross;
 
 import ca.uhn.fhir.jpa.model.dao.JpaPid;
+import ca.uhn.fhir.jpa.model.entity.PartitionablePartitionId;
 
 import java.util.Date;
 
 public class JpaResourceLookup implements IResourceLookup<JpaPid> {
+
 	private final String myResourceType;
 	private final Long myResourcePid;
 	private final Date myDeletedAt;
+	private final PartitionablePartitionId myPartitionablePartitionId;
 
-	public JpaResourceLookup(String theResourceType, Long theResourcePid, Date theDeletedAt) {
+	public JpaResourceLookup(
+			String theResourceType,
+			Long theResourcePid,
+			Date theDeletedAt,
+			PartitionablePartitionId thePartitionablePartitionId) {
 		myResourceType = theResourceType;
 		myResourcePid = theResourcePid;
 		myDeletedAt = theDeletedAt;
+		myPartitionablePartitionId = thePartitionablePartitionId;
 	}
 
 	@Override
@@ -46,6 +54,9 @@ public class JpaResourceLookup implements IResourceLookup<JpaPid> {
 
 	@Override
 	public JpaPid getPersistentId() {
-		return JpaPid.fromId(myResourcePid);
+		JpaPid jpaPid = JpaPid.fromId(myResourcePid);
+		jpaPid.setPartitionablePartitionId(myPartitionablePartitionId);
+
+		return jpaPid;
 	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/TerminologyUploaderProvider.java

@@ -475,6 +475,9 @@ public class TerminologyUploaderProvider extends BaseJpaProvider {
 	}
 
 	private static String csvEscape(String theValue) {
+		if (theValue == null) {
+			return "";
+		}
 		return '"' + theValue.replace("\"", "\"\"").replace("\n", "\\n").replace("\r", "") + '"';
 	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java

@@ -101,7 +101,6 @@ import ca.uhn.fhir.util.StringUtil;
 import ca.uhn.fhir.util.UrlUtil;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Streams;
 import com.healthmarketscience.sqlbuilder.Condition;
 import jakarta.annotation.Nonnull;
 import jakarta.annotation.Nullable;
@@ -141,7 +140,9 @@ import java.util.stream.Collectors;
 import static ca.uhn.fhir.jpa.model.util.JpaConstants.UNDESIRED_RESOURCE_LINKAGES_FOR_EVERYTHING_ON_PATIENT_INSTANCE;
 import static ca.uhn.fhir.jpa.search.builder.QueryStack.LOCATION_POSITION;
 import static ca.uhn.fhir.jpa.search.builder.QueryStack.SearchForIdsParams.with;
+import static ca.uhn.fhir.jpa.util.InClauseNormalizer.*;
 import static java.util.Objects.requireNonNull;
+import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
 import static org.apache.commons.lang3.StringUtils.defaultString;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
@@ -205,9 +206,6 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 	@Autowired(required = false)
 	private IElasticsearchSvc myIElasticsearchSvc;
 
-	@Autowired
-	private FhirContext myCtx;
-
 	@Autowired
 	private IJpaStorageResourceParser myJpaStorageResourceParser;
 
@@ -332,8 +330,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 		init(theParams, theSearchUuid, theRequestPartitionId);
 
 		if (checkUseHibernateSearch()) {
-			long count = myFulltextSearchSvc.count(myResourceName, theParams.clone());
-			return count;
+			return myFulltextSearchSvc.count(myResourceName, theParams.clone());
 		}
 
 		List<ISearchQueryExecutor> queries = createQuery(theParams.clone(), null, null, null, true, theRequest, null);
@@ -404,8 +401,16 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 				fulltextMatchIds = queryHibernateSearchForEverythingPids(theRequest);
 				resultCount = fulltextMatchIds.size();
 			} else {
-				fulltextExecutor = myFulltextSearchSvc.searchNotScrolled(
-						myResourceName, myParams, myMaxResultsToFetch, theRequest);
+				// todo performance MB - some queries must intersect with JPA (e.g. they have a chain, or we haven't
+				// enabled SP indexing).
+				// and some queries don't need JPA.  We only need the scroll when we need to intersect with JPA.
+				// It would be faster to have a non-scrolled search in this case, since creating the scroll requires
+				// extra work in Elastic.
+				// if (eligibleToSkipJPAQuery) fulltextExecutor = myFulltextSearchSvc.searchNotScrolled( ...
+
+				// we might need to intersect with JPA.  So we might need to traverse ALL results from lucene, not just
+				// a page.
+				fulltextExecutor = myFulltextSearchSvc.searchScrolled(myResourceName, myParams, theRequest);
 			}
 
 			if (fulltextExecutor == null) {
@@ -457,7 +462,8 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 				// We break the pids into chunks that fit in the 1k limit for jdbc bind params.
 				new QueryChunker<Long>()
 						.chunk(
-								Streams.stream(fulltextExecutor).collect(Collectors.toList()),
+								fulltextExecutor,
+								SearchBuilder.getMaximumPageSize(),
 								t -> doCreateChunkedQueries(
 										theParams, t, theOffset, sort, theCountOnlyFlag, theRequest, queries));
 			}
@@ -560,8 +566,9 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 			boolean theCount,
 			RequestDetails theRequest,
 			ArrayList<ISearchQueryExecutor> theQueries) {
+
 		if (thePids.size() < getMaximumPageSize()) {
-			normalizeIdListForLastNInClause(thePids);
+			thePids = normalizeIdListForInClause(thePids);
 		}
 		createChunkedQuery(theParams, sort, theOffset, thePids.size(), theCount, theRequest, thePids, theQueries);
 	}
@@ -885,41 +892,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 				&& theParams.values().stream()
 						.flatMap(Collection::stream)
 						.flatMap(Collection::stream)
-						.anyMatch(t -> t instanceof ReferenceParam);
-	}
-
-	private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) {
-		/*
-		The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying
-		numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info:
-		https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage.
-
-		Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of
-		arguments never exceeds the maximum specified below.
-		*/
-		int listSize = lastnResourceIds.size();
-
-		if (listSize > 1 && listSize < 10) {
-			padIdListWithPlaceholders(lastnResourceIds, 10);
-		} else if (listSize > 10 && listSize < 50) {
-			padIdListWithPlaceholders(lastnResourceIds, 50);
-		} else if (listSize > 50 && listSize < 100) {
-			padIdListWithPlaceholders(lastnResourceIds, 100);
-		} else if (listSize > 100 && listSize < 200) {
-			padIdListWithPlaceholders(lastnResourceIds, 200);
-		} else if (listSize > 200 && listSize < 500) {
-			padIdListWithPlaceholders(lastnResourceIds, 500);
-		} else if (listSize > 500 && listSize < 800) {
-			padIdListWithPlaceholders(lastnResourceIds, 800);
-		}
-
-		return lastnResourceIds;
-	}
-
-	private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
-		while (theIdList.size() < preferredListSize) {
-			theIdList.add(-1L);
-		}
+						.anyMatch(ReferenceParam.class::isInstance);
 	}
 
 	private void createSort(QueryStack theQueryStack, SortSpec theSort, SearchParameterMap theParams) {
@@ -1154,7 +1127,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 
 		List<Long> versionlessPids = JpaPid.toLongList(thePids);
 		if (versionlessPids.size() < getMaximumPageSize()) {
-			versionlessPids = normalizeIdListForLastNInClause(versionlessPids);
+			versionlessPids = normalizeIdListForInClause(versionlessPids);
 		}
 
 		// -- get the resource from the searchView
@@ -1243,7 +1216,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 		Map<Long, Collection<ResourceTag>> tagMap = new HashMap<>();
 
 		// -- no tags
-		if (thePidList.size() == 0) return tagMap;
+		if (thePidList.isEmpty()) return tagMap;
 
 		// -- get all tags for the idList
 		Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(thePidList);
@@ -1383,7 +1356,6 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 		EntityManager entityManager = theParameters.getEntityManager();
 		Integer maxCount = theParameters.getMaxCount();
 		FhirContext fhirContext = theParameters.getFhirContext();
-		DateRangeParam lastUpdated = theParameters.getLastUpdated();
 		RequestDetails request = theParameters.getRequestDetails();
 		String searchIdOrDescription = theParameters.getSearchIdOrDescription();
 		List<String> desiredResourceTypes = theParameters.getDesiredResourceTypes();
@@ -1922,11 +1894,10 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 		}
 		assert !targetResourceTypes.isEmpty();
 
-		Set<Long> identityHashesForTypes = targetResourceTypes.stream()
+		return targetResourceTypes.stream()
 				.map(type -> BaseResourceIndexedSearchParam.calculateHashIdentity(
 						myPartitionSettings, myRequestPartitionId, type, "url"))
 				.collect(Collectors.toSet());
-		return identityHashesForTypes;
 	}
 
 	private <T> List<Collection<T>> partition(Collection<T> theNextRoundMatches, int theMaxLoad) {
@@ -2506,7 +2477,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 
 		private void retrieveNextIteratorQuery() {
 			close();
-			if (myQueryList != null && myQueryList.size() > 0) {
+			if (isNotEmpty(myQueryList)) {
 				myResultsIterator = myQueryList.remove(0);
 				myHasNextIteratorQuery = true;
 			} else {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/InClauseNormalizer.java

@@ -0,0 +1,65 @@
+package ca.uhn.fhir.jpa.util;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/*
+	This class encapsulate the implementation providing a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying
+	numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info:
+	https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage.
+
+	Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of
+	arguments never exceeds the maximum specified below.
+*/
+public class InClauseNormalizer {
+
+	public static List<Long> normalizeIdListForInClause(List<Long> theResourceIds) {
+
+		List<Long> retVal = theResourceIds;
+
+		int listSize = theResourceIds.size();
+
+		if (listSize > 1 && listSize < 10) {
+			retVal = padIdListWithPlaceholders(theResourceIds, 10);
+		} else if (listSize > 10 && listSize < 50) {
+			retVal = padIdListWithPlaceholders(theResourceIds, 50);
+		} else if (listSize > 50 && listSize < 100) {
+			retVal = padIdListWithPlaceholders(theResourceIds, 100);
+		} else if (listSize > 100 && listSize < 200) {
+			retVal = padIdListWithPlaceholders(theResourceIds, 200);
+		} else if (listSize > 200 && listSize < 500) {
+			retVal = padIdListWithPlaceholders(theResourceIds, 500);
+		} else if (listSize > 500 && listSize < 800) {
+			retVal = padIdListWithPlaceholders(theResourceIds, 800);
+		}
+
+		return retVal;
+	}
+
+	private static List<Long> padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
+		List<Long> retVal = theIdList;
+
+		if (isUnmodifiableList(theIdList)) {
+			retVal = new ArrayList<>(preferredListSize);
+			retVal.addAll(theIdList);
+		}
+
+		while (retVal.size() < preferredListSize) {
+			retVal.add(-1L);
+		}
+
+		return retVal;
+	}
+
+	private static boolean isUnmodifiableList(List<Long> theList) {
+		try {
+			theList.addAll(Collections.emptyList());
+		} catch (Exception e) {
+			return true;
+		}
+		return false;
+	}
+
+	private InClauseNormalizer() {}
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPartitionProviderTest.java

@@ -1,76 +0,0 @@
-package ca.uhn.fhir.jpa.batch2;
-
-import ca.uhn.fhir.interceptor.model.RequestPartitionId;
-import ca.uhn.fhir.jpa.entity.PartitionEntity;
-import ca.uhn.fhir.jpa.partition.IPartitionLookupSvc;
-import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
-import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
-import ca.uhn.fhir.rest.server.provider.ProviderConstants;
-import org.assertj.core.api.Assertions;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-@ExtendWith(MockitoExtension.class)
-public class JpaJobPartitionProviderTest {
-	@Mock
-	private IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
-	@Mock
-	private IPartitionLookupSvc myPartitionLookupSvc;
-	@InjectMocks
-	private JpaJobPartitionProvider myJobPartitionProvider;
-
-	@Test
-	public void getPartitions_requestSpecificPartition_returnsPartition() {
-		// setup
-		SystemRequestDetails requestDetails = new SystemRequestDetails();
-		String operation = ProviderConstants.OPERATION_EXPORT;
-
-		RequestPartitionId partitionId = RequestPartitionId.fromPartitionId(1);
-		when(myRequestPartitionHelperSvc.determineReadPartitionForRequestForServerOperation(ArgumentMatchers.eq(requestDetails), ArgumentMatchers.eq(operation))).thenReturn(partitionId);
-
-		// test
-		List <RequestPartitionId> partitionIds = myJobPartitionProvider.getPartitions(requestDetails, operation);
-
-		// verify
-		Assertions.assertThat(partitionIds).hasSize(1);
-		Assertions.assertThat(partitionIds).containsExactlyInAnyOrder(partitionId);
-	}
-
-	@Test
-	public void getPartitions_requestAllPartitions_returnsListOfAllSpecificPartitions() {
-		// setup
-		SystemRequestDetails requestDetails = new SystemRequestDetails();
-		String operation = ProviderConstants.OPERATION_EXPORT;
-
-		when(myRequestPartitionHelperSvc.determineReadPartitionForRequestForServerOperation(ArgumentMatchers.eq(requestDetails), ArgumentMatchers.eq(operation)))
-				.thenReturn( RequestPartitionId.allPartitions());
-		List<RequestPartitionId> partitionIds = List.of(RequestPartitionId.fromPartitionIds(1), RequestPartitionId.fromPartitionIds(2));
-
-		List<PartitionEntity> partitionEntities = new ArrayList<>();
-		partitionIds.forEach(partitionId -> {
-			PartitionEntity entity = mock(PartitionEntity.class);
-			when(entity.toRequestPartitionId()).thenReturn(partitionId);
-			partitionEntities.add(entity);
-		});
-		when(myPartitionLookupSvc.listPartitions()).thenReturn(partitionEntities);
-		List<RequestPartitionId> expectedPartitionIds = new ArrayList<>(partitionIds);
-		expectedPartitionIds.add(RequestPartitionId.defaultPartition());
-
-		// test
-		List<RequestPartitionId> actualPartitionIds = myJobPartitionProvider.getPartitions(requestDetails, operation);
-
-		// verify
-		Assertions.assertThat(actualPartitionIds).hasSize(expectedPartitionIds.size());
-		Assertions.assertThat(actualPartitionIds).containsExactlyInAnyOrder(expectedPartitionIds.toArray(new RequestPartitionId[0]));
-	}
-}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/util/InClauseNormalizerTest.java

@@ -0,0 +1,72 @@
+package ca.uhn.fhir.util;
+
+import ca.uhn.fhir.jpa.util.InClauseNormalizer;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static java.util.Collections.nCopies;
+import static java.util.Collections.unmodifiableList;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class InClauseNormalizerTest {
+	private static final Long ourResourceId = 1L;
+	private static final Long ourPaddingValue = -1L;
+
+	@ParameterizedTest
+	@MethodSource("arguments")
+	public void testNormalizeUnmodifiableList_willCreateNewListAndPadToSize(int theInitialListSize, int theExpectedNormalizedListSize) {
+		List<Long> initialList = new ArrayList<>(nCopies(theInitialListSize, ourResourceId));
+		initialList = unmodifiableList(initialList);
+
+		List<Long> normalizedList = InClauseNormalizer.normalizeIdListForInClause(initialList);
+
+		assertNormalizedList(initialList, normalizedList, theInitialListSize, theExpectedNormalizedListSize);
+	}
+
+	@ParameterizedTest
+	@MethodSource("arguments")
+	public void testNormalizeListToSizeAndPad(int theInitialListSize, int theExpectedNormalizedListSize) {
+		List<Long> initialList = new ArrayList<>(nCopies(theInitialListSize, ourResourceId));
+
+		List<Long> normalizedList = InClauseNormalizer.normalizeIdListForInClause(initialList);
+
+		assertNormalizedList(initialList, normalizedList, theInitialListSize, theExpectedNormalizedListSize);
+	}
+
+	private void assertNormalizedList(List<Long> theInitialList, List<Long> theNormalizedList, int theInitialListSize, int theExpectedNormalizedListSize) {
+		List<Long> expectedPaddedSubList = new ArrayList<>(nCopies(theExpectedNormalizedListSize - theInitialListSize, ourPaddingValue));
+
+		assertThat(theNormalizedList).startsWith(listToArray(theInitialList));
+		assertThat(theNormalizedList).hasSize(theExpectedNormalizedListSize);
+		assertThat(theNormalizedList).endsWith(listToArray(expectedPaddedSubList));
+	}
+
+	static Long[] listToArray(List<Long> theList) {
+		return theList.toArray(new Long[0]);
+	}
+
+	private static Stream<Arguments> arguments(){
+		return Stream.of(
+			Arguments.of(0, 0),
+			Arguments.of(1, 1),
+			Arguments.of(2, 10),
+			Arguments.of(10, 10),
+			Arguments.of(12, 50),
+			Arguments.of(50, 50),
+			Arguments.of(51, 100),
+			Arguments.of(100, 100),
+			Arguments.of(150, 200),
+			Arguments.of(300, 500),
+			Arguments.of(500, 500),
+			Arguments.of(700, 800),
+			Arguments.of(800, 800),
+			Arguments.of(801, 801)
+		);
+	}
+
+}

hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/config/ElasticsearchWithPrefixConfig.java

@@ -13,6 +13,7 @@ import ca.uhn.fhir.jpa.test.config.BlockLargeNumbersOfParamsListener;
 import ca.uhn.fhir.jpa.test.config.TestHSearchAddInConfig;
 import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch.indices.IndexSettings;
 import co.elastic.clients.elasticsearch.indices.PutTemplateResponse;
 import co.elastic.clients.json.JsonData;
 import net.ttddyy.dsproxy.listener.logging.SLF4JLogLevel;
@@ -129,7 +130,7 @@ public class ElasticsearchWithPrefixConfig {
 				.putTemplate(b -> b
 					.name("ngram-template")
 					.indexPatterns("*resourcetable-*", "*termconcept-*")
-					.settings(Map.of("index.max_ngram_diff", JsonData.of(50))));
+					.settings(new IndexSettings.Builder().maxNgramDiff(50).build()));
 			assert acknowledgedResponse.acknowledged();
 		} catch (IOException theE) {
 			theE.printStackTrace();

hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java

@@ -56,8 +56,6 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN {
 	@Mock
 	private IHSearchEventListener mySearchEventListener;
 
-
-
 	@Test
 	public void testLastNChunking() {
 
@@ -108,7 +106,6 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN {
 		secondQueryPattern.append("\\).*");
 		assertThat(queries.get(1).toUpperCase().replaceAll(" , ", ",")).matches(secondQueryPattern.toString());
 		assertThat(queries.get(3).toUpperCase().replaceAll(" , ", ",")).matches(secondQueryPattern.toString());
-
 	}
 
 	@Test

hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java

@@ -23,6 +23,7 @@ import ca.uhn.fhir.jpa.model.dao.JpaPid;
 import ca.uhn.fhir.jpa.model.entity.NormalizedQuantitySearchLevel;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
+import ca.uhn.fhir.jpa.rp.r4.PatientResourceProvider;
 import ca.uhn.fhir.jpa.search.BaseSourceSearchParameterTestCases;
 import ca.uhn.fhir.jpa.search.CompositeSearchParameterTestCases;
 import ca.uhn.fhir.jpa.search.QuantitySearchParameterTestCases;
@@ -73,6 +74,7 @@ import org.hl7.fhir.r4.model.DateTimeType;
 import org.hl7.fhir.r4.model.DecimalType;
 import org.hl7.fhir.r4.model.DiagnosticReport;
 import org.hl7.fhir.r4.model.Encounter;
+import org.hl7.fhir.r4.model.IdType;
 import org.hl7.fhir.r4.model.Identifier;
 import org.hl7.fhir.r4.model.Meta;
 import org.hl7.fhir.r4.model.Narrative;
@@ -101,6 +103,7 @@ import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.mock.web.MockHttpServletRequest;
 import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.ContextHierarchy;
@@ -118,6 +121,7 @@ import java.io.IOException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
@@ -126,11 +130,13 @@ import java.util.stream.Collectors;
 
 import static ca.uhn.fhir.jpa.model.util.UcumServiceUtil.UCUM_CODESYSTEM_URL;
 import static ca.uhn.fhir.rest.api.Constants.CHARSET_UTF8;
+import static ca.uhn.fhir.rest.api.Constants.HEADER_CACHE_CONTROL;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
 
 @ExtendWith(SpringExtension.class)
 @ExtendWith(MockitoExtension.class)
@@ -229,6 +235,8 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
 	@Mock
 	private IHSearchEventListener mySearchEventListener;
 	@Autowired
+	private PatientResourceProvider myPatientRpR4;
+	@Autowired
 	private ElasticsearchSvcImpl myElasticsearchSvc;
 
 
@@ -954,6 +962,24 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
 		myStorageSettings.setStoreResourceInHSearchIndex(defaultConfig.isStoreResourceInHSearchIndex());
 	}
 
+	@Test
+	public void testEverythingType() {
+		Patient p = new Patient();
+		p.setId("my-patient");
+		myPatientDao.update(p);
+		IBundleProvider iBundleProvider = myPatientRpR4.patientTypeEverything(new MockHttpServletRequest(), null, null, null, null, null, null, null, null, null, null, mySrd);
+		assertEquals(iBundleProvider.getAllResources().size(),  1);
+	}
+
+	@Test
+	public void testEverythingInstance() {
+		Patient p = new Patient();
+		p.setId("my-patient");
+		myPatientDao.update(p);
+		IBundleProvider iBundleProvider = myPatientRpR4.patientInstanceEverything(new MockHttpServletRequest(), new IdType("Patient/my-patient"), null, null, null, null, null, null, null, null, null, mySrd);
+		assertEquals(iBundleProvider.getAllResources().size(),  1);
+	}
+
 	@Test
 	public void testExpandWithIsAInExternalValueSet() {
 		createExternalCsAndLocalVs();

hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java

@@ -20,6 +20,7 @@
 package ca.uhn.fhir.jpa.mdm.svc;
 
 import ca.uhn.fhir.batch2.api.IJobCoordinator;
+import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrl;
 import ca.uhn.fhir.batch2.model.JobInstanceStartRequest;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.interceptor.api.HookParams;
@@ -360,9 +361,9 @@ public class MdmControllerSvcImpl implements IMdmControllerSvc {
 		if (hasBatchSize) {
 			params.setBatchSize(theBatchSize.getValue().intValue());
 		}
-		params.setRequestPartitionId(RequestPartitionId.allPartitions());
-
-		theUrls.forEach(params::addUrl);
+		RequestPartitionId partitionId = RequestPartitionId.allPartitions();
+		theUrls.forEach(
+				url -> params.addPartitionedUrl(new PartitionedUrl().setUrl(url).setRequestPartitionId(partitionId)));
 
 		JobInstanceStartRequest request = new JobInstanceStartRequest();
 		request.setParameters(params);

hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderBatchR4Test.java

@@ -1,8 +1,10 @@
 package ca.uhn.fhir.jpa.mdm.provider;
 
 import ca.uhn.fhir.i18n.Msg;
+import ca.uhn.fhir.interceptor.api.Hook;
 import ca.uhn.fhir.interceptor.api.IInterceptorService;
 import ca.uhn.fhir.interceptor.api.Pointcut;
+import ca.uhn.fhir.jpa.subscription.model.ResourceModifiedJsonMessage;
 import ca.uhn.fhir.mdm.log.Logs;
 import ca.uhn.fhir.mdm.rules.config.MdmSettings;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
@@ -30,6 +32,7 @@ import org.junit.jupiter.params.provider.MethodSource;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Stream;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -245,16 +248,24 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test {
 		Patient janePatient = createPatientAndUpdateLinks(buildJanePatient());
 		Patient janePatient2 = createPatientAndUpdateLinks(buildJanePatient());
 		assertLinkCount(5);
-
+		final AtomicBoolean mdmSubmitBeforeMessageDeliveryHookCalled = new AtomicBoolean();
+		final Object interceptor =  new Object() {
+			@Hook(Pointcut.MDM_SUBMIT_PRE_MESSAGE_DELIVERY)
+			void hookMethod(ResourceModifiedJsonMessage theResourceModifiedJsonMessage) {
+				mdmSubmitBeforeMessageDeliveryHookCalled.set(true);
+			}
+		};
+		myInterceptorService.registerInterceptor(interceptor);
 		// When
 		clearMdmLinks();
 		afterMdmLatch.runWithExpectedCount(3, () -> {
 			myMdmProvider.mdmBatchPatientType(null , null, theSyncOrAsyncRequest);
 		});
-
 		// Then
+		assertThat(mdmSubmitBeforeMessageDeliveryHookCalled).isTrue();
 		updatePatientAndUpdateLinks(janePatient);
 		updatePatientAndUpdateLinks(janePatient2);
 		assertLinkCount(3);
+		myInterceptorService.unregisterInterceptor(interceptor);
 	}
 }

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/config/BaseSubscriptionSettings.java

@@ -27,9 +27,13 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
 public abstract class BaseSubscriptionSettings {
 	public static final String DEFAULT_EMAIL_FROM_ADDRESS = "noreply@unknown.com";
 	public static final String DEFAULT_WEBSOCKET_CONTEXT_PATH = "/websocket";
+	public static final String DEFAULT_RESTHOOK_ENDPOINTURL_VALIDATION_REGEX =
+			"((((http?|https?)://))([-%()_.!~*';/?:@&=+$,A-Za-z0-9])+)";
 
 	private final Set<Subscription.SubscriptionChannelType> mySupportedSubscriptionTypes = new HashSet<>();
 	private String myEmailFromAddress = DEFAULT_EMAIL_FROM_ADDRESS;
@@ -45,6 +49,13 @@ public abstract class BaseSubscriptionSettings {
 	 */
 	private boolean myAllowOnlyInMemorySubscriptions = false;
 
+	/**
+	 * @since 7.6.0
+	 *
+	 * Regex To perform validation on the endpoint URL for Subscription of type RESTHOOK.
+	 */
+	private String myRestHookEndpointUrlValidationRegex = DEFAULT_RESTHOOK_ENDPOINTURL_VALIDATION_REGEX;
+
 	/**
 	 * This setting indicates which subscription channel types are supported by the server.  Any subscriptions submitted
 	 * to the server matching these types will be activated.
@@ -235,4 +246,32 @@ public abstract class BaseSubscriptionSettings {
 	public void setTriggerSubscriptionsForNonVersioningChanges(boolean theTriggerSubscriptionsForNonVersioningChanges) {
 		myTriggerSubscriptionsForNonVersioningChanges = theTriggerSubscriptionsForNonVersioningChanges;
 	}
+
+	/**
+	 * Provides the regex expression to perform endpoint URL validation If rest-hook subscriptions are supported.
+	 * Default value is {@link #DEFAULT_RESTHOOK_ENDPOINTURL_VALIDATION_REGEX}.
+	 * @since 7.6.0
+	 */
+	public String getRestHookEndpointUrlValidationRegex() {
+		return myRestHookEndpointUrlValidationRegex;
+	}
+
+	/**
+	 * Configure the regex expression that will be used to validate the endpoint URL.
+	 * Set to NULL or EMPTY for no endpoint URL validation.
+	 *
+	 * @since 7.6.0
+	 */
+	public void setRestHookEndpointUrlValidationRegex(String theRestHookEndpointUrlValidationgRegex) {
+		myRestHookEndpointUrlValidationRegex = theRestHookEndpointUrlValidationgRegex;
+	}
+
+	/**
+	 * Whether an endpoint validation Regex was set for URL validation.
+	 *
+	 * @since 7.6.0
+	 */
+	public boolean hasRestHookEndpointUrlValidationRegex() {
+		return isNotBlank(myRestHookEndpointUrlValidationRegex);
+	}
 }

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/dao/JpaPid.java

@@ -19,6 +19,7 @@
  */
 package ca.uhn.fhir.jpa.model.dao;
 
+import ca.uhn.fhir.jpa.model.entity.PartitionablePartitionId;
 import ca.uhn.fhir.rest.api.server.storage.BaseResourcePersistentId;
 
 import java.util.ArrayList;
@@ -34,6 +35,7 @@ import java.util.Set;
  */
 public class JpaPid extends BaseResourcePersistentId<Long> {
 	private final Long myId;
+	private PartitionablePartitionId myPartitionablePartitionId;
 
 	private JpaPid(Long theId) {
 		super(null);
@@ -55,6 +57,15 @@ public class JpaPid extends BaseResourcePersistentId<Long> {
 		myId = theId;
 	}
 
+	public PartitionablePartitionId getPartitionablePartitionId() {
+		return myPartitionablePartitionId;
+	}
+
+	public JpaPid setPartitionablePartitionId(PartitionablePartitionId thePartitionablePartitionId) {
+		myPartitionablePartitionId = thePartitionablePartitionId;
+		return this;
+	}
+
 	public static List<Long> toLongList(Collection<JpaPid> thePids) {
 		List<Long> retVal = new ArrayList<>(thePids.size());
 		for (JpaPid next : thePids) {

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/BasePartitionable.java

@@ -25,6 +25,7 @@ import jakarta.persistence.Embedded;
 import jakarta.persistence.MappedSuperclass;
 
 import java.io.Serializable;
+import java.time.LocalDate;
 
 /**
  * This is the base class for entities with partitioning that does NOT include Hibernate Envers logging.
@@ -44,6 +45,13 @@ public abstract class BasePartitionable implements Serializable {
 	@Column(name = PartitionablePartitionId.PARTITION_ID, insertable = false, updatable = false, nullable = true)
 	private Integer myPartitionIdValue;
 
+	/**
+	 * This is here to support queries only, do not set this field directly
+	 */
+	@SuppressWarnings("unused")
+	@Column(name = PartitionablePartitionId.PARTITION_DATE, insertable = false, updatable = false, nullable = true)
+	private LocalDate myPartitionDateValue;
+
 	@Nullable
 	public PartitionablePartitionId getPartitionId() {
 		return myPartitionId;
@@ -57,6 +65,7 @@ public abstract class BasePartitionable implements Serializable {
 	public String toString() {
 		return "BasePartitionable{" + "myPartitionId="
 				+ myPartitionId + ", myPartitionIdValue="
-				+ myPartitionIdValue + '}';
+				+ myPartitionIdValue + ", myPartitionDateValue="
+				+ myPartitionDateValue + '}';
 	}
 }

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/PartitionablePartitionId.java

@@ -34,6 +34,7 @@ import java.time.LocalDate;
 public class PartitionablePartitionId implements Cloneable {
 
 	static final String PARTITION_ID = "PARTITION_ID";
+	static final String PARTITION_DATE = "PARTITION_DATE";
 
 	@Column(name = PARTITION_ID, nullable = true, insertable = true, updatable = false)
 	private Integer myPartitionId;
@@ -132,4 +133,9 @@ public class PartitionablePartitionId implements Cloneable {
 		}
 		return new PartitionablePartitionId(partitionId, theRequestPartitionId.getPartitionDate());
 	}
+
+	public static PartitionablePartitionId with(
+			@Nullable Integer thePartitionId, @Nullable LocalDate thePartitionDate) {
+		return new PartitionablePartitionId(thePartitionId, thePartitionDate);
+	}
 }

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceLink.java

@@ -19,8 +19,9 @@
  */
 package ca.uhn.fhir.jpa.model.entity;
 
-import jakarta.annotation.Nullable;
+import jakarta.persistence.AttributeOverride;
 import jakarta.persistence.Column;
+import jakarta.persistence.Embedded;
 import jakarta.persistence.Entity;
 import jakarta.persistence.FetchType;
 import jakarta.persistence.ForeignKey;
@@ -119,6 +120,11 @@ public class ResourceLink extends BaseResourceIndex {
 	@Transient
 	private transient String myTargetResourceId;
 
+	@Embedded
+	@AttributeOverride(name = "myPartitionId", column = @Column(name = "TARGET_RES_PARTITION_ID"))
+	@AttributeOverride(name = "myPartitionDate", column = @Column(name = "TARGET_RES_PARTITION_DATE"))
+	private PartitionablePartitionId myTargetResourcePartitionId;
+
 	/**
 	 * Constructor
 	 */
@@ -188,6 +194,7 @@ public class ResourceLink extends BaseResourceIndex {
 		myTargetResourceType = source.getTargetResourceType();
 		myTargetResourceVersion = source.getTargetResourceVersion();
 		myTargetResourceUrl = source.getTargetResourceUrl();
+		myTargetResourcePartitionId = source.getTargetResourcePartitionId();
 	}
 
 	public String getSourcePath() {
@@ -270,6 +277,15 @@ public class ResourceLink extends BaseResourceIndex {
 		myId = theId;
 	}
 
+	public PartitionablePartitionId getTargetResourcePartitionId() {
+		return myTargetResourcePartitionId;
+	}
+
+	public ResourceLink setTargetResourcePartitionId(PartitionablePartitionId theTargetResourcePartitionId) {
+		myTargetResourcePartitionId = theTargetResourcePartitionId;
+		return this;
+	}
+
 	@Override
 	public void clearHashes() {
 		// nothing right now
@@ -363,23 +379,113 @@ public class ResourceLink extends BaseResourceIndex {
 		return retVal;
 	}
 
-	/**
-	 * @param theTargetResourceVersion This should only be populated if the reference actually had a version
-	 */
 	public static ResourceLink forLocalReference(
-			String theSourcePath,
-			ResourceTable theSourceResource,
-			String theTargetResourceType,
-			Long theTargetResourcePid,
-			String theTargetResourceId,
-			Date theUpdated,
-			@Nullable Long theTargetResourceVersion) {
+			ResourceLinkForLocalReferenceParams theResourceLinkForLocalReferenceParams) {
+
 		ResourceLink retVal = new ResourceLink();
-		retVal.setSourcePath(theSourcePath);
-		retVal.setSourceResource(theSourceResource);
-		retVal.setTargetResource(theTargetResourceType, theTargetResourcePid, theTargetResourceId);
-		retVal.setTargetResourceVersion(theTargetResourceVersion);
-		retVal.setUpdated(theUpdated);
+		retVal.setSourcePath(theResourceLinkForLocalReferenceParams.getSourcePath());
+		retVal.setSourceResource(theResourceLinkForLocalReferenceParams.getSourceResource());
+		retVal.setTargetResource(
+				theResourceLinkForLocalReferenceParams.getTargetResourceType(),
+				theResourceLinkForLocalReferenceParams.getTargetResourcePid(),
+				theResourceLinkForLocalReferenceParams.getTargetResourceId());
+
+		retVal.setTargetResourcePartitionId(
+				theResourceLinkForLocalReferenceParams.getTargetResourcePartitionablePartitionId());
+		retVal.setTargetResourceVersion(theResourceLinkForLocalReferenceParams.getTargetResourceVersion());
+		retVal.setUpdated(theResourceLinkForLocalReferenceParams.getUpdated());
+
 		return retVal;
 	}
+
+	public static class ResourceLinkForLocalReferenceParams {
+		private String mySourcePath;
+		private ResourceTable mySourceResource;
+		private String myTargetResourceType;
+		private Long myTargetResourcePid;
+		private String myTargetResourceId;
+		private Date myUpdated;
+		private Long myTargetResourceVersion;
+		private PartitionablePartitionId myTargetResourcePartitionablePartitionId;
+
+		public static ResourceLinkForLocalReferenceParams instance() {
+			return new ResourceLinkForLocalReferenceParams();
+		}
+
+		public String getSourcePath() {
+			return mySourcePath;
+		}
+
+		public ResourceLinkForLocalReferenceParams setSourcePath(String theSourcePath) {
+			mySourcePath = theSourcePath;
+			return this;
+		}
+
+		public ResourceTable getSourceResource() {
+			return mySourceResource;
+		}
+
+		public ResourceLinkForLocalReferenceParams setSourceResource(ResourceTable theSourceResource) {
+			mySourceResource = theSourceResource;
+			return this;
+		}
+
+		public String getTargetResourceType() {
+			return myTargetResourceType;
+		}
+
+		public ResourceLinkForLocalReferenceParams setTargetResourceType(String theTargetResourceType) {
+			myTargetResourceType = theTargetResourceType;
+			return this;
+		}
+
+		public Long getTargetResourcePid() {
+			return myTargetResourcePid;
+		}
+
+		public ResourceLinkForLocalReferenceParams setTargetResourcePid(Long theTargetResourcePid) {
+			myTargetResourcePid = theTargetResourcePid;
+			return this;
+		}
+
+		public String getTargetResourceId() {
+			return myTargetResourceId;
+		}
+
+		public ResourceLinkForLocalReferenceParams setTargetResourceId(String theTargetResourceId) {
+			myTargetResourceId = theTargetResourceId;
+			return this;
+		}
+
+		public Date getUpdated() {
+			return myUpdated;
+		}
+
+		public ResourceLinkForLocalReferenceParams setUpdated(Date theUpdated) {
+			myUpdated = theUpdated;
+			return this;
+		}
+
+		public Long getTargetResourceVersion() {
+			return myTargetResourceVersion;
+		}
+
+		/**
+		 * @param theTargetResourceVersion This should only be populated if the reference actually had a version
+		 */
+		public ResourceLinkForLocalReferenceParams setTargetResourceVersion(Long theTargetResourceVersion) {
+			myTargetResourceVersion = theTargetResourceVersion;
+			return this;
+		}
+
+		public PartitionablePartitionId getTargetResourcePartitionablePartitionId() {
+			return myTargetResourcePartitionablePartitionId;
+		}
+
+		public ResourceLinkForLocalReferenceParams setTargetResourcePartitionablePartitionId(
+				PartitionablePartitionId theTargetResourcePartitionablePartitionId) {
+			myTargetResourcePartitionablePartitionId = theTargetResourcePartitionablePartitionId;
+			return this;
+		}
+	}
 }

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/interceptor/model/ReadPartitionIdRequestDetails.java

@@ -113,6 +113,24 @@ public class ReadPartitionIdRequestDetails extends PartitionIdRequestDetails {
 				null, RestOperationTypeEnum.EXTENDED_OPERATION_SERVER, null, null, null, null, theOperationName);
 	}
 
+	/**
+	 * @since 7.4.0
+	 */
+	public static ReadPartitionIdRequestDetails forDelete(@Nonnull String theResourceType, @Nonnull IIdType theId) {
+		RestOperationTypeEnum op = RestOperationTypeEnum.DELETE;
+		return new ReadPartitionIdRequestDetails(
+				theResourceType, op, theId.withResourceType(theResourceType), null, null, null, null);
+	}
+
+	/**
+	 * @since 7.4.0
+	 */
+	public static ReadPartitionIdRequestDetails forPatch(String theResourceType, IIdType theId) {
+		RestOperationTypeEnum op = RestOperationTypeEnum.PATCH;
+		return new ReadPartitionIdRequestDetails(
+				theResourceType, op, theId.withResourceType(theResourceType), null, null, null, null);
+	}
+
 	public static ReadPartitionIdRequestDetails forRead(
 			String theResourceType, @Nonnull IIdType theId, boolean theIsVread) {
 		RestOperationTypeEnum op = theIsVread ? RestOperationTypeEnum.VREAD : RestOperationTypeEnum.READ;

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/MatchUrlService.java

@@ -95,7 +95,7 @@ public class MatchUrlService {
 			}
 
 			if (Constants.PARAM_LASTUPDATED.equals(nextParamName)) {
-				if (paramList != null && paramList.size() > 0) {
+				if (!paramList.isEmpty()) {
 					if (paramList.size() > 2) {
 						throw new InvalidRequestException(Msg.code(484) + "Failed to parse match URL[" + theMatchUrl
 								+ "] - Can not have more than 2 " + Constants.PARAM_LASTUPDATED
@@ -111,9 +111,7 @@ public class MatchUrlService {
 						myFhirContext, RestSearchParameterTypeEnum.HAS, nextParamName, paramList);
 				paramMap.add(nextParamName, param);
 			} else if (Constants.PARAM_COUNT.equals(nextParamName)) {
-				if (paramList != null
-						&& paramList.size() > 0
-						&& paramList.get(0).size() > 0) {
+				if (!paramList.isEmpty() && !paramList.get(0).isEmpty()) {
 					String intString = paramList.get(0).get(0);
 					try {
 						paramMap.setCount(Integer.parseInt(intString));
@@ -123,16 +121,14 @@ public class MatchUrlService {
 					}
 				}
 			} else if (Constants.PARAM_SEARCH_TOTAL_MODE.equals(nextParamName)) {
-				if (paramList != null
-						&& !paramList.isEmpty()
-						&& !paramList.get(0).isEmpty()) {
+				if (!paramList.isEmpty() && !paramList.get(0).isEmpty()) {
 					String totalModeEnumStr = paramList.get(0).get(0);
 					SearchTotalModeEnum searchTotalMode = SearchTotalModeEnum.fromCode(totalModeEnumStr);
 					if (searchTotalMode == null) {
 						// We had an oops here supporting the UPPER CASE enum instead of the FHIR code for _total.
 						// Keep supporting it in case someone is using it.
 						try {
-							paramMap.setSearchTotalMode(SearchTotalModeEnum.valueOf(totalModeEnumStr));
+							searchTotalMode = SearchTotalModeEnum.valueOf(totalModeEnumStr);
 						} catch (IllegalArgumentException e) {
 							throw new InvalidRequestException(Msg.code(2078) + "Invalid "
 									+ Constants.PARAM_SEARCH_TOTAL_MODE + " value: " + totalModeEnumStr);
@@ -141,9 +137,7 @@ public class MatchUrlService {
 					paramMap.setSearchTotalMode(searchTotalMode);
 				}
 			} else if (Constants.PARAM_OFFSET.equals(nextParamName)) {
-				if (paramList != null
-						&& paramList.size() > 0
-						&& paramList.get(0).size() > 0) {
+				if (!paramList.isEmpty() && !paramList.get(0).isEmpty()) {
 					String intString = paramList.get(0).get(0);
 					try {
 						paramMap.setOffset(Integer.parseInt(intString));
@@ -238,40 +232,27 @@ public class MatchUrlService {
 		return getResourceSearch(theUrl, null);
 	}
 
-	public abstract static class Flag {
-
-		/**
-		 * Constructor
-		 */
-		Flag() {
-			// nothing
-		}
-
-		abstract void process(
-				String theParamName, List<QualifiedParamList> theValues, SearchParameterMap theMapToPopulate);
+	public interface Flag {
+		void process(String theParamName, List<QualifiedParamList> theValues, SearchParameterMap theMapToPopulate);
 	}
 
 	/**
 	 * Indicates that the parser should process _include and _revinclude (by default these are not handled)
 	 */
 	public static Flag processIncludes() {
-		return new Flag() {
-
-			@Override
-			void process(String theParamName, List<QualifiedParamList> theValues, SearchParameterMap theMapToPopulate) {
-				if (Constants.PARAM_INCLUDE.equals(theParamName)) {
-					for (QualifiedParamList nextQualifiedList : theValues) {
-						for (String nextValue : nextQualifiedList) {
-							theMapToPopulate.addInclude(new Include(
-									nextValue, ParameterUtil.isIncludeIterate(nextQualifiedList.getQualifier())));
-						}
+		return (theParamName, theValues, theMapToPopulate) -> {
+			if (Constants.PARAM_INCLUDE.equals(theParamName)) {
+				for (QualifiedParamList nextQualifiedList : theValues) {
+					for (String nextValue : nextQualifiedList) {
+						theMapToPopulate.addInclude(new Include(
+								nextValue, ParameterUtil.isIncludeIterate(nextQualifiedList.getQualifier())));
 					}
-				} else if (Constants.PARAM_REVINCLUDE.equals(theParamName)) {
-					for (QualifiedParamList nextQualifiedList : theValues) {
-						for (String nextValue : nextQualifiedList) {
-							theMapToPopulate.addRevInclude(new Include(
-									nextValue, ParameterUtil.isIncludeIterate(nextQualifiedList.getQualifier())));
-						}
+				}
+			} else if (Constants.PARAM_REVINCLUDE.equals(theParamName)) {
+				for (QualifiedParamList nextQualifiedList : theValues) {
+					for (String nextValue : nextQualifiedList) {
+						theMapToPopulate.addRevInclude(new Include(
+								nextValue, ParameterUtil.isIncludeIterate(nextQualifiedList.getQualifier())));
 					}
 				}
 			}

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/SearchParamExtractorService.java

@@ -37,6 +37,7 @@ import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboTokenNonUnique;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString;
 import ca.uhn.fhir.jpa.model.entity.ResourceLink;
+import ca.uhn.fhir.jpa.model.entity.ResourceLink.ResourceLinkForLocalReferenceParams;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.model.entity.SearchParamPresentEntity;
 import ca.uhn.fhir.jpa.model.entity.StorageSettings;
@@ -71,6 +72,8 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static ca.uhn.fhir.jpa.model.config.PartitionSettings.CrossPartitionReferenceMode.ALLOWED_UNQUALIFIED;
+import static ca.uhn.fhir.jpa.model.entity.ResourceLink.forLocalReference;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
@@ -105,26 +108,6 @@ public class SearchParamExtractorService {
 		mySearchParamExtractor = theSearchParamExtractor;
 	}
 
-	public void extractFromResource(
-			RequestPartitionId theRequestPartitionId,
-			RequestDetails theRequestDetails,
-			ResourceIndexedSearchParams theParams,
-			ResourceTable theEntity,
-			IBaseResource theResource,
-			TransactionDetails theTransactionDetails,
-			boolean theFailOnInvalidReference) {
-		extractFromResource(
-				theRequestPartitionId,
-				theRequestDetails,
-				theParams,
-				ResourceIndexedSearchParams.withSets(),
-				theEntity,
-				theResource,
-				theTransactionDetails,
-				theFailOnInvalidReference,
-				ISearchParamExtractor.ALL_PARAMS);
-	}
-
 	/**
 	 * This method is responsible for scanning a resource for all of the search parameter instances.
 	 * I.e. for all search parameters defined for
@@ -702,14 +685,18 @@ public class SearchParamExtractorService {
 			 * need to resolve it again
 			 */
 			myResourceLinkResolver.validateTypeOrThrowException(type);
-			resourceLink = ResourceLink.forLocalReference(
-					thePathAndRef.getPath(),
-					theEntity,
-					typeString,
-					resolvedTargetId.getId(),
-					targetId,
-					transactionDate,
-					targetVersionId);
+
+			ResourceLinkForLocalReferenceParams params = ResourceLinkForLocalReferenceParams.instance()
+					.setSourcePath(thePathAndRef.getPath())
+					.setSourceResource(theEntity)
+					.setTargetResourceType(typeString)
+					.setTargetResourcePid(resolvedTargetId.getId())
+					.setTargetResourceId(targetId)
+					.setUpdated(transactionDate)
+					.setTargetResourceVersion(targetVersionId)
+					.setTargetResourcePartitionablePartitionId(resolvedTargetId.getPartitionablePartitionId());
+
+			resourceLink = forLocalReference(params);
 
 		} else if (theFailOnInvalidReference) {
 
@@ -748,6 +735,7 @@ public class SearchParamExtractorService {
 			} else {
 				// Cache the outcome in the current transaction in case there are more references
 				JpaPid persistentId = JpaPid.fromId(resourceLink.getTargetResourcePid());
+				persistentId.setPartitionablePartitionId(resourceLink.getTargetResourcePartitionId());
 				theTransactionDetails.addResolvedResourceId(referenceElement, persistentId);
 			}
 
@@ -757,11 +745,15 @@ public class SearchParamExtractorService {
 			 * Just assume the reference is valid. This is used for in-memory matching since there
 			 * is no expectation of a database in this situation
 			 */
-			ResourceTable target;
-			target = new ResourceTable();
-			target.setResourceType(typeString);
-			resourceLink = ResourceLink.forLocalReference(
-					thePathAndRef.getPath(), theEntity, typeString, null, targetId, transactionDate, targetVersionId);
+			ResourceLinkForLocalReferenceParams params = ResourceLinkForLocalReferenceParams.instance()
+					.setSourcePath(thePathAndRef.getPath())
+					.setSourceResource(theEntity)
+					.setTargetResourceType(typeString)
+					.setTargetResourceId(targetId)
+					.setUpdated(transactionDate)
+					.setTargetResourceVersion(targetVersionId);
+
+			resourceLink = forLocalReference(params);
 		}
 
 		theNewParams.myLinks.add(resourceLink);
@@ -912,19 +904,24 @@ public class SearchParamExtractorService {
 			RequestDetails theRequest,
 			TransactionDetails theTransactionDetails) {
 		JpaPid resolvedResourceId = (JpaPid) theTransactionDetails.getResolvedResourceId(theNextId);
+
 		if (resolvedResourceId != null) {
 			String targetResourceType = theNextId.getResourceType();
 			Long targetResourcePid = resolvedResourceId.getId();
 			String targetResourceIdPart = theNextId.getIdPart();
 			Long targetVersion = theNextId.getVersionIdPartAsLong();
-			return ResourceLink.forLocalReference(
-					thePathAndRef.getPath(),
-					theEntity,
-					targetResourceType,
-					targetResourcePid,
-					targetResourceIdPart,
-					theUpdateTime,
-					targetVersion);
+
+			ResourceLinkForLocalReferenceParams params = ResourceLinkForLocalReferenceParams.instance()
+					.setSourcePath(thePathAndRef.getPath())
+					.setSourceResource(theEntity)
+					.setTargetResourceType(targetResourceType)
+					.setTargetResourcePid(targetResourcePid)
+					.setTargetResourceId(targetResourceIdPart)
+					.setUpdated(theUpdateTime)
+					.setTargetResourceVersion(targetVersion)
+					.setTargetResourcePartitionablePartitionId(resolvedResourceId.getPartitionablePartitionId());
+
+			return ResourceLink.forLocalReference(params);
 		}
 
 		/*
@@ -936,8 +933,7 @@ public class SearchParamExtractorService {
 
 		IResourceLookup<JpaPid> targetResource;
 		if (myPartitionSettings.isPartitioningEnabled()) {
-			if (myPartitionSettings.getAllowReferencesAcrossPartitions()
-					== PartitionSettings.CrossPartitionReferenceMode.ALLOWED_UNQUALIFIED) {
+			if (myPartitionSettings.getAllowReferencesAcrossPartitions() == ALLOWED_UNQUALIFIED) {
 
 				// Interceptor: Pointcut.JPA_CROSS_PARTITION_REFERENCE_DETECTED
 				if (CompositeInterceptorBroadcaster.hasHooks(
@@ -981,21 +977,25 @@ public class SearchParamExtractorService {
 		Long targetResourcePid = targetResource.getPersistentId().getId();
 		String targetResourceIdPart = theNextId.getIdPart();
 		Long targetVersion = theNextId.getVersionIdPartAsLong();
-		return ResourceLink.forLocalReference(
-				thePathAndRef.getPath(),
-				theEntity,
-				targetResourceType,
-				targetResourcePid,
-				targetResourceIdPart,
-				theUpdateTime,
-				targetVersion);
+
+		ResourceLinkForLocalReferenceParams params = ResourceLinkForLocalReferenceParams.instance()
+				.setSourcePath(thePathAndRef.getPath())
+				.setSourceResource(theEntity)
+				.setTargetResourceType(targetResourceType)
+				.setTargetResourcePid(targetResourcePid)
+				.setTargetResourceId(targetResourceIdPart)
+				.setUpdated(theUpdateTime)
+				.setTargetResourceVersion(targetVersion)
+				.setTargetResourcePartitionablePartitionId(
+						targetResource.getPersistentId().getPartitionablePartitionId());
+
+		return forLocalReference(params);
 	}
 
 	private RequestPartitionId determineResolverPartitionId(@Nonnull RequestPartitionId theRequestPartitionId) {
 		RequestPartitionId targetRequestPartitionId = theRequestPartitionId;
 		if (myPartitionSettings.isPartitioningEnabled()
-				&& myPartitionSettings.getAllowReferencesAcrossPartitions()
-						== PartitionSettings.CrossPartitionReferenceMode.ALLOWED_UNQUALIFIED) {
+				&& myPartitionSettings.getAllowReferencesAcrossPartitions() == ALLOWED_UNQUALIFIED) {
 			targetRequestPartitionId = RequestPartitionId.allPartitions();
 		}
 		return targetRequestPartitionId;

hapi-fhir-jpaserver-searchparam/src/test/java/ca/uhn/fhir/jpa/searchparam/extractor/ResourceIndexedSearchParamsTest.java

@@ -37,7 +37,7 @@ public class ResourceIndexedSearchParamsTest {
 
 	@Test
 	public void matchResourceLinksStringCompareToLong() {
-		ResourceLink link = ResourceLink.forLocalReference("organization", mySource, "Organization", 123L, LONG_ID, new Date(), null);
+		ResourceLink link = getResourceLinkForLocalReference(LONG_ID);
 		myParams.getResourceLinks().add(link);
 
 		ReferenceParam referenceParam = getReferenceParam(STRING_ID);
@@ -47,7 +47,7 @@ public class ResourceIndexedSearchParamsTest {
 
 	@Test
 	public void matchResourceLinksStringCompareToString() {
-		ResourceLink link = ResourceLink.forLocalReference("organization", mySource, "Organization", 123L, STRING_ID, new Date(), null);
+		ResourceLink link = getResourceLinkForLocalReference(STRING_ID);
 		myParams.getResourceLinks().add(link);
 
 		ReferenceParam referenceParam = getReferenceParam(STRING_ID);
@@ -57,7 +57,7 @@ public class ResourceIndexedSearchParamsTest {
 
 	@Test
 	public void matchResourceLinksLongCompareToString() {
-		ResourceLink link = ResourceLink.forLocalReference("organization", mySource, "Organization", 123L, STRING_ID, new Date(), null);
+		ResourceLink link = getResourceLinkForLocalReference(STRING_ID);
 		myParams.getResourceLinks().add(link);
 
 		ReferenceParam referenceParam = getReferenceParam(LONG_ID);
@@ -67,7 +67,7 @@ public class ResourceIndexedSearchParamsTest {
 
 	@Test
 	public void matchResourceLinksLongCompareToLong() {
-		ResourceLink link = ResourceLink.forLocalReference("organization", mySource, "Organization", 123L, LONG_ID, new Date(), null);
+		ResourceLink link = getResourceLinkForLocalReference(LONG_ID);
 		myParams.getResourceLinks().add(link);
 
 		ReferenceParam referenceParam = getReferenceParam(LONG_ID);
@@ -75,6 +75,21 @@ public class ResourceIndexedSearchParamsTest {
 		assertTrue(result);
 	}
 
+	private ResourceLink getResourceLinkForLocalReference(String theTargetResourceId){
+
+		ResourceLink.ResourceLinkForLocalReferenceParams params = ResourceLink.ResourceLinkForLocalReferenceParams
+			.instance()
+			.setSourcePath("organization")
+			.setSourceResource(mySource)
+			.setTargetResourceType("Organization")
+			.setTargetResourcePid(123L)
+			.setTargetResourceId(theTargetResourceId)
+			.setUpdated(new Date());
+
+		return ResourceLink.forLocalReference(params);
+
+	}
+
 	private ReferenceParam getReferenceParam(String theId) {
 		ReferenceParam retVal = new ReferenceParam();
 		retVal.setValue(theId);

hapi-fhir-jpaserver-searchparam/src/test/java/ca/uhn/fhir/jpa/searchparam/registry/FhirContextSearchParamRegistryTest.java

@@ -0,0 +1,38 @@
+package ca.uhn.fhir.jpa.searchparam.registry;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.context.RuntimeSearchParam;
+import ca.uhn.fhir.rest.server.util.FhirContextSearchParamRegistry;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.hl7.fhir.instance.model.api.IAnyResource.SP_RES_ID;
+import static org.hl7.fhir.instance.model.api.IAnyResource.SP_RES_LAST_UPDATED;
+import static org.hl7.fhir.instance.model.api.IAnyResource.SP_RES_PROFILE;
+import static org.hl7.fhir.instance.model.api.IAnyResource.SP_RES_SECURITY;
+import static org.hl7.fhir.instance.model.api.IAnyResource.SP_RES_TAG;
+
+class FhirContextSearchParamRegistryTest {
+
+	private static final FhirContext ourFhirContext = FhirContext.forR4();
+
+	FhirContextSearchParamRegistry mySearchParamRegistry = new FhirContextSearchParamRegistry(ourFhirContext);
+
+	@ParameterizedTest
+	@CsvSource({
+		SP_RES_ID + ", Resource.id",
+		SP_RES_LAST_UPDATED + ", Resource.meta.lastUpdated",
+		SP_RES_TAG + ", Resource.meta.tag",
+		SP_RES_PROFILE + ", Resource.meta.profile",
+		SP_RES_SECURITY + ", Resource.meta.security"
+	})
+	void testResourceLevelSearchParamsAreRegistered(String theSearchParamName, String theSearchParamPath) {
+		RuntimeSearchParam sp = mySearchParamRegistry.getActiveSearchParam("Patient", theSearchParamName);
+
+		assertThat(sp)
+			.as("path is null for search parameter: '%s'", theSearchParamName)
+			.isNotNull().extracting("path").isEqualTo(theSearchParamPath);
+	}
+
+}

hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/config/SubscriptionConfig.java

@@ -21,7 +21,7 @@ package ca.uhn.fhir.jpa.subscription.config;
 
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.subscription.match.matcher.matching.SubscriptionStrategyEvaluator;
-import ca.uhn.fhir.jpa.subscription.submit.interceptor.SubscriptionQueryValidator;
+import ca.uhn.fhir.jpa.subscription.submit.interceptor.validator.SubscriptionQueryValidator;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 

hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/match/matcher/subscriber/SubscriptionMatchingSubscriber.java

@@ -151,15 +151,17 @@ public class SubscriptionMatchingSubscriber implements MessageHandler {
 	 */
 	private boolean processSubscription(
 			ResourceModifiedMessage theMsg, IIdType theResourceId, ActiveSubscription theActiveSubscription) {
-		// skip if the partitions don't match
+
 		CanonicalSubscription subscription = theActiveSubscription.getSubscription();
+
 		if (subscription != null
 				&& theMsg.getPartitionId() != null
 				&& theMsg.getPartitionId().hasPartitionIds()
-				&& !subscription.getCrossPartitionEnabled()
+				&& !subscription.isCrossPartitionEnabled()
 				&& !theMsg.getPartitionId().hasPartitionId(subscription.getRequestPartitionId())) {
 			return false;
 		}
+
 		String nextSubscriptionId = theActiveSubscription.getId();
 
 		if (isNotBlank(theMsg.getSubscriptionId())) {

hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/submit/config/SubscriptionSubmitterConfig.java

@@ -31,6 +31,10 @@ import ca.uhn.fhir.jpa.subscription.model.config.SubscriptionModelConfig;
 import ca.uhn.fhir.jpa.subscription.submit.interceptor.SubscriptionMatcherInterceptor;
 import ca.uhn.fhir.jpa.subscription.submit.interceptor.SubscriptionSubmitInterceptorLoader;
 import ca.uhn.fhir.jpa.subscription.submit.interceptor.SubscriptionValidatingInterceptor;
+import ca.uhn.fhir.jpa.subscription.submit.interceptor.validator.IChannelTypeValidator;
+import ca.uhn.fhir.jpa.subscription.submit.interceptor.validator.RegexEndpointUrlValidationStrategy;
+import ca.uhn.fhir.jpa.subscription.submit.interceptor.validator.RestHookChannelValidator;
+import ca.uhn.fhir.jpa.subscription.submit.interceptor.validator.SubscriptionChannelTypeValidatorFactory;
 import ca.uhn.fhir.jpa.subscription.submit.svc.ResourceModifiedSubmitterSvc;
 import ca.uhn.fhir.jpa.subscription.triggering.ISubscriptionTriggeringSvc;
 import ca.uhn.fhir.jpa.subscription.triggering.SubscriptionTriggeringSvcImpl;
@@ -44,6 +48,10 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Lazy;
 
+import java.util.List;
+
+import static ca.uhn.fhir.jpa.subscription.submit.interceptor.validator.RestHookChannelValidator.noOpEndpointUrlValidationStrategy;
+
 /**
  * This Spring config should be imported by a system that submits resources to the
  * matching queue for processing
@@ -109,4 +117,23 @@ public class SubscriptionSubmitterConfig {
 		return new AsyncResourceModifiedSubmitterSvc(
 				theIResourceModifiedMessagePersistenceSvc, theResourceModifiedConsumer);
 	}
+
+	@Bean
+	public IChannelTypeValidator restHookChannelValidator(SubscriptionSettings theSubscriptionSettings) {
+		RestHookChannelValidator.IEndpointUrlValidationStrategy iEndpointUrlValidationStrategy =
+				noOpEndpointUrlValidationStrategy;
+
+		if (theSubscriptionSettings.hasRestHookEndpointUrlValidationRegex()) {
+			String endpointUrlValidationRegex = theSubscriptionSettings.getRestHookEndpointUrlValidationRegex();
+			iEndpointUrlValidationStrategy = new RegexEndpointUrlValidationStrategy(endpointUrlValidationRegex);
+		}
+
+		return new RestHookChannelValidator(iEndpointUrlValidationStrategy);
+	}
+
+	@Bean
+	public SubscriptionChannelTypeValidatorFactory subscriptionChannelTypeValidatorFactory(
+			List<IChannelTypeValidator> theValidorList) {
+		return new SubscriptionChannelTypeValidatorFactory(theValidorList);
+	}
 }

hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/submit/interceptor/SubscriptionValidatingInterceptor.java

@@ -36,8 +36,10 @@ import ca.uhn.fhir.jpa.subscription.match.matcher.matching.SubscriptionStrategyE
 import ca.uhn.fhir.jpa.subscription.match.registry.SubscriptionCanonicalizer;
 import ca.uhn.fhir.jpa.subscription.model.CanonicalSubscription;
 import ca.uhn.fhir.jpa.subscription.model.CanonicalSubscriptionChannelType;
+import ca.uhn.fhir.jpa.subscription.submit.interceptor.validator.IChannelTypeValidator;
+import ca.uhn.fhir.jpa.subscription.submit.interceptor.validator.SubscriptionChannelTypeValidatorFactory;
+import ca.uhn.fhir.jpa.subscription.submit.interceptor.validator.SubscriptionQueryValidator;
 import ca.uhn.fhir.parser.DataFormatException;
-import ca.uhn.fhir.rest.api.EncodingEnum;
 import ca.uhn.fhir.rest.api.server.IBundleProvider;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
@@ -87,6 +89,9 @@ public class SubscriptionValidatingInterceptor {
 	@Autowired
 	private SubscriptionQueryValidator mySubscriptionQueryValidator;
 
+	@Autowired
+	private SubscriptionChannelTypeValidatorFactory mySubscriptionChannelTypeValidatorFactory;
+
 	@Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED)
 	public void resourcePreCreate(
 			IBaseResource theResource, RequestDetails theRequestDetails, RequestPartitionId theRequestPartitionId) {
@@ -149,7 +154,7 @@ public class SubscriptionValidatingInterceptor {
 				break;
 		}
 
-		validatePermissions(theSubscription, subscription, theRequestDetails, theRequestPartitionId, thePointcut);
+		validatePermissions(theSubscription, theRequestDetails, theRequestPartitionId, thePointcut);
 
 		mySubscriptionCanonicalizer.setMatchingStrategyTag(theSubscription, null);
 
@@ -167,7 +172,7 @@ public class SubscriptionValidatingInterceptor {
 
 			try {
 				SubscriptionMatchingStrategy strategy = mySubscriptionStrategyEvaluator.determineStrategy(subscription);
-				if (!(SubscriptionMatchingStrategy.IN_MEMORY == strategy)
+				if (SubscriptionMatchingStrategy.IN_MEMORY != strategy
 						&& mySubscriptionSettings.isOnlyAllowInMemorySubscriptions()) {
 					throw new InvalidRequestException(
 							Msg.code(2367)
@@ -236,12 +241,11 @@ public class SubscriptionValidatingInterceptor {
 
 	protected void validatePermissions(
 			IBaseResource theSubscription,
-			CanonicalSubscription theCanonicalSubscription,
 			RequestDetails theRequestDetails,
 			RequestPartitionId theRequestPartitionId,
 			Pointcut thePointcut) {
 		// If the subscription has the cross partition tag
-		if (SubscriptionUtil.isCrossPartition(theSubscription)
+		if (SubscriptionUtil.isDefinedAsCrossPartitionSubcription(theSubscription)
 				&& !(theRequestDetails instanceof SystemRequestDetails)) {
 			if (!mySubscriptionSettings.isCrossPartitionSubscriptionEnabled()) {
 				throw new UnprocessableEntityException(
@@ -319,27 +323,11 @@ public class SubscriptionValidatingInterceptor {
 	protected void validateChannelType(CanonicalSubscription theSubscription) {
 		if (theSubscription.getChannelType() == null) {
 			throw new UnprocessableEntityException(Msg.code(20) + "Subscription.channel.type must be populated");
-		} else if (theSubscription.getChannelType() == CanonicalSubscriptionChannelType.RESTHOOK) {
-			validateChannelPayload(theSubscription);
-			validateChannelEndpoint(theSubscription);
 		}
-	}
 
-	@SuppressWarnings("WeakerAccess")
-	protected void validateChannelEndpoint(CanonicalSubscription theResource) {
-		if (isBlank(theResource.getEndpointUrl())) {
-			throw new UnprocessableEntityException(
-					Msg.code(21) + "Rest-hook subscriptions must have Subscription.channel.endpoint defined");
-		}
-	}
-
-	@SuppressWarnings("WeakerAccess")
-	protected void validateChannelPayload(CanonicalSubscription theResource) {
-		if (!isBlank(theResource.getPayloadString())
-				&& EncodingEnum.forContentType(theResource.getPayloadString()) == null) {
-			throw new UnprocessableEntityException(Msg.code(1985) + "Invalid value for Subscription.channel.payload: "
-					+ theResource.getPayloadString());
-		}
+		IChannelTypeValidator iChannelTypeValidator =
+				mySubscriptionChannelTypeValidatorFactory.getValidatorForChannelType(theSubscription.getChannelType());
+		iChannelTypeValidator.validateChannelType(theSubscription);
 	}
 
 	@SuppressWarnings("WeakerAccess")
@@ -371,4 +359,10 @@ public class SubscriptionValidatingInterceptor {
 		mySubscriptionStrategyEvaluator = theSubscriptionStrategyEvaluator;
 		mySubscriptionQueryValidator = new SubscriptionQueryValidator(myDaoRegistry, theSubscriptionStrategyEvaluator);
 	}
+
+	@VisibleForTesting
+	public void setSubscriptionChannelTypeValidatorFactoryForUnitTest(
+			SubscriptionChannelTypeValidatorFactory theSubscriptionChannelTypeValidatorFactory) {
+		mySubscriptionChannelTypeValidatorFactory = theSubscriptionChannelTypeValidatorFactory;
+	}
 }

hapi-fhir-jpaserver-subscription/src/main/java/ca/uhn/fhir/jpa/subscription/submit/interceptor/validator/IChannelTypeValidator.java

@@ -0,0 +1,11 @@
+package ca.uhn.fhir.jpa.subscription.submit.interceptor.validator;
+
+import ca.uhn.fhir.jpa.subscription.model.CanonicalSubscription;
+import ca.uhn.fhir.jpa.subscription.model.CanonicalSubscriptionChannelType;
+
+public interface IChannelTypeValidator {
+
+	void validateChannelType(CanonicalSubscription theSubscription);
+
+	CanonicalSubscriptionChannelType getSubscriptionChannelType();
+}