hibernate search cannot be used for count query if non-active parameters are present (#6027)

fixing count query with fulltextsearch bug

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6024-full-index-search-with-filter-parameters.yaml

@@ -0,0 +1,10 @@
+---
+type: fix
+issue: 6024
+title: "Fixed a bug in search where requesting a count with HSearch indexing
+        and FilterParameter enabled and using the _filter parameter would result
+        in inaccurate results being returned.
+        This happened because the count query would use an incorrect set of parameters
+        to find the count, and the regular search when then try and ensure its results
+        matched the count query (which it couldn't because it had different parameters).
+"

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java

@@ -34,6 +34,7 @@ import ca.uhn.fhir.jpa.dao.search.IHSearchSortHelper;
 import ca.uhn.fhir.jpa.dao.search.LastNOperation;
 import ca.uhn.fhir.jpa.model.dao.JpaPid;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
+import ca.uhn.fhir.jpa.model.search.ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams;
 import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
 import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
 import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions;
@@ -141,17 +142,17 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 	}
 
 	@Override
-	public boolean supportsSomeOf(SearchParameterMap myParams) {
+	public boolean canUseHibernateSearch(String theResourceType, SearchParameterMap myParams) {
-
-		// keep this in sync with the guts of doSearch
 		boolean requiresHibernateSearchAccess = myParams.containsKey(Constants.PARAM_CONTENT)
 				|| myParams.containsKey(Constants.PARAM_TEXT)
 				|| myParams.isLastN();
+		// we have to use it - _text and _content searches only use hibernate
+		if (requiresHibernateSearchAccess) {
+			return true;
+		}
 
-		requiresHibernateSearchAccess |=
+		return myStorageSettings.isAdvancedHSearchIndexing()
-				myStorageSettings.isAdvancedHSearchIndexing() && myAdvancedIndexQueryBuilder.isSupportsSomeOf(myParams);
+				&& myAdvancedIndexQueryBuilder.canUseHibernateSearch(theResourceType, myParams, mySearchParamRegistry);
-
-		return requiresHibernateSearchAccess;
 	}
 
 	@Override
@@ -174,6 +175,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 	}
 
 	// keep this in sync with supportsSomeOf();
+	@SuppressWarnings("rawtypes")
 	private ISearchQueryExecutor doSearch(
 			String theResourceType,
 			SearchParameterMap theParams,
@@ -208,6 +210,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 		return DEFAULT_MAX_NON_PAGED_SIZE;
 	}
 
+	@SuppressWarnings("rawtypes")
 	private SearchQueryOptionsStep<?, Long, SearchLoadingOptionsStep, ?, ?> getSearchQueryOptionsStep(
 			String theResourceType, SearchParameterMap theParams, IResourcePersistentId theReferencingPid) {
 
@@ -230,6 +233,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 		return query;
 	}
 
+	@SuppressWarnings("rawtypes")
 	private PredicateFinalStep buildWhereClause(
 			SearchPredicateFactory f,
 			String theResourceType,
@@ -271,8 +275,12 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 			 * Handle other supported parameters
 			 */
 			if (myStorageSettings.isAdvancedHSearchIndexing() && theParams.getEverythingMode() == null) {
-				myAdvancedIndexQueryBuilder.addAndConsumeAdvancedQueryClauses(
+				ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams params =
-						builder, theResourceType, theParams, mySearchParamRegistry);
+						new ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams();
+				params.setSearchParamRegistry(mySearchParamRegistry)
+						.setResourceType(theResourceType)
+						.setSearchParameterMap(theParams);
+				myAdvancedIndexQueryBuilder.addAndConsumeAdvancedQueryClauses(builder, params);
 			}
 			// DROP EARLY HERE IF BOOL IS EMPTY?
 		});
@@ -283,11 +291,13 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 		return Search.session(myEntityManager);
 	}
 
+	@SuppressWarnings("rawtypes")
 	private List<IResourcePersistentId> convertLongsToResourcePersistentIds(List<Long> theLongPids) {
 		return theLongPids.stream().map(JpaPid::fromId).collect(Collectors.toList());
 	}
 
 	@Override
+	@SuppressWarnings({"rawtypes", "unchecked"})
 	public List<IResourcePersistentId> everything(
 			String theResourceName,
 			SearchParameterMap theParams,
@@ -336,6 +346,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 
 	@Transactional()
 	@Override
+	@SuppressWarnings("unchecked")
 	public List<IResourcePersistentId> search(
 			String theResourceName, SearchParameterMap theParams, RequestDetails theRequestDetails) {
 		validateHibernateSearchIsEnabled();
@@ -347,6 +358,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 	/**
 	 * Adapt our async interface to the legacy concrete List
 	 */
+	@SuppressWarnings("rawtypes")
 	private List<IResourcePersistentId> toList(ISearchQueryExecutor theSearchResultStream, long theMaxSize) {
 		return StreamSupport.stream(Spliterators.spliteratorUnknownSize(theSearchResultStream, 0), false)
 				.map(JpaPid::fromId)
@@ -384,6 +396,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 	}
 
 	@Override
+	@SuppressWarnings("rawtypes")
 	public List<IResourcePersistentId> lastN(SearchParameterMap theParams, Integer theMaximumResults) {
 		ensureElastic();
 		dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java

@@ -32,6 +32,7 @@ import org.hl7.fhir.instance.model.api.IBaseResource;
 import java.util.Collection;
 import java.util.List;
 
+@SuppressWarnings({"rawtypes"})
 public interface IFulltextSearchSvc {
 
 	/**
@@ -79,11 +80,18 @@ public interface IFulltextSearchSvc {
 	ExtendedHSearchIndexData extractLuceneIndexData(
 			IBaseResource theResource, ResourceIndexedSearchParams theNewParams);
 
-	boolean supportsSomeOf(SearchParameterMap myParams);
+	/**
+	 * Returns true if the parameter map can be handled for hibernate search.
+	 * We have to filter out any queries that might use search params
+	 * we only know how to handle in JPA.
+	 * -
+	 * See {@link ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder#addAndConsumeAdvancedQueryClauses}
+	 */
+	boolean canUseHibernateSearch(String theResourceType, SearchParameterMap theParameterMap);
 
 	/**
 	 * Re-publish the resource to the full-text index.
-	 *
+	 * -
 	 * During update, hibernate search only republishes the entity if it has changed.
 	 * During $reindex, we want to force the re-index.
 	 *

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchSearchBuilder.java

@@ -20,6 +20,7 @@
 package ca.uhn.fhir.jpa.dao.search;
 
 import ca.uhn.fhir.context.RuntimeSearchParam;
+import ca.uhn.fhir.jpa.model.search.ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil;
 import ca.uhn.fhir.model.api.IQueryParameterType;
@@ -34,6 +35,7 @@ import ca.uhn.fhir.rest.param.StringParam;
 import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.param.UriParam;
 import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
+import ca.uhn.fhir.rest.server.util.ResourceSearchParams;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.apache.commons.collections4.CollectionUtils;
@@ -44,6 +46,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static ca.uhn.fhir.rest.api.Constants.PARAMQUALIFIER_MISSING;
 
@@ -59,19 +62,57 @@ public class ExtendedHSearchSearchBuilder {
 	public static final Set<String> ourUnsafeSearchParmeters = Sets.newHashSet("_id", "_meta");
 
 	/**
-	 * Are any of the queries supported by our indexing?
+	 * Determine if ExtendedHibernateSearchBuilder can support this parameter
+	 * @param theParamName param name
+	 * @param theActiveParamsForResourceType active search parameters for the desired resource type
+	 * @return whether or not this search parameter is supported in hibernate
 	 */
-	public boolean isSupportsSomeOf(SearchParameterMap myParams) {
+	public boolean supportsSearchParameter(String theParamName, ResourceSearchParams theActiveParamsForResourceType) {
-		return myParams.getSort() != null
+		if (theActiveParamsForResourceType == null) {
-				|| myParams.getLastUpdated() != null
+			return false;
-				|| myParams.entrySet().stream()
+		}
-						.filter(e -> !ourUnsafeSearchParmeters.contains(e.getKey()))
+		if (ourUnsafeSearchParmeters.contains(theParamName)) {
-						// each and clause may have a different modifier, so split down to the ORs
+			return false;
-						.flatMap(andList -> andList.getValue().stream())
+		}
+		if (!theActiveParamsForResourceType.containsParamName(theParamName)) {
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Are any of the queries supported by our indexing?
+	 * -
+	 * If not, do not use hibernate, because the results will
+	 * be inaccurate and wrong.
+	 */
+	public boolean canUseHibernateSearch(
+			String theResourceType, SearchParameterMap myParams, ISearchParamRegistry theSearchParamRegistry) {
+		boolean canUseHibernate = true;
+		ResourceSearchParams resourceActiveSearchParams = theSearchParamRegistry.getActiveSearchParams(theResourceType);
+		for (String paramName : myParams.keySet()) {
+			// is this parameter supported?
+			if (!supportsSearchParameter(paramName, resourceActiveSearchParams)) {
+				canUseHibernate = false;
+			} else {
+				// are the parameter values supported?
+				canUseHibernate =
+						myParams.get(paramName).stream()
 								.flatMap(Collection::stream)
+								.collect(Collectors.toList())
+								.stream()
 								.anyMatch(this::isParamTypeSupported);
 			}
 
+			// if not supported, don't use
+			if (!canUseHibernate) {
+				return false;
+			}
+		}
+
+		return canUseHibernate;
+	}
+
 	/**
 	 * Are all the queries supported by our indexing?
 	 */
@@ -166,86 +207,91 @@ public class ExtendedHSearchSearchBuilder {
 	}
 
 	public void addAndConsumeAdvancedQueryClauses(
-			ExtendedHSearchClauseBuilder builder,
+			ExtendedHSearchClauseBuilder theBuilder,
-			String theResourceType,
+			ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams theMethodParams) {
-			SearchParameterMap theParams,
+		SearchParameterMap searchParameterMap = theMethodParams.getSearchParameterMap();
-			ISearchParamRegistry theSearchParamRegistry) {
+		String resourceType = theMethodParams.getResourceType();
+		ISearchParamRegistry searchParamRegistry = theMethodParams.getSearchParamRegistry();
+
 		// copy the keys to avoid concurrent modification error
-		ArrayList<String> paramNames = compileParamNames(theParams);
+		ArrayList<String> paramNames = compileParamNames(searchParameterMap);
+		ResourceSearchParams activeSearchParams = searchParamRegistry.getActiveSearchParams(resourceType);
 		for (String nextParam : paramNames) {
-			if (ourUnsafeSearchParmeters.contains(nextParam)) {
+			if (!supportsSearchParameter(nextParam, activeSearchParams)) {
-				continue;
-			}
-			RuntimeSearchParam activeParam = theSearchParamRegistry.getActiveSearchParam(theResourceType, nextParam);
-			if (activeParam == null) {
 				// ignore magic params handled in JPA
 				continue;
 			}
+			RuntimeSearchParam activeParam = activeSearchParams.get(nextParam);
 
 			// NOTE - keep this in sync with isParamSupported() above.
 			switch (activeParam.getParamType()) {
 				case TOKEN:
 					List<List<IQueryParameterType>> tokenTextAndOrTerms =
-							theParams.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_TOKEN_TEXT);
+							searchParameterMap.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_TOKEN_TEXT);
-					builder.addStringTextSearch(nextParam, tokenTextAndOrTerms);
+					theBuilder.addStringTextSearch(nextParam, tokenTextAndOrTerms);
 
 					List<List<IQueryParameterType>> tokenUnmodifiedAndOrTerms =
-							theParams.removeByNameUnmodified(nextParam);
+							searchParameterMap.removeByNameUnmodified(nextParam);
-					builder.addTokenUnmodifiedSearch(nextParam, tokenUnmodifiedAndOrTerms);
+					theBuilder.addTokenUnmodifiedSearch(nextParam, tokenUnmodifiedAndOrTerms);
 					break;
 
 				case STRING:
 					List<List<IQueryParameterType>> stringTextAndOrTerms =
-							theParams.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_TOKEN_TEXT);
+							searchParameterMap.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_TOKEN_TEXT);
-					builder.addStringTextSearch(nextParam, stringTextAndOrTerms);
+					theBuilder.addStringTextSearch(nextParam, stringTextAndOrTerms);
 
-					List<List<IQueryParameterType>> stringExactAndOrTerms =
+					List<List<IQueryParameterType>> stringExactAndOrTerms = searchParameterMap.removeByNameAndModifier(
-							theParams.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_STRING_EXACT);
+							nextParam, Constants.PARAMQUALIFIER_STRING_EXACT);
-					builder.addStringExactSearch(nextParam, stringExactAndOrTerms);
+					theBuilder.addStringExactSearch(nextParam, stringExactAndOrTerms);
 
 					List<List<IQueryParameterType>> stringContainsAndOrTerms =
-							theParams.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_STRING_CONTAINS);
+							searchParameterMap.removeByNameAndModifier(
-					builder.addStringContainsSearch(nextParam, stringContainsAndOrTerms);
+									nextParam, Constants.PARAMQUALIFIER_STRING_CONTAINS);
+					theBuilder.addStringContainsSearch(nextParam, stringContainsAndOrTerms);
 
-					List<List<IQueryParameterType>> stringAndOrTerms = theParams.removeByNameUnmodified(nextParam);
+					List<List<IQueryParameterType>> stringAndOrTerms =
-					builder.addStringUnmodifiedSearch(nextParam, stringAndOrTerms);
+							searchParameterMap.removeByNameUnmodified(nextParam);
+					theBuilder.addStringUnmodifiedSearch(nextParam, stringAndOrTerms);
 					break;
 
 				case QUANTITY:
-					List<List<IQueryParameterType>> quantityAndOrTerms = theParams.removeByNameUnmodified(nextParam);
+					List<List<IQueryParameterType>> quantityAndOrTerms =
-					builder.addQuantityUnmodifiedSearch(nextParam, quantityAndOrTerms);
+							searchParameterMap.removeByNameUnmodified(nextParam);
+					theBuilder.addQuantityUnmodifiedSearch(nextParam, quantityAndOrTerms);
 					break;
 
 				case REFERENCE:
-					List<List<IQueryParameterType>> referenceAndOrTerms = theParams.removeByNameUnmodified(nextParam);
+					List<List<IQueryParameterType>> referenceAndOrTerms =
-					builder.addReferenceUnchainedSearch(nextParam, referenceAndOrTerms);
+							searchParameterMap.removeByNameUnmodified(nextParam);
+					theBuilder.addReferenceUnchainedSearch(nextParam, referenceAndOrTerms);
 					break;
 
 				case DATE:
 					List<List<IQueryParameterType>> dateAndOrTerms = nextParam.equalsIgnoreCase("_lastupdated")
-							? getLastUpdatedAndOrList(theParams)
+							? getLastUpdatedAndOrList(searchParameterMap)
-							: theParams.removeByNameUnmodified(nextParam);
+							: searchParameterMap.removeByNameUnmodified(nextParam);
-					builder.addDateUnmodifiedSearch(nextParam, dateAndOrTerms);
+					theBuilder.addDateUnmodifiedSearch(nextParam, dateAndOrTerms);
 					break;
 
 				case COMPOSITE:
-					List<List<IQueryParameterType>> compositeAndOrTerms = theParams.removeByNameUnmodified(nextParam);
+					List<List<IQueryParameterType>> compositeAndOrTerms =
+							searchParameterMap.removeByNameUnmodified(nextParam);
 					// RuntimeSearchParam only points to the subs by reference.  Resolve here while we have
 					// ISearchParamRegistry
 					List<RuntimeSearchParam> subSearchParams =
-							JpaParamUtil.resolveCompositeComponentsDeclaredOrder(theSearchParamRegistry, activeParam);
+							JpaParamUtil.resolveCompositeComponentsDeclaredOrder(searchParamRegistry, activeParam);
-					builder.addCompositeUnmodifiedSearch(activeParam, subSearchParams, compositeAndOrTerms);
+					theBuilder.addCompositeUnmodifiedSearch(activeParam, subSearchParams, compositeAndOrTerms);
 					break;
 
 				case URI:
 					List<List<IQueryParameterType>> uriUnmodifiedAndOrTerms =
-							theParams.removeByNameUnmodified(nextParam);
+							searchParameterMap.removeByNameUnmodified(nextParam);
-					builder.addUriUnmodifiedSearch(nextParam, uriUnmodifiedAndOrTerms);
+					theBuilder.addUriUnmodifiedSearch(nextParam, uriUnmodifiedAndOrTerms);
 					break;
 
 				case NUMBER:
-					List<List<IQueryParameterType>> numberUnmodifiedAndOrTerms = theParams.remove(nextParam);
+					List<List<IQueryParameterType>> numberUnmodifiedAndOrTerms = searchParameterMap.remove(nextParam);
-					builder.addNumberUnmodifiedSearch(nextParam, numberUnmodifiedAndOrTerms);
+					theBuilder.addNumberUnmodifiedSearch(nextParam, numberUnmodifiedAndOrTerms);
 					break;
 
 				default:

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/LastNOperation.java

@@ -22,6 +22,7 @@ package ca.uhn.fhir.jpa.dao.search;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.model.entity.StorageSettings;
+import ca.uhn.fhir.jpa.model.search.ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
 import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
@@ -67,8 +68,12 @@ public class LastNOperation {
 					b.must(f.match().field("myResourceType").matching(OBSERVATION_RES_TYPE));
 					ExtendedHSearchClauseBuilder builder =
 							new ExtendedHSearchClauseBuilder(myFhirContext, myStorageSettings, b, f);
-					myExtendedHSearchSearchBuilder.addAndConsumeAdvancedQueryClauses(
+					ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams params =
-							builder, OBSERVATION_RES_TYPE, theParams.clone(), mySearchParamRegistry);
+							new ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams();
+					params.setResourceType(OBSERVATION_RES_TYPE)
+							.setSearchParameterMap(theParams.clone())
+							.setSearchParamRegistry(mySearchParamRegistry);
+					myExtendedHSearchSearchBuilder.addAndConsumeAdvancedQueryClauses(builder, params);
 				}))
 				.aggregation(observationsByCodeKey, f -> f.fromJson(lastNAggregation.toAggregation()))
 				.fetch(0);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/search/ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams.java

@@ -0,0 +1,73 @@
+/*-
+ * #%L
+ * HAPI FHIR JPA Server
+ * %%
+ * Copyright (C) 2014 - 2024 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+package ca.uhn.fhir.jpa.model.search;
+
+import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchClauseBuilder;
+import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
+
+/**
+ * This is a parameter class for the
+ * {@link ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder#addAndConsumeAdvancedQueryClauses(ExtendedHSearchClauseBuilder, ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams)}
+ * method, so that we can keep the signature manageable (small) and allow for updates without breaking
+ * implementers so often.
+ */
+public class ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams {
+	/**
+	 * Resource type
+	 */
+	private String myResourceType;
+	/**
+	 * The registered search
+	 */
+	private SearchParameterMap mySearchParameterMap;
+	/**
+	 * Search param registry
+	 */
+	private ISearchParamRegistry mySearchParamRegistry;
+
+	public String getResourceType() {
+		return myResourceType;
+	}
+
+	public ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams setResourceType(String theResourceType) {
+		myResourceType = theResourceType;
+		return this;
+	}
+
+	public SearchParameterMap getSearchParameterMap() {
+		return mySearchParameterMap;
+	}
+
+	public ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams setSearchParameterMap(SearchParameterMap theParams) {
+		mySearchParameterMap = theParams;
+		return this;
+	}
+
+	public ISearchParamRegistry getSearchParamRegistry() {
+		return mySearchParamRegistry;
+	}
+
+	public ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams setSearchParamRegistry(
+			ISearchParamRegistry theSearchParamRegistry) {
+		mySearchParamRegistry = theSearchParamRegistry;
+		return this;
+	}
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java

@@ -79,7 +79,7 @@ public class PersistedJpaSearchFirstPageBundleProvider extends PersistedJpaBundl
 		ourLog.trace("Done fetching search resource PIDs");
 
 		int countOfPids = pids.size();
-		;
+
 		int maxSize = Math.min(theToIndex - theFromIndex, countOfPids);
 		thePageBuilder.setTotalRequestedResourcesFetched(countOfPids);
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java

@@ -95,6 +95,7 @@ import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
 import ca.uhn.fhir.util.StopWatch;
 import ca.uhn.fhir.util.StringUtil;
 import ca.uhn.fhir.util.UrlUtil;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Streams;
 import com.healthmarketscience.sqlbuilder.Condition;
 import jakarta.annotation.Nonnull;
@@ -165,7 +166,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 	public static boolean myUseMaxPageSize50ForTest = false;
 	protected final IInterceptorBroadcaster myInterceptorBroadcaster;
 	protected final IResourceTagDao myResourceTagDao;
-	String myResourceName;
+	private String myResourceName;
 	private final Class<? extends IBaseResource> myResourceType;
 	private final HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory;
 	private final SqlObjectFactory mySqlBuilderFactory;
@@ -206,6 +207,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 	/**
 	 * Constructor
 	 */
+	@SuppressWarnings({"rawtypes", "unchecked"})
 	public SearchBuilder(
 			IDao theDao,
 			String theResourceName,
@@ -240,6 +242,11 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 		myIdHelperService = theIdHelperService;
 	}
 
+	@VisibleForTesting
+	void setResourceName(String theName) {
+		myResourceName = theName;
+	}
+
 	@Override
 	public void setMaxResultsToFetch(Integer theMaxResultsToFetch) {
 		myMaxResultsToFetch = theMaxResultsToFetch;
@@ -265,8 +272,6 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 			attemptComboUniqueSpProcessing(theQueryStack, theParams, theRequest);
 		}
 
-		SearchContainedModeEnum searchContainedMode = theParams.getSearchContainedMode();
-
 		// Handle _id and _tag last, since they can typically be tacked onto a different parameter
 		List<String> paramNames = myParams.keySet().stream()
 				.filter(t -> !t.equals(IAnyResource.SP_RES_ID))
@@ -399,7 +404,8 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 			}
 
 			if (fulltextExecutor == null) {
-				fulltextExecutor = SearchQueryExecutors.from(fulltextMatchIds);
+				fulltextExecutor =
+						SearchQueryExecutors.from(fulltextMatchIds != null ? fulltextMatchIds : new ArrayList<>());
 			}
 
 			if (theSearchRuntimeDetails != null) {
@@ -486,7 +492,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 		return fulltextEnabled
 				&& myParams != null
 				&& myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE
-				&& myFulltextSearchSvc.supportsSomeOf(myParams)
+				&& myFulltextSearchSvc.canUseHibernateSearch(myResourceName, myParams)
 				&& myFulltextSearchSvc.supportsAllSortTerms(myResourceName, myParams);
 	}
 
@@ -538,8 +544,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 
 			pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue);
 		}
-		List<JpaPid> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid, theRequestDetails);
+		return myFulltextSearchSvc.everything(myResourceName, myParams, pid, theRequestDetails);
-		return pids;
 	}
 
 	private void doCreateChunkedQueries(
@@ -862,13 +867,8 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 			theQueryStack.addSortOnLastUpdated(ascending);
 
 		} else {
-
+			RuntimeSearchParam param =
-			RuntimeSearchParam param = null;
+					mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName());
-
-			if (param == null) {
-				// do we have a composition param defined for the whole chain?
-				param = mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName());
-			}
 
 			/*
 			 * If we have a sort like _sort=subject.name and we  have an
@@ -896,9 +896,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 							mySearchParamRegistry.getActiveSearchParam(myResourceName, referenceParam);
 					if (outerParam == null) {
 						throwInvalidRequestExceptionForUnknownSortParameter(myResourceName, referenceParam);
-					}
+					} else if (outerParam.hasUpliftRefchain(targetParam)) {
-
-					if (outerParam.hasUpliftRefchain(targetParam)) {
 						for (String nextTargetType : outerParam.getTargets()) {
 							if (referenceParamTargetType != null && !referenceParamTargetType.equals(nextTargetType)) {
 								continue;
@@ -945,6 +943,9 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 				throwInvalidRequestExceptionForUnknownSortParameter(getResourceName(), paramName);
 			}
 
+			// param will never be null here (the above line throws if it does)
+			// this is just to prevent the warning
+			assert param != null;
 			if (isNotBlank(chainName) && param.getParamType() != RestSearchParameterTypeEnum.REFERENCE) {
 				throw new InvalidRequestException(
 						Msg.code(2285) + "Invalid chain, " + paramName + " is not a reference SearchParameter");
@@ -1121,11 +1122,15 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 						resourceType, next, tagMap.get(next.getId()), theForHistoryOperation);
 			}
 			if (resource == null) {
+				if (next != null) {
 					ourLog.warn(
 							"Unable to find resource {}/{}/_history/{} in database",
 							next.getResourceType(),
 							next.getIdDt().getIdPart(),
 							next.getVersion());
+				} else {
+					ourLog.warn("Unable to find resource in database.");
+				}
 				continue;
 			}
 
@@ -1196,7 +1201,6 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 			RequestDetails theDetails) {
 		if (thePids.isEmpty()) {
 			ourLog.debug("The include pids are empty");
-			// return;
 		}
 
 		// Dupes will cause a crash later anyhow, but this is expensive so only do it
@@ -1256,10 +1260,9 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 		// only impl
 		// to handle lastN?
 		if (myStorageSettings.isAdvancedHSearchIndexing() && myStorageSettings.isStoreResourceInHSearchIndex()) {
-			List<Long> pidList = thePids.stream().map(pid -> (pid).getId()).collect(Collectors.toList());
+			List<Long> pidList = thePids.stream().map(JpaPid::getId).collect(Collectors.toList());
 
-			List<IBaseResource> resources = myFulltextSearchSvc.getResources(pidList);
+			return myFulltextSearchSvc.getResources(pidList);
-			return resources;
 		} else if (!Objects.isNull(myParams) && myParams.isLastN()) {
 			// legacy LastN implementation
 			return myIElasticsearchSvc.getObservationResources(thePids);
@@ -1344,7 +1347,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 
 			for (Iterator<Include> iter = includes.iterator(); iter.hasNext(); ) {
 				Include nextInclude = iter.next();
-				if (nextInclude.isRecurse() == false) {
+				if (!nextInclude.isRecurse()) {
 					iter.remove();
 				}
 
@@ -1707,6 +1710,8 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 	}
 
 	/**
+	 * Calls Performance Trace Hook
+	 * @param request the request deatils
 	 * Sends a raw SQL query to the Pointcut for raw SQL queries.
 	 */
 	private void callRawSqlHookWithCurrentThreadQueries(RequestDetails request) {
@@ -1890,7 +1895,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 			for (RuntimeSearchParam nextCandidate : candidateComboParams) {
 				List<String> nextCandidateParamNames =
 						JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, nextCandidate).stream()
-								.map(t -> t.getName())
+								.map(RuntimeSearchParam::getName)
 								.collect(Collectors.toList());
 				if (theParams.keySet().containsAll(nextCandidateParamNames)) {
 					comboParam = nextCandidate;
@@ -1902,7 +1907,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 
 		if (comboParam != null) {
 			// Since we're going to remove elements below
-			theParams.values().forEach(nextAndList -> ensureSubListsAreWritable(nextAndList));
+			theParams.values().forEach(this::ensureSubListsAreWritable);
 
 			StringBuilder sb = new StringBuilder();
 			sb.append(myResourceName);

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/builder/SearchBuilderTest.java

@@ -39,7 +39,7 @@ class SearchBuilderTest {
 
 	@BeforeEach
 	public void beforeEach() {
-		mySearchBuilder.myResourceName = "QuestionnaireResponse";
+		mySearchBuilder.setResourceName("QuestionnaireResponse");
 		when(myDaoRegistry.getRegisteredDaoTypes()).thenReturn(ourCtx.getResourceTypes());
 	}
 

hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java

@@ -26,6 +26,7 @@ import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
 import ca.uhn.fhir.jpa.search.BaseSourceSearchParameterTestCases;
 import ca.uhn.fhir.jpa.search.CompositeSearchParameterTestCases;
 import ca.uhn.fhir.jpa.search.QuantitySearchParameterTestCases;
+import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
 import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
@@ -54,6 +55,9 @@ import ca.uhn.fhir.test.utilities.LogbackLevelOverrideExtension;
 import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
 import ca.uhn.fhir.validation.FhirValidator;
 import ca.uhn.fhir.validation.ValidationResult;
+import ca.uhn.test.util.LogbackTestExtension;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
 import jakarta.annotation.Nonnull;
 import jakarta.persistence.EntityManager;
 import org.hl7.fhir.instance.model.api.IBaseCoding;
@@ -118,6 +122,7 @@ import static ca.uhn.fhir.jpa.model.util.UcumServiceUtil.UCUM_CODESYSTEM_URL;
 import static ca.uhn.fhir.rest.api.Constants.CHARSET_UTF8;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -168,6 +173,9 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
 	TestDaoSearch myTestDaoSearch;
 	@RegisterExtension
 	LogbackLevelOverrideExtension myLogbackLevelOverrideExtension = new LogbackLevelOverrideExtension();
+
+	@RegisterExtension
+	LogbackTestExtension myLogbackTestExtension = new LogbackTestExtension();
 	@Autowired
 	@Qualifier("myCodeSystemDaoR4")
 	private IFhirResourceDao<CodeSystem> myCodeSystemDao;
@@ -742,19 +750,21 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
 	 */
 	@Test
 	public void testDirectPathWholeResourceNotIndexedWorks() {
+		// setup
+		myLogbackLevelOverrideExtension.setLogLevel(SearchBuilder.class, Level.WARN);
 		IIdType id1 = myTestDataBuilder.createObservation(List.of(myTestDataBuilder.withObservationCode("http://example.com/", "theCode")));
 
 		// set it after creating resource, so search doesn't find it in the index
 		myStorageSettings.setStoreResourceInHSearchIndex(true);
 
-		myCaptureQueriesListener.clear();
+		List<IBaseResource> result = searchForFastResources("Observation?code=theCode&_count=10&_total=accurate");
-
-		List<IBaseResource> result = searchForFastResources("Observation?code=theCode");
-		myCaptureQueriesListener.logSelectQueriesForCurrentThread();
 
 		assertThat(result).hasSize(1);
 		assertEquals(((Observation) result.get(0)).getIdElement().getIdPart(), id1.getIdPart());
-		assertThat(myCaptureQueriesListener.getSelectQueriesForCurrentThread().size()).as("JPA search for IDs and for resources").isEqualTo(2);
+
+		List<ILoggingEvent> events = myLogbackTestExtension.filterLoggingEventsWithPredicate(e -> e.getLevel() == Level.WARN);
+		assertFalse(events.isEmpty());
+		assertTrue(events.stream().anyMatch(e -> e.getFormattedMessage().contains("Some resources were not found in index. Make sure all resources were indexed. Resorting to database search.")));
 
 		// restore changed property
 		JpaStorageSettings defaultConfig = new JpaStorageSettings();

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java

@@ -2120,6 +2120,8 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 	@SuppressWarnings("unused")
 	@Test
 	public void testFullTextSearch() throws Exception {
+		IParser parser = myFhirContext.newJsonParser();
+
 		Observation obs1 = new Observation();
 		obs1.getCode().setText("Systolic Blood Pressure");
 		obs1.setStatus(ObservationStatus.FINAL);
@@ -2131,13 +2133,21 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 		obs2.setStatus(ObservationStatus.FINAL);
 		obs2.setValue(new Quantity(81));
 		IIdType id2 = myObservationDao.create(obs2, mySrd).getId().toUnqualifiedVersionless();
+		obs2.setId(id2);
+
+		myStorageSettings.setAdvancedHSearchIndexing(true);
 
 		HttpGet get = new HttpGet(myServerBase + "/Observation?_content=systolic&_pretty=true");
+		get.addHeader("Content-Type", "application/json");
 		try (CloseableHttpResponse response = ourHttpClient.execute(get)) {
 			assertEquals(200, response.getStatusLine().getStatusCode());
 			String responseString = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(responseString);
-			assertThat(responseString).contains(id1.getIdPart());
+			Bundle bundle = parser.parseResource(Bundle.class, responseString);
+			assertEquals(1, bundle.getTotal());
+			Resource resource = bundle.getEntry().get(0).getResource();
+			assertEquals("Observation", resource.fhirType());
+			assertEquals(id1.getIdPart(), resource.getIdPart());
 		}
 	}
 

hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java

@@ -398,7 +398,7 @@ public abstract class BaseJpaR5Test extends BaseJpaTest implements ITestDataBuil
 
 	private PerformanceTracingLoggingInterceptor myPerformanceTracingLoggingInterceptor;
 	@Autowired
-	private DaoRegistry myDaoRegistry;
+	protected DaoRegistry myDaoRegistry;
 	@Autowired
 	private IBulkDataExportJobSchedulingHelper myBulkDataSchedulerHelper;
 

hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/UpliftedRefchainsAndChainedSortingR5Test.java

@@ -13,16 +13,17 @@ import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.util.BundleBuilder;
 import ca.uhn.fhir.util.HapiExtensions;
+import jakarta.annotation.Nonnull;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
-import org.hl7.fhir.r5.model.Composition;
-import org.hl7.fhir.r5.model.IdType;
 import org.hl7.fhir.r5.model.Bundle;
 import org.hl7.fhir.r5.model.CodeType;
+import org.hl7.fhir.r5.model.Composition;
 import org.hl7.fhir.r5.model.DateType;
 import org.hl7.fhir.r5.model.Encounter;
 import org.hl7.fhir.r5.model.Enumerations;
 import org.hl7.fhir.r5.model.Extension;
+import org.hl7.fhir.r5.model.IdType;
 import org.hl7.fhir.r5.model.Identifier;
 import org.hl7.fhir.r5.model.Organization;
 import org.hl7.fhir.r5.model.Patient;
@@ -35,7 +36,6 @@ import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.context.ContextConfiguration;
 
-import jakarta.annotation.Nonnull;
 import java.util.List;
 
 import static org.apache.commons.lang3.StringUtils.countMatches;

hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5Test.java

@@ -1,11 +1,26 @@
 package ca.uhn.fhir.jpa.provider.r5;
 
+import ca.uhn.fhir.batch2.jobs.reindex.ReindexAppCtx;
+import ca.uhn.fhir.batch2.jobs.reindex.ReindexJobParameters;
+import ca.uhn.fhir.batch2.model.JobInstanceStartRequest;
+import ca.uhn.fhir.interceptor.model.RequestPartitionId;
 import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
+import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse;
 import ca.uhn.fhir.jpa.entity.Search;
 import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
+import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.model.api.Include;
+import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.parser.StrictErrorHandler;
 import ca.uhn.fhir.rest.api.Constants;
+import ca.uhn.fhir.rest.api.SearchTotalModeEnum;
+import ca.uhn.fhir.rest.api.SortOrderEnum;
+import ca.uhn.fhir.rest.api.SortSpec;
+import ca.uhn.fhir.rest.api.server.IBundleProvider;
+import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
 import ca.uhn.fhir.rest.client.interceptor.CapturingInterceptor;
+import ca.uhn.fhir.rest.param.StringParam;
 import ca.uhn.fhir.rest.server.exceptions.NotImplementedOperationException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException;
 import ca.uhn.fhir.util.BundleBuilder;
@@ -24,13 +39,18 @@ import org.hl7.fhir.r5.model.CarePlan;
 import org.hl7.fhir.r5.model.CodeableConcept;
 import org.hl7.fhir.r5.model.Condition;
 import org.hl7.fhir.r5.model.DateTimeType;
+import org.hl7.fhir.r5.model.Extension;
 import org.hl7.fhir.r5.model.MedicationRequest;
+import org.hl7.fhir.r5.model.MedicinalProductDefinition;
 import org.hl7.fhir.r5.model.Observation;
 import org.hl7.fhir.r5.model.Observation.ObservationComponentComponent;
 import org.hl7.fhir.r5.model.Organization;
 import org.hl7.fhir.r5.model.Parameters;
 import org.hl7.fhir.r5.model.Patient;
 import org.hl7.fhir.r5.model.Quantity;
+import org.hl7.fhir.r5.model.SearchParameter;
+import org.hl7.fhir.r5.model.StringType;
+import org.intellij.lang.annotations.Language;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -43,11 +63,13 @@ import org.springframework.util.comparator.Comparators;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.apache.commons.lang3.StringUtils.leftPad;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 @SuppressWarnings("Duplicates")
 public class ResourceProviderR5Test extends BaseResourceProviderR5Test {
@@ -206,7 +228,181 @@ public class ResourceProviderR5Test extends BaseResourceProviderR5Test {
 			assertEquals(501, e.getStatusCode());
 			assertThat(e.getMessage()).contains("Some Failure Message");
 		}
+	}
 
+	@Test
+	public void searchForNewerResources_fullTextSearchWithFilterAndCount_shouldReturnAccurateResults() {
+		IParser parser = myFhirContext.newJsonParser();
+		int count = 10;
+
+		boolean presetFilterParameterEnabled = myStorageSettings.isFilterParameterEnabled();
+		boolean presetAdvancedHSearchIndexing = myStorageSettings.isAdvancedHSearchIndexing();
+
+		try {
+			// fullTextSearch means Advanced Hibernate Search
+			myStorageSettings.setFilterParameterEnabled(true);
+			myStorageSettings.setAdvancedHSearchIndexing(true);
+
+			// create custom search parameters - the _filter and _include are needed
+			{
+				@SuppressWarnings("unchecked")
+				IFhirResourceDao<SearchParameter> spDao = myDaoRegistry.getResourceDao("SearchParameter");
+				SearchParameter sp;
+
+				@Language("JSON")
+				String includeParam = """
+						{
+						  "resourceType": "SearchParameter",
+						  "id": "9905463e-e817-4db0-9a3e-ff6aa3427848",
+						  "meta": {
+						    "versionId": "2",
+						    "lastUpdated": "2024-03-28T12:53:57.874+00:00",
+						    "source": "#7b34a4bfa42fe3ae"
+						  },
+						  "title": "Medicinal Product Manfacturer",
+						  "status": "active",
+						  "publisher": "MOH-IDMS",
+						  "code": "productmanufacturer",
+						  "base": [
+						    "MedicinalProductDefinition"
+						  ],
+						  "type": "reference",
+						  "expression": "MedicinalProductDefinition.operation.organization"
+						}
+					""";
+				sp = parser.parseResource(SearchParameter.class, includeParam);
+				spDao.create(sp, new SystemRequestDetails());
+				sp = null;
+				@Language("JSON")
+				String filterParam = """
+						{
+						  "resourceType": "SearchParameter",
+						  "id": "SEARCH-PARAMETER-MedicinalProductDefinition-SearchableString",
+						  "meta": {
+						    "versionId": "2",
+						    "lastUpdated": "2024-03-27T19:20:25.200+00:00",
+						    "source": "#384dd6bccaeafa6c"
+						  },
+						  "url": "https://health.gov.on.ca/idms/fhir/SearchParameter/MedicinalProductDefinition-SearchableString",
+						  "version": "1.0.0",
+						  "name": "MedicinalProductDefinitionSearchableString",
+						  "status": "active",
+						  "publisher": "MOH-IDMS",
+						  "description": "Search Parameter for the MedicinalProductDefinition Searchable String Extension",
+						  "code": "MedicinalProductDefinitionSearchableString",
+						  "base": [
+						    "MedicinalProductDefinition"
+						  ],
+						  "type": "string",
+						  "expression": "MedicinalProductDefinition.extension('https://health.gov.on.ca/idms/fhir/StructureDefinition/SearchableExtraString')",
+						  "target": [
+						    "MedicinalProductDefinition"
+						  ]
+						}
+					""";
+				sp = parser.parseResource(SearchParameter.class, filterParam);
+				spDao.create(sp, new SystemRequestDetails());
+			}
+			// create MedicinalProductDefinitions
+			MedicinalProductDefinition mdr;
+			{
+				@Language("JSON")
+				String mpdstr = """
+					{
+					                "resourceType": "MedicinalProductDefinition",
+					                "id": "36fb418b-4b1f-414c-bbb1-731bc8744b93",
+					                "meta": {
+					                    "versionId": "17",
+					                    "lastUpdated": "2024-06-10T16:52:23.907+00:00",
+					                    "source": "#3a309416d5f52c5b",
+					                    "profile": [
+					                        "https://health.gov.on.ca/idms/fhir/StructureDefinition/IDMS_MedicinalProductDefinition"
+					                    ]
+					                },
+					                "extension": [
+					                    {
+					                        "url": "https://health.gov.on.ca/idms/fhir/StructureDefinition/SearchableExtraString",
+					                        "valueString": "zahidbrand0610-2up|genupuu|qwewqe2 111|11111115|DF other des|Biologic|Oncology|Private Label"
+					                    }
+					                ],
+					                "status": {
+					                    "coding": [
+					                        {
+					                            "system": "http://hl7.org/fhir/ValueSet/publication-status",
+					                            "code": "active",
+					                            "display": "Active"
+					                        }
+					                    ]
+					                },
+					                "name": [
+					                    {
+					                        "productName": "zahidbrand0610-2up"
+					                    }
+					                ]
+					            }
+					""";
+				mdr = parser.parseResource(MedicinalProductDefinition.class, mpdstr);
+			}
+			IFhirResourceDao<MedicinalProductDefinition> mdrdao = myDaoRegistry.getResourceDao(MedicinalProductDefinition.class);
+
+			/*
+			 * We actually want a bunch of non-matching resources in the db
+			 * that won't match the filter before we get to the one that will.
+			 *
+			 * To this end, we're going to insert more than we plan
+			 * on retrieving to ensure the _filter is being used in both the
+			 * count query and the actual db hit
+			 */
+			List<MedicinalProductDefinition.MedicinalProductDefinitionNameComponent> productNames = mdr.getName();
+			mdr.setName(null);
+			List<Extension> extensions = mdr.getExtension();
+			mdr.setExtension(null);
+			// we need at least 10 of these; 20 should be good
+			for (int i = 0; i < 2 * count; i++) {
+				mdr.addName(new MedicinalProductDefinition.MedicinalProductDefinitionNameComponent("Product " + i));
+				mdr.addExtension()
+					.setUrl("https://health.gov.on.ca/idms/fhir/StructureDefinition/SearchableExtraString")
+					.setValue(new StringType("Non-matching string " + i));
+				mdrdao.create(mdr, new SystemRequestDetails());
+			}
+			mdr.setName(productNames);
+			mdr.setExtension(extensions);
+			mdrdao.create(mdr, new SystemRequestDetails());
+
+			// do a reindex
+			ReindexJobParameters jobParameters = new ReindexJobParameters();
+			jobParameters.setRequestPartitionId(RequestPartitionId.allPartitions());
+			JobInstanceStartRequest request = new JobInstanceStartRequest();
+			request.setJobDefinitionId(ReindexAppCtx.JOB_REINDEX);
+			request.setParameters(jobParameters);
+			Batch2JobStartResponse response = myJobCoordinator.startInstance(new SystemRequestDetails(), request);
+
+			myBatch2JobHelper.awaitJobCompletion(response);
+
+			// query like:
+			// MedicinalProductDefinition?_getpagesoffset=0&_count=10&_total=accurate&_sort:asc=name&status=active&_include=MedicinalProductDefinition:productmanufacturer&_filter=MedicinalProductDefinitionSearchableString%20co%20%22zah%22
+			SearchParameterMap map = new SearchParameterMap();
+			map.setCount(10);
+			map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
+			map.setSort(new SortSpec().setOrder(SortOrderEnum.ASC).setParamName("name"));
+			map.setIncludes(Set.of(
+				new Include("MedicinalProductDefinition:productmanufacturer")
+			));
+			map.add("_filter", new StringParam("MedicinalProductDefinitionSearchableString co \"zah\""));
+
+			// test
+			IBundleProvider result = mdrdao.search(map, new SystemRequestDetails());
+
+			// validate
+			// we expect to find our 1 matching resource
+			assertEquals(1, result.getAllResources().size());
+			assertNotNull(result.size());
+			assertEquals(1, result.size());
+		} finally {
+			// reset values
+			myStorageSettings.setFilterParameterEnabled(presetFilterParameterEnabled);
+			myStorageSettings.setAdvancedHSearchIndexing(presetAdvancedHSearchIndexing);
+		}
 	}
 
 	@Test
@@ -609,4 +805,5 @@ public class ResourceProviderR5Test extends BaseResourceProviderR5Test {
 		}
 		return retVal;
 	}
+
 }