4799 searching on paramname profile with and missing modifier does not match resources (#4840)

* initial test

* Adding changelog

* Updated to remove the discarding of search params starting with an underscore

* test commit

* Updated handling of search parameters with underscores

* Revert work in progress

* Updated storage settings with new property for inline tag storage mode

* Updated storage settings to tell when inline tag storage mode is set

* simplified test failure

* Added tests for _tag and _system

* Add PreviousVersion service with tests (#4902)

* started writing PreviousVersionReader

* started writing PreviousVersionReader

* moar tests

* add partitioning to the test

* switch subscription to use previous version reader

---------

Co-authored-by: Ken Stevens <ken@smilecdr.com>

* Updated logic to be less confusing

* Bumping version to 6.7.4

* Reverted hapi bump, added improve logic for handling missing search params

* updates

* Updated failing tests

* remove redundant tests

* more updates

* Adding Jira number.

---------

Co-authored-by: peartree <etienne.poirier@smilecdr.com>
Co-authored-by: David Raeside <davidraeside@smilecdr.com>
Co-authored-by: volodymyr <volodymyr.korzh@smilecdr.com>
Co-authored-by: David Raeside <david.raeside@smilecdr.com>
Co-authored-by: Ken Stevens <khstevens@gmail.com>
Co-authored-by: Ken Stevens <ken@smilecdr.com>

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_8_0/4799-searching-with-param-profile-and-modifier-missing-returns-incorrect-results.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 4799
+jira: SMILE-6669
+title: "There is an issue where searching on paramName '_profile' with modifier 'missing' will not provide the correct search result.  This has been fixed."

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java

@@ -34,6 +34,7 @@ import ca.uhn.fhir.jpa.model.entity.StorageSettings;
 import ca.uhn.fhir.jpa.model.util.UcumServiceUtil;
 import ca.uhn.fhir.jpa.searchparam.SearchParamConstants;
 import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil;
+import ca.uhn.fhir.jpa.searchparam.util.RuntimeSearchParamHelper;
 import ca.uhn.fhir.model.api.IQueryParameterType;
 import ca.uhn.fhir.model.primitive.BoundCodeDt;
 import ca.uhn.fhir.rest.api.Constants;
@@ -1338,7 +1339,7 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
 			}
 
 			// See the method javadoc for an explanation of this
-			if (startsWith(nextSpDef.getPath(), "Resource.")) {
+			if (RuntimeSearchParamHelper.isResourceLevel(nextSpDef)) {
 				continue;
 			}
 

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/ResourceIndexedSearchParams.java

@@ -25,7 +25,9 @@ import ca.uhn.fhir.jpa.model.config.PartitionSettings;
 import ca.uhn.fhir.jpa.model.entity.*;
 import ca.uhn.fhir.jpa.model.entity.StorageSettings;
 import ca.uhn.fhir.jpa.model.util.UcumServiceUtil;
+import ca.uhn.fhir.jpa.searchparam.util.RuntimeSearchParamHelper;
 import ca.uhn.fhir.model.api.IQueryParameterType;
+import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
 import ca.uhn.fhir.rest.param.QuantityParam;
 import ca.uhn.fhir.rest.param.ReferenceParam;
@@ -62,6 +64,8 @@ public final class ResourceIndexedSearchParams {
 	final public Collection<SearchParamPresentEntity> mySearchParamPresentEntities = new HashSet<>();
 	final public Collection<ResourceIndexedSearchParamComposite>  myCompositeParams = new HashSet<>();
 
+	private static final Set<String> myIgnoredParams = Set.of(Constants.PARAM_TEXT, Constants.PARAM_CONTENT);
+
 	public ResourceIndexedSearchParams() {
 	}
 
@@ -344,11 +348,15 @@ public final class ResourceIndexedSearchParams {
 	private <RT extends BaseResourceIndexedSearchParam> void findMissingSearchParams(PartitionSettings thePartitionSettings, StorageSettings theStorageSettings, ResourceTable theEntity, ResourceSearchParams activeSearchParams, RestSearchParameterTypeEnum type,
 																												Collection<RT> paramCollection) {
 		for (String nextParamName : activeSearchParams.getSearchParamNames()) {
-			if (nextParamName == null || nextParamName.startsWith("_")) {
+			if (nextParamName == null || myIgnoredParams.contains(nextParamName)) {
 				continue;
 			}
 
 			RuntimeSearchParam searchParam = activeSearchParams.get(nextParamName);
+			if (RuntimeSearchParamHelper.isResourceLevel(searchParam)) {
+				continue;
+			}
+
 			if (searchParam.getParamType() == type) {
 				boolean haveParam = false;
 				for (BaseResourceIndexedSearchParam nextParam : paramCollection) {

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/RuntimeSearchParamHelper.java

@@ -0,0 +1,18 @@
+package ca.uhn.fhir.jpa.searchparam.util;
+
+import ca.uhn.fhir.context.RuntimeSearchParam;
+
+import static org.apache.commons.lang3.StringUtils.startsWith;
+
+public class RuntimeSearchParamHelper {
+
+	/**
+	 * Helper function to determine if a RuntimeSearchParam is a resource level search param.
+	 *
+	 * @param theSearchParam the parameter to check
+	 * @return return boolean
+	 */
+	public static boolean isResourceLevel(RuntimeSearchParam theSearchParam) {
+		return startsWith(theSearchParam.getPath(), "Resource.");
+	}
+}

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/imprt2/ConsumeFilesStepR4Test.java

@@ -17,7 +17,6 @@ import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.springframework.beans.factory.annotation.Autowired;
 
-import javax.servlet.ServletException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java

@@ -3,10 +3,14 @@ package ca.uhn.fhir.jpa.provider.r4;
 import ca.uhn.fhir.i18n.HapiLocalizer;
 import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
 import ca.uhn.fhir.jpa.dao.data.ISearchDao;
 import ca.uhn.fhir.jpa.entity.Search;
 import ca.uhn.fhir.jpa.model.entity.NormalizedQuantitySearchLevel;
 import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
+import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
+import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri;
+import ca.uhn.fhir.jpa.model.entity.StorageSettings;
 import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.model.util.UcumServiceUtil;
 import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test;
@@ -27,6 +31,7 @@ import ca.uhn.fhir.rest.api.MethodOutcome;
 import ca.uhn.fhir.rest.api.PreferReturnEnum;
 import ca.uhn.fhir.rest.api.SearchTotalModeEnum;
 import ca.uhn.fhir.rest.api.SummaryEnum;
+import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.client.apache.ResourceEntity;
 import ca.uhn.fhir.rest.client.api.IClientInterceptor;
 import ca.uhn.fhir.rest.client.api.IGenericClient;
@@ -199,11 +204,13 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasProperty;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.matchesPattern;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 import static org.hamcrest.Matchers.stringContainsInOrder;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -337,6 +344,61 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 		}
 	}
 
+	@Test
+	public void testSearchResourcesOnProfile_whenProfileIsMissing_returnsResourcesWithMissingProfile(){
+		// given
+		myStorageSettings.setIndexMissingFields(StorageSettings.IndexEnabledEnum.ENABLED);
+		myStorageSettings.setTagStorageMode(JpaStorageSettings.TagStorageModeEnum.INLINE);
+
+		SearchParameter searchParameter = new SearchParameter();
+		searchParameter.addBase("Organization");
+		searchParameter.setCode("_profile");
+		searchParameter.setType(Enumerations.SearchParamType.URI);
+		searchParameter.setExpression("meta.profile");
+		searchParameter.setStatus(Enumerations.PublicationStatus.ACTIVE);
+
+		IFhirResourceDao<SearchParameter> searchParameterDao = myDaoRegistry.getResourceDao(SearchParameter.class);
+		searchParameterDao.create(searchParameter, (RequestDetails) null);
+
+		IFhirResourceDao<Organization> organizationDao = myDaoRegistry.getResourceDao(Organization.class);
+		Organization organizationWithNoProfile = new Organization();
+		organizationWithNoProfile.setName("noProfile");
+		organizationDao.create(organizationWithNoProfile);
+
+		Organization organizationWithProfile = new Organization();
+		organizationWithProfile.setName("withProfile");
+		organizationWithProfile.getMeta().addProfile("http://foo");
+		organizationDao.create(organizationWithProfile);
+
+		runInTransaction(() -> {
+			List<ResourceIndexedSearchParamUri> matched = myResourceIndexedSearchParamUriDao.findAll().stream()
+				.filter(theRow -> theRow.getResourceType().equals("Organization"))
+				.filter(theRow -> theRow.getParamName().equals("_profile"))
+				.collect(Collectors.toList());
+
+			assertThat(matched, hasSize(2));
+			assertThat(matched, hasItems(
+				hasProperty("uri", is(nullValue())),
+				hasProperty("uri", is("http://foo"))
+			));
+		});
+
+		// when
+		runInTransaction(() -> {
+			List<ResourceIndexedSearchParamUri> matched = myResourceIndexedSearchParamUriDao.findAll().stream()
+				.filter(theRow -> theRow.getResourceType().equals("Organization"))
+				.filter(theRow -> theRow.getParamName().equals("_profile"))
+				.filter(theRow -> theRow.isMissing() == true)
+				.collect(Collectors.toList());
+
+			// then
+			assertThat(matched, hasSize(1));
+			assertThat(matched, hasItem(
+				hasProperty("uri", is(nullValue()))
+			));
+		});
+	}
+
 	@Test
 	public void testSearchForTokenValueOnlyUsesValueHash() {
 

hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/search/reindex/InstanceReindexServiceImplR5Test.java

@@ -72,6 +72,7 @@ public class InstanceReindexServiceImplR5Test extends BaseJpaR5Test {
 			.sorted()
 			.toList();
 		assertThat(indexInstances.toString(), indexInstances, contains(
+			"_id NO_CHANGE Token true",
 			"active NO_CHANGE Token true",
 			"address NO_CHANGE String true",
 			"address-city NO_CHANGE String true",