4115 Combining 'indexMissingFields' and 'advancedHSearchIndexing' functionalities will prevent creating a resource (#4122)

* Added Test

* Added Integration Test

* Made changes based on comments

* Implemented solution

* Implemented solution

* Debug

* Changelog

* Changes made based on comments

* Changelog

* Made changes based on comments

* Made changes based on comments

Co-authored-by: Karneet Kaur <karneet.kaur@smilecdr.com>

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4115-combining-indexmissingfields-and-advancedhsearchindexing-functionalities-will-prevent-creating-a-resource.yaml

@@ -0,0 +1,7 @@
+---
+type: fix
+issue: 4115
+title: "Previously, enabling 'indexMissingFields' and 'advancedHSearchIndexing'
+      functionalities would cause creating a Patient resource to return HTTP 500. 
+      Upon further investigation, the same happened for creating an Observation. 
+      This has been fixed."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchIndexExtractor.java

@@ -87,18 +87,25 @@ public class ExtendedHSearchIndexExtractor {
 		// wipmb mb add a flag ot DaoConfig to suppress this
 		extractAutocompleteTokens(theResource, retVal);
 
-		theNewParams.myStringParams.forEach(nextParam ->
+		theNewParams.myStringParams.stream()
-			retVal.addStringIndexData(nextParam.getParamName(), nextParam.getValueExact()));
+			.filter(nextParam -> !nextParam.isMissing())
+			.forEach(nextParam -> retVal.addStringIndexData(nextParam.getParamName(), nextParam.getValueExact()));
 
-		theNewParams.myTokenParams.forEach(nextParam ->
+		theNewParams.myTokenParams.stream()
-			retVal.addTokenIndexDataIfNotPresent(nextParam.getParamName(), nextParam.getSystem(), nextParam.getValue()));
+			.filter(nextParam -> !nextParam.isMissing())
+			.forEach(nextParam -> retVal.addTokenIndexDataIfNotPresent(nextParam.getParamName(), nextParam.getSystem(), nextParam.getValue()));
 
-		theNewParams.myNumberParams.forEach(nextParam ->
+		theNewParams.myNumberParams.stream()
-			retVal.addNumberIndexDataIfNotPresent(nextParam.getParamName(), nextParam.getValue()));
+			.filter(nextParam -> !nextParam.isMissing())
+			.forEach(nextParam -> retVal.addNumberIndexDataIfNotPresent(nextParam.getParamName(), nextParam.getValue()));
 
-		theNewParams.myDateParams.forEach(nextParam -> retVal.addDateIndexData(nextParam.getParamName(), convertDate(nextParam)));
+		theNewParams.myDateParams.stream()
+			.filter(nextParam -> !nextParam.isMissing())
+			.forEach(nextParam -> retVal.addDateIndexData(nextParam.getParamName(), convertDate(nextParam)));
 
-		theNewParams.myQuantityParams.forEach(nextParam -> retVal.addQuantityIndexData(nextParam.getParamName(), convertQuantity(nextParam)));
+		theNewParams.myQuantityParams.stream()
+			.filter(nextParam -> !nextParam.isMissing())
+			.forEach(nextParam -> retVal.addQuantityIndexData(nextParam.getParamName(), convertQuantity(nextParam)));
 
 		theResource.getMeta().getTag().forEach(tag ->
 			retVal.addTokenIndexData("_tag", tag));

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchSearchBuilder.java

@@ -40,6 +40,7 @@ import com.google.common.collect.Sets;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
+import static ca.uhn.fhir.rest.api.Constants.PARAMQUALIFIER_MISSING;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -127,7 +128,12 @@ public class ExtendedHSearchSearchBuilder {
 			return modifier.equals(EMPTY_MODIFIER);
 
 		} else if (param instanceof CompositeParam) {
+			switch(modifier) {
+				case PARAMQUALIFIER_MISSING:
+					return false;
+				default:
 					return true;
+			}
 
 		} else if (param instanceof ReferenceParam) {
 			//We cannot search by chain.

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/ExtendedHSearchIndexData.java

@@ -141,6 +141,8 @@ public class ExtendedHSearchIndexData {
 		mySearchParamDates.put(theSpName, value);
 	}
 
+	public SetMultimap<String, DateSearchIndexData> getDateIndexData() { return mySearchParamDates; }
+
 	public void addNumberIndexDataIfNotPresent(String theParamName, BigDecimal theValue) {
 		mySearchParamNumbers.put(theParamName, theValue);
 	}
@@ -149,6 +151,8 @@ public class ExtendedHSearchIndexData {
 		mySearchParamQuantities.put(theSpName, value);
 	}
 
+	public SetMultimap<String,QuantitySearchIndexData> getQuantityIndexData () {return mySearchParamQuantities;}
+
 	public void setForcedId(String theForcedId) {
 		myForcedId = theForcedId;
 	}

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java

@@ -240,6 +240,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 		myDaoConfig.setSearchPreFetchThresholds(new DaoConfig().getSearchPreFetchThresholds());
 		myDaoConfig.setAllowContainsSearches(new DaoConfig().isAllowContainsSearches());
 		myDaoConfig.setIndexMissingFields(new DaoConfig().getIndexMissingFields());
+		myDaoConfig.setAdvancedHSearchIndexing(new DaoConfig().isAdvancedHSearchIndexing());
 
 		mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null);
 		mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE);
@@ -6167,6 +6168,34 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 		}
 	}
 
+	@Test
+	public void testCreateResourcesWithAdvancedHSearchIndexingAndIndexMissingFieldsEnableSucceeds() throws Exception {
+		myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.ENABLED);
+		myDaoConfig.setAdvancedHSearchIndexing(true);
+		String identifierValue = "someValue";
+		String searchPatientURIWithMissingBirthdate = "Patient?birthdate:missing=true";
+		String searchObsURIWithMissingValueQuantity = "Observation?value-quantity:missing=true";
+
+		//create patient
+		Patient patient = new Patient();
+		patient.addIdentifier().setSystem("urn:system").setValue(identifierValue);
+		MethodOutcome outcome = myClient.create().resource(patient).execute();
+		assertTrue(outcome.getCreated());
+
+		//create observation
+		Observation obs = new Observation();
+		obs.addIdentifier().setSystem("urn:system").setValue(identifierValue);
+		outcome = myClient.create().resource(obs).execute();
+		assertTrue(outcome.getCreated());
+
+		// search
+		Bundle patientsWithMissingBirthdate = myClient.search().byUrl(searchPatientURIWithMissingBirthdate).returnBundle(Bundle.class).execute();
+		assertEquals(1, patientsWithMissingBirthdate.getTotal());
+		Bundle obsWithMissingValueQuantity = myClient.search().byUrl(searchObsURIWithMissingValueQuantity).returnBundle(Bundle.class).execute();
+		assertEquals(1, obsWithMissingValueQuantity.getTotal());
+
+	}
+
 	@Test
 	public void testTryToCreateResourceWithReferenceThatDoesntExist() {
 		Patient p1 = new Patient();

hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchIndexExtractorTest.java

@@ -5,9 +5,13 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.model.config.PartitionSettings;
 import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
 import ca.uhn.fhir.jpa.model.entity.ModelConfig;
+import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate;
+import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantity;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
 import ca.uhn.fhir.jpa.model.search.CompositeSearchIndexData;
+import ca.uhn.fhir.jpa.model.search.DateSearchIndexData;
 import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
+import ca.uhn.fhir.jpa.model.search.QuantitySearchIndexData;
 import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor;
 import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParamComposite;
 import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
@@ -17,6 +21,7 @@ import ca.uhn.fhir.rest.server.util.FhirContextSearchParamRegistry;
 import ca.uhn.fhir.rest.server.util.ResourceSearchParams;
 import ca.uhn.fhir.test.utilities.ITestDataBuilder;
 import org.hl7.fhir.r4.model.Observation;
+import org.hl7.fhir.r4.model.SearchParameter;
 import org.junit.jupiter.api.Test;
 
 import java.util.Set;
@@ -59,6 +64,32 @@ class ExtendedHSearchIndexExtractorTest implements ITestDataBuilder.WithSupport
 		assertThat(spIndexData, hasSize(1));
 	}
 
+	@Test
+	void testExtract_withParamMarkedAsMissing_willBeIgnored() {
+		//setup
+		ResourceIndexedSearchParams searchParams = new ResourceIndexedSearchParams();
+		ResourceIndexedSearchParamDate searchParamDate = new ResourceIndexedSearchParamDate(new PartitionSettings(), "SearchParameter", "Date", null, null, null, null, null);
+		searchParamDate.setMissing(true);
+		searchParams.myDateParams.add(searchParamDate);
+
+		ResourceIndexedSearchParamQuantity searchParamQuantity = new ResourceIndexedSearchParamQuantity(new PartitionSettings(), "SearchParameter","Quantity", null, null,null);
+		searchParamQuantity.setMissing(true);
+		searchParams.myQuantityParams.add(searchParamQuantity);
+
+		// run: now translate to HSearch
+		ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams("Patient");
+		ExtendedHSearchIndexExtractor extractor = new ExtendedHSearchIndexExtractor(
+			myDaoConfig, myFhirContext, activeSearchParams, mySearchParamExtractor, myModelConfig);
+		ExtendedHSearchIndexData indexData = extractor.extract(new SearchParameter(), searchParams);
+
+		// validate
+		Set<DateSearchIndexData> dIndexData = indexData.getDateIndexData().get("Date");
+		assertThat(dIndexData, hasSize(0));
+		Set<QuantitySearchIndexData> qIndexData = indexData.getQuantityIndexData().get("Quantity");
+		assertThat(qIndexData, hasSize(0));
+
+	}
+
 	@Override
 	public Support getTestDataBuilderSupport() {
 		return new SupportNoDao(myFhirContext);