increase string value storage length to accomodate larger contains searches (#6037)

* wip test writing

* add test to ES suite as well

* wip

* Changelog

* spotless

* Address review comments

* Test corrections

* Fixes for tests

* Fix changelog

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6036-increase-value-length.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 6036
+jira: SMILE-8605
+title: "The maximum length of `SP_VALUE_EXACT` and `SP_VALUE_NORMALIZED` in the `HFJ_SPIDX_STRING` table has been increased from 200 to 768. This allows for longer `:contains` searches. If you have previously stored values longer than 200 characters in String search 
+parameters and want to perform a `:contains` search on the new longer maximum size, a reindex of the affected data is required."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java

@@ -439,18 +439,34 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 											t -> ResourceIndexedComboStringUnique.calculateHashComplete2(
 													t.getString("IDX_STRING")))
 									.setColumnName("HASH_COMPLETE"));
-		}
 
-		{
+			{
-			version.onTable("TRM_CONCEPT_DESIG")
+				version.onTable("TRM_CONCEPT_DESIG")
-					.modifyColumn("20240705.10", "VAL")
+						.modifyColumn("20240705.10", "VAL")
-					.nullable()
+						.nullable()
-					.withType(ColumnTypeEnum.STRING, 2000);
+						.withType(ColumnTypeEnum.STRING, 2000);
 
-			version.onTable("TRM_CONCEPT_DESIG")
+				version.onTable("TRM_CONCEPT_DESIG")
-					.addColumn("20240705.20", "VAL_VC")
+						.addColumn("20240705.20", "VAL_VC")
-					.nullable()
+						.nullable()
-					.type(ColumnTypeEnum.TEXT);
+						.type(ColumnTypeEnum.TEXT);
+			}
+			{ // These migrations permit much longer values to be stored in SPIDX_TOKEN and SPIDX_STRING value
+				Builder.BuilderWithTableName spidxString = version.onTable("HFJ_SPIDX_STRING");
+				// components.
+				// This is mostly helpful for `:contains` searches on long values, since exact searches use the hash
+				// anyhow.
+				spidxString
+						.modifyColumn("20240708.10", "SP_VALUE_EXACT")
+						.nullable()
+						.withType(ColumnTypeEnum.STRING, 768)
+						.failureAllowed();
+				spidxString
+						.modifyColumn("20240708.20", "SP_VALUE_NORMALIZED")
+						.nullable()
+						.withType(ColumnTypeEnum.STRING, 768)
+						.failureAllowed();
+			}
 		}
 	}
 

hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java

@@ -91,6 +91,8 @@ import org.hl7.fhir.r4.model.Questionnaire;
 import org.hl7.fhir.r4.model.QuestionnaireResponse;
 import org.hl7.fhir.r4.model.Reference;
 import org.hl7.fhir.r4.model.RiskAssessment;
+import org.hl7.fhir.r4.model.SearchParameter;
+import org.hl7.fhir.r4.model.ServiceRequest;
 import org.hl7.fhir.r4.model.StringType;
 import org.hl7.fhir.r4.model.ValueSet;
 import org.junit.jupiter.api.AfterEach;
@@ -222,6 +224,12 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
 	@Qualifier("myQuestionnaireResponseDaoR4")
 	private IFhirResourceDao<QuestionnaireResponse> myQuestionnaireResponseDao;
 	@Autowired
+	@Qualifier("myServiceRequestDaoR4")
+	private IFhirResourceDao<ServiceRequest> myServiceRequestDao;
+	@Autowired
+	@Qualifier("mySearchParameterDaoR4")
+	private IFhirResourceDao<SearchParameter> mySearchParameterDao;
+	@Autowired
 	private TestHSearchEventDispatcher myHSearchEventDispatcher;
 	@Autowired
 	ElasticsearchContainer myElasticsearchContainer;
@@ -285,6 +293,8 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
 			return messages;
 		}
 	}
+
+
 	@Test
 	public void testFullTextSearchesArePerformanceLogged() {
 

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamString.java

@@ -73,7 +73,7 @@ public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchP
 	/*
 	 * Note that MYSQL chokes on unique indexes for lengths > 255 so be careful here
 	 */
-	public static final int MAX_LENGTH = 200;
+	public static final int MAX_LENGTH = 768;
 	public static final int HASH_PREFIX_LENGTH = 1;
 	private static final long serialVersionUID = 1L;
 

hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchNoFtTest.java

@@ -1437,7 +1437,7 @@ public class FhirResourceDaoDstu2SearchNoFtTest extends BaseJpaDstu2Test {
 	@Test
 	public void testSearchStringParamReallyLong() {
 		String methodName = "testSearchStringParamReallyLong";
-		String value = StringUtils.rightPad(methodName, 200, 'a');
+		String value = StringUtils.rightPad(methodName, ResourceIndexedSearchParamString.MAX_LENGTH, 'a');
 
 		IIdType longId;
 		IIdType shortId;

hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java

@@ -2645,7 +2645,7 @@ public class FhirResourceDaoDstu2Test extends BaseJpaDstu2Test {
 		myStorageSettings.setAdvancedHSearchIndexing(false);
 
 		Organization org = new Organization();
-		String str = "testStringParamLong__lvdaoy843s89tll8gvs89l4s3gelrukveilufyebrew8r87bv4b77feli7fsl4lv3vb7rexloxe7olb48vov4o78ls7bvo7vb48o48l4bb7vbvx";
+		String str = "testStringParamLong__" + RandomStringUtils.randomAlphabetic(ResourceIndexedSearchParamString.MAX_LENGTH);
 		str = str + str;
 		org.getNameElement().setValue(str);
 
@@ -2764,8 +2764,8 @@ public class FhirResourceDaoDstu2Test extends BaseJpaDstu2Test {
 		org.getNameElement().setValue("testTokenParamWhichIsTooLong");
 		org.getType().addCoding().setSystem(longStr1).setCode(longStr2);
 
-		String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
+		String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
-		String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
+		String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
 		List<IResourcePersistentId> val = myOrganizationDao.searchForIds(new SearchParameterMap("type", new IdentifierDt(subStr1, subStr2)), null);
 		int initial = val.size();
 

hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java

@@ -2023,7 +2023,7 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test {
 	@Test
 	public void testSearchStringParamReallyLong() {
 		String methodName = "testSearchStringParamReallyLong";
-		String value = StringUtils.rightPad(methodName, 200, 'a');
+		String value = StringUtils.rightPad(methodName, ResourceIndexedSearchParamString.MAX_LENGTH, 'a');
 
 		IIdType longId;
 		IIdType shortId;

hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3Test.java

@@ -3236,7 +3236,7 @@ public class FhirResourceDaoDstu3Test extends BaseJpaDstu3Test {
 	public void testStringParamWhichIsTooLong() {
 
 		Organization org = new Organization();
-		String str = "testStringParamLong__lvdaoy843s89tll8gvs89l4s3gelrukveilufyebrew8r87bv4b77feli7fsl4lv3vb7rexloxe7olb48vov4o78ls7bvo7vb48o48l4bb7vbvx";
+		String str = "testStringParamLong__" + RandomStringUtils.randomAlphanumeric(ResourceIndexedSearchParamString.MAX_LENGTH);
 		str = str + str;
 		org.getNameElement().setValue(str);
 
@@ -3419,8 +3419,8 @@ public class FhirResourceDaoDstu3Test extends BaseJpaDstu3Test {
 		org.getNameElement().setValue("testTokenParamWhichIsTooLong");
 		org.addType().addCoding().setSystem(longStr1).setCode(longStr2);
 
-		String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
+		String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
-		String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
+		String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
 		List<IResourcePersistentId> val = myOrganizationDao.searchForIds(new SearchParameterMap("type", new TokenParam(subStr1, subStr2)), null);
 		int initial = val.size();
 

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchCustomSearchParamTest.java

@@ -11,6 +11,7 @@ import ca.uhn.fhir.interceptor.api.Pointcut;
 import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
 import ca.uhn.fhir.jpa.model.entity.NormalizedQuantitySearchLevel;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
+import ca.uhn.fhir.jpa.model.entity.StorageSettings;
 import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.test.BaseJpaR4Test;
@@ -21,6 +22,7 @@ import ca.uhn.fhir.rest.param.NumberParam;
 import ca.uhn.fhir.rest.param.ReferenceOrListParam;
 import ca.uhn.fhir.rest.param.ReferenceParam;
 import ca.uhn.fhir.rest.param.StringParam;
+import ca.uhn.fhir.rest.param.TokenOrListParam;
 import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
@@ -581,6 +583,40 @@ public class FhirResourceDaoR4SearchCustomSearchParamTest extends BaseJpaR4Test
 		assertEquals(RuntimeSearchParam.RuntimeSearchParamStatusEnum.ACTIVE, sp.getStatus());
 	}
 
+	@Test
+	public void testSearchParamStringOnExtensionForVeryLongContainsSearch() {
+		myStorageSettings.setAllowContainsSearches(true);
+		String body = "{\n" +
+			"  \"resourceType\": \"SearchParameter\",\n" +
+			"  \"url\": \"https://health.gov.on.ca/idms/fhir/SearchParameter/ServiceRequest-Indication\",\n" +
+			"  \"title\": \"ServiceRequest Indication\",\n" +
+			"  \"status\": \"active\",\n" +
+			"  \"publisher\": \"MOH-IDMS\",\n" +
+			"  \"code\": \"ServiceRequestIndication\",\n" +
+			"  \"base\": [\n" +
+			"    \"ServiceRequest\"\n" +
+			"  ],\n" +
+			"  \"type\": \"string\",\n" +
+			"  \"expression\": \"ServiceRequest.extension('https://health.gov.on.ca/idms/fhir/StructureDefinition/Extension-Indication')\"\n" +
+			"}";
+		SearchParameter searchParameter = myFhirContext.newJsonParser().parseResource(SearchParameter.class, body);
+
+		mySearchParameterDao.create(searchParameter, mySrd);
+		mySearchParamRegistry.forceRefresh();
+
+		ServiceRequest sr = new ServiceRequest();
+		sr.addExtension().setUrl("https://health.gov.on.ca/idms/fhir/StructureDefinition/Extension-Indication").setValue(new StringType("Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapib"));
+
+		myServiceRequestDao.create(sr, mySrd);
+
+		SearchParameterMap searchParameter1 = new SearchParameterMap();
+		searchParameter1.add("ServiceRequestIndication", new StringParam("Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapib").setContains(true));
+		IBundleProvider search = myServiceRequestDao.search(searchParameter1, mySrd);
+		assertThat(search.size()).isEqualTo(1);
+
+		myStorageSettings.setAllowContainsSearches(new StorageSettings().isAllowContainsSearches());
+	}
+
 
 	@Test
 	public void testOverrideAndDisableBuiltInSearchParametersWithOverridingDisabled() {

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java

@@ -3723,7 +3723,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
 	@Test
 	public void testSearchStringParamReallyLong() {
 		String methodName = "testSearchStringParamReallyLong";
-		String value = StringUtils.rightPad(methodName, 200, 'a');
+		String value = StringUtils.rightPad(methodName, ResourceIndexedSearchParamString.MAX_LENGTH, 'a');
 
 		IIdType longId;
 		IIdType shortId;

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java

@@ -3954,7 +3954,7 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test {
 	public void testStringParamWhichIsTooLong() {
 
 		Organization org = new Organization();
-		String str = "testStringParamLong__lvdaoy843s89tll8gvs89l4s3gelrukveilufyebrew8r87bv4b77feli7fsl4lv3vb7rexloxe7olb48vov4o78ls7bvo7vb48o48l4bb7vbvx";
+		String str = "testStringParamLong__" + RandomStringUtils.randomAlphanumeric(ResourceIndexedSearchParamString.MAX_LENGTH);
 		str = str + str;
 		org.getNameElement().setValue(str);
 
@@ -4137,8 +4137,8 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test {
 		org.getNameElement().setValue("testTokenParamWhichIsTooLong");
 		org.addType().addCoding().setSystem(longStr1).setCode(longStr2);
 
-		String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
+		String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
-		String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
+		String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
 		List<IResourcePersistentId> val = myOrganizationDao.searchForIds(new SearchParameterMap("type", new TokenParam(subStr1, subStr2)), null);
 		int initial = val.size();
 

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/subscription/module/matcher/InMemorySubscriptionMatcherR4Test.java

@@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
+import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
 import ca.uhn.fhir.jpa.model.entity.StorageSettings;
 import ca.uhn.fhir.jpa.model.entity.NormalizedQuantitySearchLevel;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString;
@@ -725,7 +726,7 @@ public class InMemorySubscriptionMatcherR4Test {
 	@Test
 	public void testSearchStringParamReallyLong() {
 		String methodName = "testSearchStringParamReallyLong";
-		String value = StringUtils.rightPad(methodName, 200, 'a');
+		String value = StringUtils.rightPad(methodName, ResourceIndexedSearchParamString.MAX_LENGTH, 'a');
 
 		Patient patient = new Patient();
 		patient.addIdentifier().setSystem("urn:system").setValue("001");