honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

increase string value storage length to accomodate larger contains searches (#6037) 

* wip test writing

* add test to ES suite as well

* wip

* Changelog

* spotless

* Address review comments

* Test corrections

* Fixes for tests

* Fix changelog
This commit is contained in:
Tadgh 2024-07-08 23:54:24 -07:00 committed by GitHub
parent fee1b75242
commit 3ffb695b6b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 93 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6036-increase-value-length.yaml Normal file
View File

@ -0,0 +1,6 @@
---
type: fix
issue: 6036
jira: SMILE-8605
title: "The maximum length of `SP_VALUE_EXACT` and `SP_VALUE_NORMALIZED` in the `HFJ_SPIDX_STRING` table has been increased from 200 to 768. This allows for longer `:contains` searches. If you have previously stored values longer than 200 characters in String search
parameters and want to perform a `:contains` search on the new longer maximum size, a reindex of the affected data is required."

36
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java
View File

@ -439,18 +439,34 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
t -> ResourceIndexedComboStringUnique.calculateHashComplete2(
t.getString("IDX_STRING")))
.setColumnName("HASH_COMPLETE"));
}
{
version.onTable("TRM_CONCEPT_DESIG")
.modifyColumn("20240705.10", "VAL")
.nullable()
.withType(ColumnTypeEnum.STRING, 2000);
{
version.onTable("TRM_CONCEPT_DESIG")
.modifyColumn("20240705.10", "VAL")
.nullable()
.withType(ColumnTypeEnum.STRING, 2000);
version.onTable("TRM_CONCEPT_DESIG")
.addColumn("20240705.20", "VAL_VC")
.nullable()
.type(ColumnTypeEnum.TEXT);
version.onTable("TRM_CONCEPT_DESIG")
.addColumn("20240705.20", "VAL_VC")
.nullable()
.type(ColumnTypeEnum.TEXT);
}
{ // These migrations permit much longer values to be stored in SPIDX_TOKEN and SPIDX_STRING value
Builder.BuilderWithTableName spidxString = version.onTable("HFJ_SPIDX_STRING");
// components.
// This is mostly helpful for `:contains` searches on long values, since exact searches use the hash
// anyhow.
spidxString
.modifyColumn("20240708.10", "SP_VALUE_EXACT")
.nullable()
.withType(ColumnTypeEnum.STRING, 768)
.failureAllowed();
spidxString
.modifyColumn("20240708.20", "SP_VALUE_NORMALIZED")
.nullable()
.withType(ColumnTypeEnum.STRING, 768)
.failureAllowed();
}
}
}

10
hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java
View File

@ -91,6 +91,8 @@ import org.hl7.fhir.r4.model.Questionnaire;
import org.hl7.fhir.r4.model.QuestionnaireResponse;
import org.hl7.fhir.r4.model.Reference;
import org.hl7.fhir.r4.model.RiskAssessment;
import org.hl7.fhir.r4.model.SearchParameter;
import org.hl7.fhir.r4.model.ServiceRequest;
import org.hl7.fhir.r4.model.StringType;
import org.hl7.fhir.r4.model.ValueSet;
import org.junit.jupiter.api.AfterEach;
@ -222,6 +224,12 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
@Qualifier("myQuestionnaireResponseDaoR4")
private IFhirResourceDao<QuestionnaireResponse> myQuestionnaireResponseDao;
@Autowired
@Qualifier("myServiceRequestDaoR4")
private IFhirResourceDao<ServiceRequest> myServiceRequestDao;
@Autowired
@Qualifier("mySearchParameterDaoR4")
private IFhirResourceDao<SearchParameter> mySearchParameterDao;
@Autowired
private TestHSearchEventDispatcher myHSearchEventDispatcher;
@Autowired
ElasticsearchContainer myElasticsearchContainer;
@ -285,6 +293,8 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
return messages;
}
}
@Test
public void testFullTextSearchesArePerformanceLogged() {

2
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamString.java
View File

@ -73,7 +73,7 @@ public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchP
/*
* Note that MYSQL chokes on unique indexes for lengths > 255 so be careful here
*/
public static final int MAX_LENGTH = 200;
public static final int MAX_LENGTH = 768;
public static final int HASH_PREFIX_LENGTH = 1;
private static final long serialVersionUID = 1L;

2
hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchNoFtTest.java
View File

@ -1437,7 +1437,7 @@ public class FhirResourceDaoDstu2SearchNoFtTest extends BaseJpaDstu2Test {
@Test
public void testSearchStringParamReallyLong() {
String methodName = "testSearchStringParamReallyLong";
String value = StringUtils.rightPad(methodName, 200, 'a');
String value = StringUtils.rightPad(methodName, ResourceIndexedSearchParamString.MAX_LENGTH, 'a');
IIdType longId;
IIdType shortId;

6
hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java
View File

@ -2645,7 +2645,7 @@ public class FhirResourceDaoDstu2Test extends BaseJpaDstu2Test {
myStorageSettings.setAdvancedHSearchIndexing(false);
Organization org = new Organization();
String str = "testStringParamLong__lvdaoy843s89tll8gvs89l4s3gelrukveilufyebrew8r87bv4b77feli7fsl4lv3vb7rexloxe7olb48vov4o78ls7bvo7vb48o48l4bb7vbvx";
String str = "testStringParamLong__" + RandomStringUtils.randomAlphabetic(ResourceIndexedSearchParamString.MAX_LENGTH);
str = str + str;
org.getNameElement().setValue(str);
@ -2764,8 +2764,8 @@ public class FhirResourceDaoDstu2Test extends BaseJpaDstu2Test {
org.getNameElement().setValue("testTokenParamWhichIsTooLong");
org.getType().addCoding().setSystem(longStr1).setCode(longStr2);
String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
List<IResourcePersistentId> val = myOrganizationDao.searchForIds(new SearchParameterMap("type", new IdentifierDt(subStr1, subStr2)), null);
int initial = val.size();

2
hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java
View File

@ -2023,7 +2023,7 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test {
@Test
public void testSearchStringParamReallyLong() {
String methodName = "testSearchStringParamReallyLong";
String value = StringUtils.rightPad(methodName, 200, 'a');
String value = StringUtils.rightPad(methodName, ResourceIndexedSearchParamString.MAX_LENGTH, 'a');
IIdType longId;
IIdType shortId;

6
hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3Test.java
View File

@ -3236,7 +3236,7 @@ public class FhirResourceDaoDstu3Test extends BaseJpaDstu3Test {
public void testStringParamWhichIsTooLong() {
Organization org = new Organization();
String str = "testStringParamLong__lvdaoy843s89tll8gvs89l4s3gelrukveilufyebrew8r87bv4b77feli7fsl4lv3vb7rexloxe7olb48vov4o78ls7bvo7vb48o48l4bb7vbvx";
String str = "testStringParamLong__" + RandomStringUtils.randomAlphanumeric(ResourceIndexedSearchParamString.MAX_LENGTH);
str = str + str;
org.getNameElement().setValue(str);
@ -3419,8 +3419,8 @@ public class FhirResourceDaoDstu3Test extends BaseJpaDstu3Test {
org.getNameElement().setValue("testTokenParamWhichIsTooLong");
org.addType().addCoding().setSystem(longStr1).setCode(longStr2);
String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
List<IResourcePersistentId> val = myOrganizationDao.searchForIds(new SearchParameterMap("type", new TokenParam(subStr1, subStr2)), null);
int initial = val.size();

36
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchCustomSearchParamTest.java
View File

@ -11,6 +11,7 @@ import ca.uhn.fhir.interceptor.api.Pointcut;
import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
import ca.uhn.fhir.jpa.model.entity.NormalizedQuantitySearchLevel;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
import ca.uhn.fhir.jpa.model.entity.StorageSettings;
import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.test.BaseJpaR4Test;
@ -21,6 +22,7 @@ import ca.uhn.fhir.rest.param.NumberParam;
import ca.uhn.fhir.rest.param.ReferenceOrListParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenOrListParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
@ -581,6 +583,40 @@ public class FhirResourceDaoR4SearchCustomSearchParamTest extends BaseJpaR4Test
assertEquals(RuntimeSearchParam.RuntimeSearchParamStatusEnum.ACTIVE, sp.getStatus());
}
@Test
public void testSearchParamStringOnExtensionForVeryLongContainsSearch() {
myStorageSettings.setAllowContainsSearches(true);
String body = "{\n" +
" \"resourceType\": \"SearchParameter\",\n" +
" \"url\": \"https://health.gov.on.ca/idms/fhir/SearchParameter/ServiceRequest-Indication\",\n" +
" \"title\": \"ServiceRequest Indication\",\n" +
" \"status\": \"active\",\n" +
" \"publisher\": \"MOH-IDMS\",\n" +
" \"code\": \"ServiceRequestIndication\",\n" +
" \"base\": [\n" +
" \"ServiceRequest\"\n" +
" ],\n" +
" \"type\": \"string\",\n" +
" \"expression\": \"ServiceRequest.extension('https://health.gov.on.ca/idms/fhir/StructureDefinition/Extension-Indication')\"\n" +
"}";
SearchParameter searchParameter = myFhirContext.newJsonParser().parseResource(SearchParameter.class, body);
mySearchParameterDao.create(searchParameter, mySrd);
mySearchParamRegistry.forceRefresh();
ServiceRequest sr = new ServiceRequest();
sr.addExtension().setUrl("https://health.gov.on.ca/idms/fhir/StructureDefinition/Extension-Indication").setValue(new StringType("Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapib"));
myServiceRequestDao.create(sr, mySrd);
SearchParameterMap searchParameter1 = new SearchParameterMap();
searchParameter1.add("ServiceRequestIndication", new StringParam("Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapib").setContains(true));
IBundleProvider search = myServiceRequestDao.search(searchParameter1, mySrd);
assertThat(search.size()).isEqualTo(1);
myStorageSettings.setAllowContainsSearches(new StorageSettings().isAllowContainsSearches());
}
@Test
public void testOverrideAndDisableBuiltInSearchParametersWithOverridingDisabled() {

2
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java
View File

@ -3723,7 +3723,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
@Test
public void testSearchStringParamReallyLong() {
String methodName = "testSearchStringParamReallyLong";
String value = StringUtils.rightPad(methodName, 200, 'a');
String value = StringUtils.rightPad(methodName, ResourceIndexedSearchParamString.MAX_LENGTH, 'a');
IIdType longId;
IIdType shortId;

6
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java
View File

@ -3954,7 +3954,7 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test {
public void testStringParamWhichIsTooLong() {
Organization org = new Organization();
String str = "testStringParamLong__lvdaoy843s89tll8gvs89l4s3gelrukveilufyebrew8r87bv4b77feli7fsl4lv3vb7rexloxe7olb48vov4o78ls7bvo7vb48o48l4bb7vbvx";
String str = "testStringParamLong__" + RandomStringUtils.randomAlphanumeric(ResourceIndexedSearchParamString.MAX_LENGTH);
str = str + str;
org.getNameElement().setValue(str);
@ -4137,8 +4137,8 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test {
org.getNameElement().setValue("testTokenParamWhichIsTooLong");
org.addType().addCoding().setSystem(longStr1).setCode(longStr2);
String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
String subStr1 = longStr1.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
String subStr2 = longStr2.substring(0, ResourceIndexedSearchParamToken.MAX_LENGTH);
List<IResourcePersistentId> val = myOrganizationDao.searchForIds(new SearchParameterMap("type", new TokenParam(subStr1, subStr2)), null);
int initial = val.size();

3
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/subscription/module/matcher/InMemorySubscriptionMatcherR4Test.java
View File

@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
import ca.uhn.fhir.jpa.model.entity.StorageSettings;
import ca.uhn.fhir.jpa.model.entity.NormalizedQuantitySearchLevel;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString;
@ -725,7 +726,7 @@ public class InMemorySubscriptionMatcherR4Test {
@Test
public void testSearchStringParamReallyLong() {
String methodName = "testSearchStringParamReallyLong";
String value = StringUtils.rightPad(methodName, 200, 'a');
String value = StringUtils.rightPad(methodName, ResourceIndexedSearchParamString.MAX_LENGTH, 'a');
Patient patient = new Patient();
patient.addIdentifier().setSystem("urn:system").setValue("001");