V2 index for string (#4629)

v2 indexing for String for when sorting.

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4629-uri-string-index.yaml

@@ -0,0 +1,4 @@
+---
+type: perf
+issue: 4629
+title: "String and URI indexing has been improved in some multi-clause queries."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/upgrade.md

@@ -0,0 +1,5 @@
+This release changes database indexing for string and uri SearchParameters.
+The database migration may take several minutes.
+These changes will be applied automatically on first startup.
+To avoid this delay on first startup, run the migration manually.
+

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceIndexedSearchParamUriDao.java

@@ -30,8 +30,8 @@ import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri;
 
 public interface IResourceIndexedSearchParamUriDao extends JpaRepository<ResourceIndexedSearchParamUri, Long>, IHapiFhirJpaRepository {
 	
-	@Query("SELECT DISTINCT p.myUri FROM ResourceIndexedSearchParamUri p WHERE p.myResourceType = :resource_type AND p.myParamName = :param_name")
+	@Query("SELECT DISTINCT p.myUri FROM ResourceIndexedSearchParamUri p WHERE p.myHashIdentity = :hash_identity")
-	public Collection<String> findAllByResourceTypeAndParamName(@Param("resource_type") String theResourceType, @Param("param_name") String theParamName);
+	public Collection<String> findAllByHashIdentity(@Param("hash_identity") long theHashIdentity);
 
 	@Modifying
 	@Query("delete from ResourceIndexedSearchParamUri t WHERE t.myResourcePid = :resid")

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java

@@ -135,6 +135,33 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 		resSearchUrlTable.addIndex("20230227.2", "IDX_RESSEARCHURL_RES").unique(false).withColumns("RES_ID");
 		resSearchUrlTable.addIndex("20230227.3", "IDX_RESSEARCHURL_TIME").unique(false).withColumns("CREATED_TIME");
 
+		{
+			// string search index
+			Builder.BuilderWithTableName stringTable = version.onTable("HFJ_SPIDX_STRING");
+
+			// add res_id to indentity to speed up sorts.
+			stringTable
+				.addIndex("20230303.1", "IDX_SP_STRING_HASH_IDENT_V2")
+				.unique(false)
+				.online(true)
+				.withColumns("HASH_IDENTITY", "RES_ID", "PARTITION_ID");
+			stringTable.dropIndexOnline("20230303.2", "IDX_SP_STRING_HASH_IDENT");
+
+			// add hash_norm to res_id to speed up joins on a second string.
+			stringTable
+				.addIndex("20230303.3", "IDX_SP_STRING_RESID_V2")
+				.unique(false)
+				.online(true)
+				.withColumns("RES_ID", "HASH_NORM_PREFIX", "PARTITION_ID");
+
+			// drop and recreate FK_SPIDXSTR_RESOURCE since it will be useing the old IDX_SP_STRING_RESID
+			stringTable.dropForeignKey("20230303.4", "FK_SPIDXSTR_RESOURCE", "HFJ_RESOURCE");
+			stringTable.dropIndexOnline("20230303.5", "IDX_SP_STRING_RESID");
+			stringTable.addForeignKey("20230303.6", "FK_SPIDXSTR_RESOURCE")
+				.toColumn("RES_ID").references("HFJ_RESOURCE", "RES_ID");
+
+		}
+
 		final String revColumnName = "REV";
 		final String enversRevisionTable = "HFJ_REVINFO";
 		final String enversMpiLinkAuditTable = "MPI_LINK_AUD";
@@ -207,6 +234,26 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
            .addColumn("20230323.1", "SEARCH_URL_PRESENT")
            .nullable()
            .type(ColumnTypeEnum.BOOLEAN);
+
+
+		{
+			Builder.BuilderWithTableName uriTable = version.onTable("HFJ_SPIDX_URI");
+			uriTable
+				.addIndex("20230324.1", "IDX_SP_URI_HASH_URI_V2")
+				.unique(true)
+				.online(true)
+				.withColumns("HASH_URI","RES_ID","PARTITION_ID");
+			uriTable
+				.addIndex("20230324.2", "IDX_SP_URI_HASH_IDENTITY_V2")
+				.unique(true)
+				.online(true)
+				.withColumns("HASH_IDENTITY","SP_URI","RES_ID","PARTITION_ID");
+			uriTable.dropIndex("20230324.3", "IDX_SP_URI_RESTYPE_NAME");
+			uriTable.dropIndex("20230324.4", "IDX_SP_URI_UPDATED");
+			uriTable.dropIndex("20230324.5", "IDX_SP_URI");
+			uriTable.dropIndex("20230324.6", "IDX_SP_URI_HASH_URI");
+			uriTable.dropIndex("20230324.7", "IDX_SP_URI_HASH_IDENTITY");
+		}
 	}
 
 	protected void init640() {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/UriPredicateBuilder.java

@@ -25,6 +25,7 @@ import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
 import ca.uhn.fhir.interceptor.api.Pointcut;
 import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao;
 import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser;
+import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri;
 import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
 import ca.uhn.fhir.jpa.util.QueryParameterUtils;
@@ -113,7 +114,8 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
 						.add(StorageProcessingMessage.class, message);
 					CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_WARNING, params);
 
-					Collection<String> candidates = myResourceIndexedSearchParamUriDao.findAllByResourceTypeAndParamName(getResourceType(), theParamName);
+					long hashIdentity = BaseResourceIndexedSearchParam.calculateHashIdentity(getPartitionSettings(), getRequestPartitionId(), getResourceType(), theParamName);
+					Collection<String> candidates = myResourceIndexedSearchParamUriDao.findAllByHashIdentity(hashIdentity);
 					List<String> toFind = new ArrayList<>();
 					for (String next : candidates) {
 						if (value.length() >= next.length()) {

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamString.java

@@ -56,12 +56,12 @@ import static org.apache.commons.lang3.StringUtils.defaultString;
 	 */
 
 	// This is used for sorting, and for :contains queries currently
-	@Index(name = "IDX_SP_STRING_HASH_IDENT", columnList = "HASH_IDENTITY"),
+	@Index(name = "IDX_SP_STRING_HASH_IDENT_V2", columnList = "HASH_IDENTITY,RES_ID,PARTITION_ID"),
 
 	@Index(name = "IDX_SP_STRING_HASH_NRM_V2", columnList = "HASH_NORM_PREFIX,SP_VALUE_NORMALIZED,RES_ID,PARTITION_ID"),
 	@Index(name = "IDX_SP_STRING_HASH_EXCT_V2", columnList = "HASH_EXACT,RES_ID,PARTITION_ID"),
 
-	@Index(name = "IDX_SP_STRING_RESID", columnList = "RES_ID")
+	@Index(name = "IDX_SP_STRING_RESID_V2", columnList = "RES_ID,HASH_NORM_PREFIX,PARTITION_ID")
 })
 public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchParam {
 

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamUri.java

@@ -48,11 +48,11 @@ import static org.apache.commons.lang3.StringUtils.defaultString;
 @Embeddable
 @Entity
 @Table(name = "HFJ_SPIDX_URI", indexes = {
-	@Index(name = "IDX_SP_URI", columnList = "RES_TYPE,SP_NAME,SP_URI"),
+	// for queries
-	@Index(name = "IDX_SP_URI_HASH_IDENTITY", columnList = "HASH_IDENTITY,SP_URI"),
+	@Index(name = "IDX_SP_URI_HASH_URI_V2", columnList = "HASH_URI,RES_ID,PARTITION_ID", unique = true),
-	@Index(name = "IDX_SP_URI_HASH_URI", columnList = "HASH_URI"),
+	// for sorting
-	@Index(name = "IDX_SP_URI_RESTYPE_NAME", columnList = "RES_TYPE,SP_NAME"),
+	@Index(name = "IDX_SP_URI_HASH_IDENTITY_V2", columnList = "HASH_IDENTITY,SP_URI,RES_ID,PARTITION_ID", unique = true),
-	@Index(name = "IDX_SP_URI_UPDATED", columnList = "SP_UPDATED"),
+	// for index create/delete
 	@Index(name = "IDX_SP_URI_COORDS", columnList = "RES_ID")
 })
 public class ResourceIndexedSearchParamUri extends BaseResourceIndexedSearchParam {

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4StandardQueriesNoFTTest.java

@@ -20,6 +20,8 @@ import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.test.annotation.DirtiesContext;
@@ -31,10 +33,12 @@ import java.util.List;
 import java.util.function.Consumer;
 
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.not;
+import static org.junit.jupiter.api.Assertions.fail;
 
 @ExtendWith(SpringExtension.class)
 @ContextConfiguration(classes = {
@@ -73,6 +77,87 @@ public class FhirResourceDaoR4StandardQueriesNoFTTest extends BaseJpaTest {
 	}
 
 
+	@Nested
+	public class StringSearch {
+		@ParameterizedTest
+		@CsvSource({
+			"normal search matches exact        , Flintstones,  =Flintstones,  True",
+			"normal search matches prefix       , Flintstones,  =Flints     ,  True",
+			"normal search matches upper prefix , Flintstones,  =FLINTS     ,  True",
+			"normal search matches lower prefix , Flintstones,  =flints     ,  True",
+			"normal search matches mixed prefix , Flintstones,  =fLiNtS     ,  True",
+			"normal search ignores accents      , Flíntstones,  =Flintstone ,  True",
+			"normal search no match suffix      , Flintstones,  =intstones  ,  False",
+			"normal search matches first letter , Flintstones,  =f          ,  True",
+			"exact  search matches exact        , Flintstones,  :exact=Flintstones ,  True",
+			"exact  search no match wrong case  , Flintstones,  :exact=flintstones ,  False",
+			"exact  search no match prefix      , Flintstones,  :exact=Flint       ,  False",
+//			"contains search match prefix      ,  Flintstones,  :contains=flint       ,  True",
+//			"contains search match prefix      ,  Flintstones,  :contains=Flint       ,  True",
+		})
+		void stringSearches(String theDescription, String theString, String theQuery, boolean theExpectMatchFlag) {
+		    // given
+			IIdType id = myDataBuilder.createPatient(myDataBuilder.withFamily(theString));
+
+		    // when
+			List<String> foundIds = myTestDaoSearch.searchForIds("Patient?name" + theQuery);
+
+			// then
+			if (theExpectMatchFlag) {
+				assertThat(theDescription, foundIds, hasItem(id.getIdPart()));
+			} else {
+				assertThat(theDescription, foundIds, not(hasItem(id.getIdPart())));
+			}
+		}
+
+		@Test
+		void searchTwoFields() {
+			// given
+			IIdType id = myDataBuilder.createPatient(
+				myDataBuilder.withGiven("Fred"),
+				myDataBuilder.withFamily("Flintstone"));
+
+			List<String> foundIds = myTestDaoSearch.searchForIds("Patient?family=flint&given:exact=Fred");
+
+			// then
+			assertThat(foundIds, hasItem(id.getIdPart()));		    // then
+		}
+
+		@Test
+		void sort() {
+			// given
+			String idWilma = myDataBuilder.createPatient(myDataBuilder.withGiven("Wilma"), myDataBuilder.withFamily("Flintstone")).getIdPart();
+			String idFred = myDataBuilder.createPatient(myDataBuilder.withGiven("Fred"), myDataBuilder.withFamily("Flintstone")).getIdPart();
+			String idBarney = myDataBuilder.createPatient(myDataBuilder.withGiven("Barney"), myDataBuilder.withFamily("Rubble")).getIdPart();
+			String idCoolFred = myDataBuilder.createPatient(myDataBuilder.withGiven("Fred"), myDataBuilder.withFamily("Jones")).getIdPart();
+			String idPolka = myDataBuilder.createPatient(myDataBuilder.withGiven("Polkaroo"), myDataBuilder.withFamily("Polkaroo")).getIdPart();
+
+			List<String> foundIds = myTestDaoSearch.searchForIds("Patient?_sort=family,given");
+
+			// then
+			assertThat(foundIds, contains(idFred, idWilma, idCoolFred, idPolka, idBarney));		    // then
+		}
+
+
+		@Test
+		void sortWithAge() {
+			// given
+			DaoTestDataBuilder b = myDataBuilder;
+			String idWilma = b.createPatient(
+				b.withGiven("Wilma"), b.withFamily("Flintstone"), b.withBirthdate("1945")).getIdPart();
+			String idFred = b.createPatient(b.withGiven("Fred"), b.withFamily("Flintstone"), b.withBirthdate("1940")).getIdPart();
+			String idBarney = b.createPatient(b.withGiven("Barney"), b.withFamily("Rubble"), b.withBirthdate("1941")).getIdPart();
+			String idCoolFred = b.createPatient(b.withGiven("Fred"), b.withFamily("Jones"), b.withBirthdate("1965")).getIdPart();
+			String idPolka = b.createPatient(b.withGiven("Polkaroo"), b.withFamily("Polkaroo"), b.withBirthdate("1980")).getIdPart();
+
+			List<String> foundIds = myTestDaoSearch.searchForIds("Patient?birthdate=lt1960&_sort=family,given");
+
+			// then
+			assertThat(foundIds, contains(idFred, idWilma, idBarney));		    // then
+		}
+
+	}
+
 	@Nested
 	public class DateSearchTests extends BaseDateSearchDaoTests {
 		@Override