honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

V2 index for string (#4629) 

v2 indexing for String for when sorting.
This commit is contained in:
michaelabuckley 2023-03-24 15:45:14 -04:00 committed by GitHub
parent bcc1ca7593
commit 7d26a7a38d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 154 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4629-uri-string-index.yaml Normal file
View File

@ -0,0 +1,4 @@
---
type: perf
issue: 4629
title: "String and URI indexing has been improved in some multi-clause queries."

5
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/upgrade.md
View File

@ -0,0 +1,5 @@
This release changes database indexing for string and uri SearchParameters.
The database migration may take several minutes.
These changes will be applied automatically on first startup.
To avoid this delay on first startup, run the migration manually.

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceIndexedSearchParamUriDao.java
View File

@ -30,8 +30,8 @@ import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri;
public interface IResourceIndexedSearchParamUriDao extends JpaRepository<ResourceIndexedSearchParamUri, Long>, IHapiFhirJpaRepository {
@Query("SELECT DISTINCT p.myUri FROM ResourceIndexedSearchParamUri p WHERE p.myResourceType = :resource_type AND p.myParamName = :param_name")
public Collection<String> findAllByResourceTypeAndParamName(@Param("resource_type") String theResourceType, @Param("param_name") String theParamName);
@Query("SELECT DISTINCT p.myUri FROM ResourceIndexedSearchParamUri p WHERE p.myHashIdentity = :hash_identity")
public Collection<String> findAllByHashIdentity(@Param("hash_identity") long theHashIdentity);
@Modifying
@Query("delete from ResourceIndexedSearchParamUri t WHERE t.myResourcePid = :resid")

49
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java
View File

@ -135,6 +135,33 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
resSearchUrlTable.addIndex("20230227.2", "IDX_RESSEARCHURL_RES").unique(false).withColumns("RES_ID");
resSearchUrlTable.addIndex("20230227.3", "IDX_RESSEARCHURL_TIME").unique(false).withColumns("CREATED_TIME");
{
// string search index
Builder.BuilderWithTableName stringTable = version.onTable("HFJ_SPIDX_STRING");
// add res_id to indentity to speed up sorts.
stringTable
.addIndex("20230303.1", "IDX_SP_STRING_HASH_IDENT_V2")
.unique(false)
.online(true)
.withColumns("HASH_IDENTITY", "RES_ID", "PARTITION_ID");
stringTable.dropIndexOnline("20230303.2", "IDX_SP_STRING_HASH_IDENT");
// add hash_norm to res_id to speed up joins on a second string.
stringTable
.addIndex("20230303.3", "IDX_SP_STRING_RESID_V2")
.unique(false)
.online(true)
.withColumns("RES_ID", "HASH_NORM_PREFIX", "PARTITION_ID");
// drop and recreate FK_SPIDXSTR_RESOURCE since it will be useing the old IDX_SP_STRING_RESID
stringTable.dropForeignKey("20230303.4", "FK_SPIDXSTR_RESOURCE", "HFJ_RESOURCE");
stringTable.dropIndexOnline("20230303.5", "IDX_SP_STRING_RESID");
stringTable.addForeignKey("20230303.6", "FK_SPIDXSTR_RESOURCE")
.toColumn("RES_ID").references("HFJ_RESOURCE", "RES_ID");
}
final String revColumnName = "REV";
final String enversRevisionTable = "HFJ_REVINFO";
final String enversMpiLinkAuditTable = "MPI_LINK_AUD";
@ -201,12 +228,32 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
.nullable()
.type(ColumnTypeEnum.DATE_ONLY);
}
version
.onTable(ResourceTable.HFJ_RESOURCE)
.addColumn("20230323.1", "SEARCH_URL_PRESENT")
.nullable()
.type(ColumnTypeEnum.BOOLEAN);
{
Builder.BuilderWithTableName uriTable = version.onTable("HFJ_SPIDX_URI");
uriTable
.addIndex("20230324.1", "IDX_SP_URI_HASH_URI_V2")
.unique(true)
.online(true)
.withColumns("HASH_URI","RES_ID","PARTITION_ID");
uriTable
.addIndex("20230324.2", "IDX_SP_URI_HASH_IDENTITY_V2")
.unique(true)
.online(true)
.withColumns("HASH_IDENTITY","SP_URI","RES_ID","PARTITION_ID");
uriTable.dropIndex("20230324.3", "IDX_SP_URI_RESTYPE_NAME");
uriTable.dropIndex("20230324.4", "IDX_SP_URI_UPDATED");
uriTable.dropIndex("20230324.5", "IDX_SP_URI");
uriTable.dropIndex("20230324.6", "IDX_SP_URI_HASH_URI");
uriTable.dropIndex("20230324.7", "IDX_SP_URI_HASH_IDENTITY");
}
}
protected void init640() {

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/UriPredicateBuilder.java
View File

@ -25,6 +25,7 @@ import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
import ca.uhn.fhir.interceptor.api.Pointcut;
import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao;
import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser;
import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri;
import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
import ca.uhn.fhir.jpa.util.QueryParameterUtils;
@ -113,7 +114,8 @@ public class UriPredicateBuilder extends BaseSearchParamPredicateBuilder {
.add(StorageProcessingMessage.class, message);
CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_WARNING, params);
Collection<String> candidates = myResourceIndexedSearchParamUriDao.findAllByResourceTypeAndParamName(getResourceType(), theParamName);
long hashIdentity = BaseResourceIndexedSearchParam.calculateHashIdentity(getPartitionSettings(), getRequestPartitionId(), getResourceType(), theParamName);
Collection<String> candidates = myResourceIndexedSearchParamUriDao.findAllByHashIdentity(hashIdentity);
List<String> toFind = new ArrayList<>();
for (String next : candidates) {
if (value.length() >= next.length()) {

4
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamString.java
View File

@ -56,12 +56,12 @@ import static org.apache.commons.lang3.StringUtils.defaultString;
*/
// This is used for sorting, and for :contains queries currently
@Index(name = "IDX_SP_STRING_HASH_IDENT", columnList = "HASH_IDENTITY"),
@Index(name = "IDX_SP_STRING_HASH_IDENT_V2", columnList = "HASH_IDENTITY,RES_ID,PARTITION_ID"),
@Index(name = "IDX_SP_STRING_HASH_NRM_V2", columnList = "HASH_NORM_PREFIX,SP_VALUE_NORMALIZED,RES_ID,PARTITION_ID"),
@Index(name = "IDX_SP_STRING_HASH_EXCT_V2", columnList = "HASH_EXACT,RES_ID,PARTITION_ID"),
@Index(name = "IDX_SP_STRING_RESID", columnList = "RES_ID")
@Index(name = "IDX_SP_STRING_RESID_V2", columnList = "RES_ID,HASH_NORM_PREFIX,PARTITION_ID")
})
public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchParam {

10
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamUri.java
View File

@ -48,11 +48,11 @@ import static org.apache.commons.lang3.StringUtils.defaultString;
@Embeddable
@Entity
@Table(name = "HFJ_SPIDX_URI", indexes = {
@Index(name = "IDX_SP_URI", columnList = "RES_TYPE,SP_NAME,SP_URI"),
@Index(name = "IDX_SP_URI_HASH_IDENTITY", columnList = "HASH_IDENTITY,SP_URI"),
@Index(name = "IDX_SP_URI_HASH_URI", columnList = "HASH_URI"),
@Index(name = "IDX_SP_URI_RESTYPE_NAME", columnList = "RES_TYPE,SP_NAME"),
@Index(name = "IDX_SP_URI_UPDATED", columnList = "SP_UPDATED"),
// for queries
@Index(name = "IDX_SP_URI_HASH_URI_V2", columnList = "HASH_URI,RES_ID,PARTITION_ID", unique = true),
// for sorting
@Index(name = "IDX_SP_URI_HASH_IDENTITY_V2", columnList = "HASH_IDENTITY,SP_URI,RES_ID,PARTITION_ID", unique = true),
// for index create/delete
@Index(name = "IDX_SP_URI_COORDS", columnList = "RES_ID")
})
public class ResourceIndexedSearchParamUri extends BaseResourceIndexedSearchParam {

85
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4StandardQueriesNoFTTest.java
View File

@ -20,6 +20,8 @@ import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.test.annotation.DirtiesContext;
@ -31,10 +33,12 @@ import java.util.List;
import java.util.function.Consumer;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.not;
import static org.junit.jupiter.api.Assertions.fail;
@ExtendWith(SpringExtension.class)
@ContextConfiguration(classes = {
@ -73,6 +77,87 @@ public class FhirResourceDaoR4StandardQueriesNoFTTest extends BaseJpaTest {
}
@Nested
public class StringSearch {
@ParameterizedTest
@CsvSource({
"normal search matches exact , Flintstones, =Flintstones, True",
"normal search matches prefix , Flintstones, =Flints , True",
"normal search matches upper prefix , Flintstones, =FLINTS , True",
"normal search matches lower prefix , Flintstones, =flints , True",
"normal search matches mixed prefix , Flintstones, =fLiNtS , True",
"normal search ignores accents , Flíntstones, =Flintstone , True",
"normal search no match suffix , Flintstones, =intstones , False",
"normal search matches first letter , Flintstones, =f , True",
"exact search matches exact , Flintstones, :exact=Flintstones , True",
"exact search no match wrong case , Flintstones, :exact=flintstones , False",
"exact search no match prefix , Flintstones, :exact=Flint , False",
// "contains search match prefix , Flintstones, :contains=flint , True",
// "contains search match prefix , Flintstones, :contains=Flint , True",
})
void stringSearches(String theDescription, String theString, String theQuery, boolean theExpectMatchFlag) {
// given
IIdType id = myDataBuilder.createPatient(myDataBuilder.withFamily(theString));
// when
List<String> foundIds = myTestDaoSearch.searchForIds("Patient?name" + theQuery);
// then
if (theExpectMatchFlag) {
assertThat(theDescription, foundIds, hasItem(id.getIdPart()));
} else {
assertThat(theDescription, foundIds, not(hasItem(id.getIdPart())));
}
}
@Test
void searchTwoFields() {
// given
IIdType id = myDataBuilder.createPatient(
myDataBuilder.withGiven("Fred"),
myDataBuilder.withFamily("Flintstone"));
List<String> foundIds = myTestDaoSearch.searchForIds("Patient?family=flint&given:exact=Fred");
// then
assertThat(foundIds, hasItem(id.getIdPart())); // then
}
@Test
void sort() {
// given
String idWilma = myDataBuilder.createPatient(myDataBuilder.withGiven("Wilma"), myDataBuilder.withFamily("Flintstone")).getIdPart();
String idFred = myDataBuilder.createPatient(myDataBuilder.withGiven("Fred"), myDataBuilder.withFamily("Flintstone")).getIdPart();
String idBarney = myDataBuilder.createPatient(myDataBuilder.withGiven("Barney"), myDataBuilder.withFamily("Rubble")).getIdPart();
String idCoolFred = myDataBuilder.createPatient(myDataBuilder.withGiven("Fred"), myDataBuilder.withFamily("Jones")).getIdPart();
String idPolka = myDataBuilder.createPatient(myDataBuilder.withGiven("Polkaroo"), myDataBuilder.withFamily("Polkaroo")).getIdPart();
List<String> foundIds = myTestDaoSearch.searchForIds("Patient?_sort=family,given");
// then
assertThat(foundIds, contains(idFred, idWilma, idCoolFred, idPolka, idBarney)); // then
}
@Test
void sortWithAge() {
// given
DaoTestDataBuilder b = myDataBuilder;
String idWilma = b.createPatient(
b.withGiven("Wilma"), b.withFamily("Flintstone"), b.withBirthdate("1945")).getIdPart();
String idFred = b.createPatient(b.withGiven("Fred"), b.withFamily("Flintstone"), b.withBirthdate("1940")).getIdPart();
String idBarney = b.createPatient(b.withGiven("Barney"), b.withFamily("Rubble"), b.withBirthdate("1941")).getIdPart();
String idCoolFred = b.createPatient(b.withGiven("Fred"), b.withFamily("Jones"), b.withBirthdate("1965")).getIdPart();
String idPolka = b.createPatient(b.withGiven("Polkaroo"), b.withFamily("Polkaroo"), b.withBirthdate("1980")).getIdPart();
List<String> foundIds = myTestDaoSearch.searchForIds("Patient?birthdate=lt1960&_sort=family,given");
// then
assertThat(foundIds, contains(idFred, idWilma, idBarney)); // then
}
}
@Nested
public class DateSearchTests extends BaseDateSearchDaoTests {
@Override