honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 4180 process slows down to a crawl while loading uscore ig (#4201) 

* Make query synchronous

* Use outer joins only for sorting.

* Adjust test

* Add changelog

* Fix changelog typo

Co-authored-by: juan.marchionatto <juan.marchionatto@smilecdr.com>
This commit is contained in:
jmarchionatto 2022-11-07 14:20:28 -05:00 committed by GitHub
parent 8ff870495e
commit 6c9fe710ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 31 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_3_0/4180-process-very-slow-loading-us-core-ig.yaml Normal file
View File

@ -0,0 +1,6 @@
---
type: fix
issue: 4180
title: "Before DB table joins were `LEFT OUTER` by default, which was causing some queries to skip table indexes, making them
very slow. This was more noticeable in H2 database, but affected all DB types in different measures. This has been fixed by
using `INNER` joins, unless specific use case requires otherwise."

25
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java
View File

@ -200,7 +200,7 @@ public class SearchQueryBuilder {
Validate.isTrue(theSourceJoinColumn != null); Validate.isTrue(theSourceJoinColumn != null);
ForcedIdPredicateBuilder retVal = mySqlBuilderFactory.newForcedIdPredicateBuilder(this); ForcedIdPredicateBuilder retVal = mySqlBuilderFactory.newForcedIdPredicateBuilder(this);
addTable(retVal, theSourceJoinColumn); addTableForSorting(retVal, theSourceJoinColumn);
return retVal; return retVal;
} }
@ -379,11 +379,19 @@ public class SearchQueryBuilder {
* Add and return a predicate builder (or a root query if no root query exists yet) for an arbitrary table * Add and return a predicate builder (or a root query if no root query exists yet) for an arbitrary table
*/ */
private void addTable(BaseJoiningPredicateBuilder thePredicateBuilder, @Nullable DbColumn theSourceJoinColumn) { private void addTable(BaseJoiningPredicateBuilder thePredicateBuilder, @Nullable DbColumn theSourceJoinColumn) {
addTable(thePredicateBuilder, theSourceJoinColumn, SelectQuery.JoinType.INNER);
}
private void addTableForSorting(BaseJoiningPredicateBuilder thePredicateBuilder, @Nullable DbColumn theSourceJoinColumn) {
addTable(thePredicateBuilder, theSourceJoinColumn, SelectQuery.JoinType.LEFT_OUTER);
}
private void addTable(BaseJoiningPredicateBuilder thePredicateBuilder, @Nullable DbColumn theSourceJoinColumn, SelectQuery.JoinType theJoinType) {
if (theSourceJoinColumn != null) { if (theSourceJoinColumn != null) {
DbTable fromTable = theSourceJoinColumn.getTable(); DbTable fromTable = theSourceJoinColumn.getTable();
DbTable toTable = thePredicateBuilder.getTable(); DbTable toTable = thePredicateBuilder.getTable();
DbColumn toColumn = thePredicateBuilder.getResourceIdColumn(); DbColumn toColumn = thePredicateBuilder.getResourceIdColumn();
addJoin(fromTable, toTable, theSourceJoinColumn, toColumn); addJoin(fromTable, toTable, theSourceJoinColumn, toColumn, theJoinType);
} else { } else {
if (myFirstPredicateBuilder == null) { if (myFirstPredicateBuilder == null) {
@ -415,20 +423,25 @@ public class SearchQueryBuilder {
DbTable toTable = thePredicateBuilder.getTable(); DbTable toTable = thePredicateBuilder.getTable();
DbColumn fromColumn = myFirstPredicateBuilder.getResourceIdColumn(); DbColumn fromColumn = myFirstPredicateBuilder.getResourceIdColumn();
DbColumn toColumn = thePredicateBuilder.getResourceIdColumn(); DbColumn toColumn = thePredicateBuilder.getResourceIdColumn();
addJoin(fromTable, toTable, fromColumn, toColumn); addJoin(fromTable, toTable, fromColumn, toColumn, theJoinType);
} }
} }
public void addJoin(DbTable theFromTable, DbTable theToTable, DbColumn theFromColumn, DbColumn theToColumn, SelectQuery.JoinType theJoinType) {
Join join = new DbJoin(mySpec, theFromTable, theToTable, new DbColumn[]{theFromColumn}, new DbColumn[]{theToColumn});
mySelect.addJoins(theJoinType, join);
}
public void addJoin(DbTable theFromTable, DbTable theToTable, DbColumn theFromColumn, DbColumn theToColumn) { public void addJoin(DbTable theFromTable, DbTable theToTable, DbColumn theFromColumn, DbColumn theToColumn) {
Join join = new DbJoin(mySpec, theFromTable, theToTable, new DbColumn[]{theFromColumn}, new DbColumn[]{theToColumn}); Join join = new DbJoin(mySpec, theFromTable, theToTable, new DbColumn[]{theFromColumn}, new DbColumn[]{theToColumn});
mySelect.addJoins(SelectQuery.JoinType.LEFT_OUTER, join); mySelect.addJoins(SelectQuery.JoinType.INNER, join);
} }
public void addJoinWithCustomOnCondition(DbTable theFromTable, DbTable theToTable, DbColumn theFromColumn, DbColumn theToColumn, Condition theCondition) { public void addJoinWithCustomOnCondition(DbTable theFromTable, DbTable theToTable, DbColumn theFromColumn, DbColumn theToColumn, Condition theCondition) {
Join join = new DbJoin(mySpec, theFromTable, theToTable, new DbColumn[]{theFromColumn}, new DbColumn[]{theToColumn}); Join join = new DbJoin(mySpec, theFromTable, theToTable, new DbColumn[]{theFromColumn}, new DbColumn[]{theToColumn});
// add hashIdentity codition here // add hashIdentity codition here
mySelect.addJoins(SelectQuery.JoinType.LEFT_OUTER, join); mySelect.addJoins(SelectQuery.JoinType.INNER, join);
} }
/** /**

2
hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/SearchParameterHelper.java
View File

@ -47,7 +47,7 @@ public class SearchParameterHelper {
return Optional.empty(); return Optional.empty();
} }
SearchParameterMap retVal = new SearchParameterMap(); SearchParameterMap retVal = SearchParameterMap.newSynchronous();
String theCode = canonicalSearchParam.getName(); String theCode = canonicalSearchParam.getName();
List<String> theBases = List.copyOf(canonicalSearchParam.getBase()); List<String> theBases = List.copyOf(canonicalSearchParam.getBase());

2
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java
View File

@ -198,7 +198,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
assertThat(actual, containsInAnyOrder(id1.toUnqualifiedVersionless().getValue())); assertThat(actual, containsInAnyOrder(id1.toUnqualifiedVersionless().getValue()));
String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false); String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false);
assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 LEFT OUTER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.IDX_STRING = 'Patient?family=FAMILY1%5C%7C&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale&given=GIVEN1') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND ((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202'))))", sql); assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.IDX_STRING = 'Patient?family=FAMILY1%5C%7C&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale&given=GIVEN1') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND ((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202'))))", sql);
logCapturedMessages(); logCapturedMessages();
assertThat(myMessages.toString(), containsString("[INFO Using NON_UNIQUE index for query for search: Patient?family=FAMILY1%5C%7C&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale&given=GIVEN1]")); assertThat(myMessages.toString(), containsString("[INFO Using NON_UNIQUE index for query for search: Patient?family=FAMILY1%5C%7C&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale&given=GIVEN1]"));

4
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java
View File

@ -3724,7 +3724,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
String searchQuery = queries.get(0); String searchQuery = queries.get(0);
assertEquals(3, countMatches(searchQuery.toUpperCase(), "HFJ_SPIDX_TOKEN"), searchQuery); assertEquals(3, countMatches(searchQuery.toUpperCase(), "HFJ_SPIDX_TOKEN"), searchQuery);
assertEquals(5, countMatches(searchQuery.toUpperCase(), "LEFT OUTER JOIN"), searchQuery); assertEquals(5, countMatches(searchQuery.toUpperCase(), "INNER JOIN"), searchQuery);
} }
@Test @Test
@ -3748,7 +3748,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
String searchQuery = queries.get(0); String searchQuery = queries.get(0);
assertEquals(1, countMatches(searchQuery.toUpperCase(), "HFJ_SPIDX_TOKEN"), searchQuery); assertEquals(1, countMatches(searchQuery.toUpperCase(), "HFJ_SPIDX_TOKEN"), searchQuery);
assertEquals(1, countMatches(searchQuery.toUpperCase(), "LEFT OUTER JOIN"), searchQuery); assertEquals(1, countMatches(searchQuery.toUpperCase(), "INNER JOIN"), searchQuery);
assertEquals(2, countMatches(searchQuery.toUpperCase(), "RES_UPDATED"), searchQuery); assertEquals(2, countMatches(searchQuery.toUpperCase(), "RES_UPDATED"), searchQuery);
} }

4
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchSqlTest.java
View File

@ -66,7 +66,7 @@ public class FhirResourceDaoR4SearchSqlTest extends BaseJpaR4Test {
myPatientDao.search(map); myPatientDao.search(map);
assertEquals(1, myCaptureQueriesListener.countSelectQueries()); assertEquals(1, myCaptureQueriesListener.countSelectQueries());
String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false); String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false);
assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 LEFT OUTER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_SYS_AND_VALUE = ?))", sql); assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_SYS_AND_VALUE = ?))", sql);
} }
@ -89,7 +89,7 @@ public class FhirResourceDaoR4SearchSqlTest extends BaseJpaR4Test {
assertEquals(3, myCaptureQueriesListener.countSelectQueries()); assertEquals(3, myCaptureQueriesListener.countSelectQueries());
// Query 1 - Find resources: Make sure we search for tag type+system+code always // Query 1 - Find resources: Make sure we search for tag type+system+code always
String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false); String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false);
assertEquals("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_TAG t1 ON (t0.RES_ID = t1.RES_ID) LEFT OUTER JOIN HFJ_TAG_DEF t2 ON (t1.TAG_ID = t2.TAG_ID) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t2.TAG_TYPE = ?) AND (t2.TAG_SYSTEM = ?) AND (t2.TAG_CODE = ?)))", sql); assertEquals("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 INNER JOIN HFJ_RES_TAG t1 ON (t0.RES_ID = t1.RES_ID) INNER JOIN HFJ_TAG_DEF t2 ON (t1.TAG_ID = t2.TAG_ID) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t2.TAG_TYPE = ?) AND (t2.TAG_SYSTEM = ?) AND (t2.TAG_CODE = ?)))", sql);
// Query 2 - Load resourece contents // Query 2 - Load resourece contents
sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(1).getSql(false, false); sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(1).getSql(false, false);
assertThat(sql, containsString("where resourcese0_.RES_ID in (?)")); assertThat(sql, containsString("where resourcese0_.RES_ID in (?)"));