diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_3_0/4180-process-very-slow-loading-us-core-ig.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_3_0/4180-process-very-slow-loading-us-core-ig.yaml new file mode 100644 index 00000000000..8a27544b4c7 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_3_0/4180-process-very-slow-loading-us-core-ig.yaml @@ -0,0 +1,6 @@ +--- +type: fix +issue: 4180 +title: "Before DB table joins were `LEFT OUTER` by default, which was causing some queries to skip table indexes, making them + very slow. This was more noticeable in H2 database, but affected all DB types in different measures. This has been fixed by + using `INNER` joins, unless specific use case requires otherwise." diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java index 9fe34698b6e..1eb281a0943 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java @@ -200,7 +200,7 @@ public class SearchQueryBuilder { Validate.isTrue(theSourceJoinColumn != null); ForcedIdPredicateBuilder retVal = mySqlBuilderFactory.newForcedIdPredicateBuilder(this); - addTable(retVal, theSourceJoinColumn); + addTableForSorting(retVal, theSourceJoinColumn); return retVal; } @@ -379,11 +379,19 @@ public class SearchQueryBuilder { * Add and return a predicate builder (or a root query if no root query exists yet) for an arbitrary table */ private void addTable(BaseJoiningPredicateBuilder thePredicateBuilder, @Nullable DbColumn theSourceJoinColumn) { + addTable(thePredicateBuilder, theSourceJoinColumn, SelectQuery.JoinType.INNER); + } + + private void addTableForSorting(BaseJoiningPredicateBuilder thePredicateBuilder, @Nullable DbColumn theSourceJoinColumn) { + addTable(thePredicateBuilder, theSourceJoinColumn, SelectQuery.JoinType.LEFT_OUTER); + } + + private void addTable(BaseJoiningPredicateBuilder thePredicateBuilder, @Nullable DbColumn theSourceJoinColumn, SelectQuery.JoinType theJoinType) { if (theSourceJoinColumn != null) { DbTable fromTable = theSourceJoinColumn.getTable(); DbTable toTable = thePredicateBuilder.getTable(); DbColumn toColumn = thePredicateBuilder.getResourceIdColumn(); - addJoin(fromTable, toTable, theSourceJoinColumn, toColumn); + addJoin(fromTable, toTable, theSourceJoinColumn, toColumn, theJoinType); } else { if (myFirstPredicateBuilder == null) { @@ -415,20 +423,25 @@ public class SearchQueryBuilder { DbTable toTable = thePredicateBuilder.getTable(); DbColumn fromColumn = myFirstPredicateBuilder.getResourceIdColumn(); DbColumn toColumn = thePredicateBuilder.getResourceIdColumn(); - addJoin(fromTable, toTable, fromColumn, toColumn); - + addJoin(fromTable, toTable, fromColumn, toColumn, theJoinType); } } + + public void addJoin(DbTable theFromTable, DbTable theToTable, DbColumn theFromColumn, DbColumn theToColumn, SelectQuery.JoinType theJoinType) { + Join join = new DbJoin(mySpec, theFromTable, theToTable, new DbColumn[]{theFromColumn}, new DbColumn[]{theToColumn}); + mySelect.addJoins(theJoinType, join); + } + public void addJoin(DbTable theFromTable, DbTable theToTable, DbColumn theFromColumn, DbColumn theToColumn) { Join join = new DbJoin(mySpec, theFromTable, theToTable, new DbColumn[]{theFromColumn}, new DbColumn[]{theToColumn}); - mySelect.addJoins(SelectQuery.JoinType.LEFT_OUTER, join); + mySelect.addJoins(SelectQuery.JoinType.INNER, join); } public void addJoinWithCustomOnCondition(DbTable theFromTable, DbTable theToTable, DbColumn theFromColumn, DbColumn theToColumn, Condition theCondition) { Join join = new DbJoin(mySpec, theFromTable, theToTable, new DbColumn[]{theFromColumn}, new DbColumn[]{theToColumn}); // add hashIdentity codition here - mySelect.addJoins(SelectQuery.JoinType.LEFT_OUTER, join); + mySelect.addJoins(SelectQuery.JoinType.INNER, join); } /** diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/SearchParameterHelper.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/SearchParameterHelper.java index dac2781701e..43ca917144e 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/SearchParameterHelper.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/SearchParameterHelper.java @@ -47,7 +47,7 @@ public class SearchParameterHelper { return Optional.empty(); } - SearchParameterMap retVal = new SearchParameterMap(); + SearchParameterMap retVal = SearchParameterMap.newSynchronous(); String theCode = canonicalSearchParam.getName(); List theBases = List.copyOf(canonicalSearchParam.getBase()); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java index 3c7df14f3eb..1d9bc6dde35 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java @@ -198,7 +198,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T assertThat(actual, containsInAnyOrder(id1.toUnqualifiedVersionless().getValue())); String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false); - assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 LEFT OUTER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.IDX_STRING = 'Patient?family=FAMILY1%5C%7C&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale&given=GIVEN1') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND ((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202'))))", sql); + assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.IDX_STRING = 'Patient?family=FAMILY1%5C%7C&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale&given=GIVEN1') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND ((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202'))))", sql); logCapturedMessages(); assertThat(myMessages.toString(), containsString("[INFO Using NON_UNIQUE index for query for search: Patient?family=FAMILY1%5C%7C&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale&given=GIVEN1]")); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java index dcccaf71c62..8cb34b5b417 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java @@ -3724,7 +3724,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { String searchQuery = queries.get(0); assertEquals(3, countMatches(searchQuery.toUpperCase(), "HFJ_SPIDX_TOKEN"), searchQuery); - assertEquals(5, countMatches(searchQuery.toUpperCase(), "LEFT OUTER JOIN"), searchQuery); + assertEquals(5, countMatches(searchQuery.toUpperCase(), "INNER JOIN"), searchQuery); } @Test @@ -3748,7 +3748,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { String searchQuery = queries.get(0); assertEquals(1, countMatches(searchQuery.toUpperCase(), "HFJ_SPIDX_TOKEN"), searchQuery); - assertEquals(1, countMatches(searchQuery.toUpperCase(), "LEFT OUTER JOIN"), searchQuery); + assertEquals(1, countMatches(searchQuery.toUpperCase(), "INNER JOIN"), searchQuery); assertEquals(2, countMatches(searchQuery.toUpperCase(), "RES_UPDATED"), searchQuery); } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchSqlTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchSqlTest.java index 3994e94f0d8..619642b02e4 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchSqlTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchSqlTest.java @@ -66,7 +66,7 @@ public class FhirResourceDaoR4SearchSqlTest extends BaseJpaR4Test { myPatientDao.search(map); assertEquals(1, myCaptureQueriesListener.countSelectQueries()); String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false); - assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 LEFT OUTER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_SYS_AND_VALUE = ?))", sql); + assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_SYS_AND_VALUE = ?))", sql); } @@ -89,7 +89,7 @@ public class FhirResourceDaoR4SearchSqlTest extends BaseJpaR4Test { assertEquals(3, myCaptureQueriesListener.countSelectQueries()); // Query 1 - Find resources: Make sure we search for tag type+system+code always String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false); - assertEquals("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_TAG t1 ON (t0.RES_ID = t1.RES_ID) LEFT OUTER JOIN HFJ_TAG_DEF t2 ON (t1.TAG_ID = t2.TAG_ID) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t2.TAG_TYPE = ?) AND (t2.TAG_SYSTEM = ?) AND (t2.TAG_CODE = ?)))", sql); + assertEquals("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 INNER JOIN HFJ_RES_TAG t1 ON (t0.RES_ID = t1.RES_ID) INNER JOIN HFJ_TAG_DEF t2 ON (t1.TAG_ID = t2.TAG_ID) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t2.TAG_TYPE = ?) AND (t2.TAG_SYSTEM = ?) AND (t2.TAG_CODE = ?)))", sql); // Query 2 - Load resourece contents sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(1).getSql(false, false); assertThat(sql, containsString("where resourcese0_.RES_ID in (?)"));