honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improved performance of date searching (#6353)

This commit is contained in:
TipzCM 2024-10-15 16:56:35 -04:00 committed by GitHub
parent 96cc20dc0d
commit eadd8c6f0d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 465 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/param/DateRangeParam.java
View File

@ -112,6 +112,8 @@ public class DateRangeParam implements IQueryParameterAnd<DateParam> {
theDateParam.setValueAsString(DateUtils.getCompletedDate(theDateParam.getValueAsString())
.getRight());
}
// there is only one value; we will set it as the lower bound
// as a >= operation
validateAndSet(theDateParam, null);
break;
case ENDS_BEFORE:
@ -121,6 +123,9 @@ public class DateRangeParam implements IQueryParameterAnd<DateParam> {
theDateParam.setValueAsString(DateUtils.getCompletedDate(theDateParam.getValueAsString())
.getLeft());
}
// there is only one value; we will set it as the upper bound
// as a <= operation
validateAndSet(null, theDateParam);
break;
default:
@ -318,8 +323,8 @@ public class DateRangeParam implements IQueryParameterAnd<DateParam> {
case NOT_EQUAL:
break;
case LESSTHAN:
case APPROXIMATE:
case LESSTHAN_OR_EQUALS:
case APPROXIMATE:
case ENDS_BEFORE:
throw new IllegalStateException(
Msg.code(1926) + "Invalid lower bound comparator: " + myLowerBound.getPrefix());
@ -383,9 +388,9 @@ public class DateRangeParam implements IQueryParameterAnd<DateParam> {
case NOT_EQUAL:
case GREATERTHAN_OR_EQUALS:
break;
case LESSTHAN_OR_EQUALS:
case LESSTHAN:
case APPROXIMATE:
case LESSTHAN_OR_EQUALS:
case ENDS_BEFORE:
throw new IllegalStateException(
Msg.code(1928) + "Invalid lower bound comparator: " + theLowerBound.getPrefix());
@ -470,10 +475,13 @@ public class DateRangeParam implements IQueryParameterAnd<DateParam> {
if (myLowerBound != null && myLowerBound.getMissing() != null) {
retVal.add((myLowerBound));
} else {
if (myLowerBound != null && !myLowerBound.isEmpty()) {
boolean hasLowerBound = myLowerBound != null && !myLowerBound.isEmpty();
boolean hasUpperBound = myUpperBound != null && !myUpperBound.isEmpty();
if (hasLowerBound) {
retVal.add((myLowerBound));
}
if (myUpperBound != null && !myUpperBound.isEmpty()) {
if (hasUpperBound) {
retVal.add((myUpperBound));
}
}

8
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6345-date-searching-performance-tweaking.yaml Normal file
View File

@ -0,0 +1,8 @@
---
type: perf
issue: 6345
title: "Date searches using equality would perform badly as the query planner
does not know that our LOW_VALUE columns are always < HIGH_VALUE
columns, and HIGH_VALUE is always > LOW_VALUE columns.
These queries have been fixed to account for this.
"

369
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/DatePredicateBuilder.java
View File

@ -97,152 +97,9 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
private Condition createPredicateDateFromRange(
DateRangeParam theRange, SearchFilterParser.CompareOperation theOperation) {
Date lowerBoundInstant = theRange.getLowerBoundAsInstant();
Date upperBoundInstant = theRange.getUpperBoundAsInstant();
DatePredicateBounds datePredicateBounds = new DatePredicateBounds(theRange);
DateParam lowerBound = theRange.getLowerBound();
DateParam upperBound = theRange.getUpperBound();
Integer lowerBoundAsOrdinal = theRange.getLowerBoundAsDateInteger();
Integer upperBoundAsOrdinal = theRange.getUpperBoundAsDateInteger();
Comparable<?> genericLowerBound;
Comparable<?> genericUpperBound;
/*
* If all present search parameters are of DAY precision, and {@link ca.uhn.fhir.jpa.model.entity.StorageSettings#getUseOrdinalDatesForDayPrecisionSearches()} is true,
* then we attempt to use the ordinal field for date comparisons instead of the date field.
*/
boolean isOrdinalComparison = isNullOrDatePrecision(lowerBound)
&& isNullOrDatePrecision(upperBound)
&& myStorageSettings.getUseOrdinalDatesForDayPrecisionSearches();
Condition lt;
Condition gt;
Condition lb = null;
Condition ub = null;
DatePredicateBuilder.ColumnEnum lowValueField;
DatePredicateBuilder.ColumnEnum highValueField;
if (isOrdinalComparison) {
lowValueField = DatePredicateBuilder.ColumnEnum.LOW_DATE_ORDINAL;
highValueField = DatePredicateBuilder.ColumnEnum.HIGH_DATE_ORDINAL;
genericLowerBound = lowerBoundAsOrdinal;
genericUpperBound = upperBoundAsOrdinal;
if (upperBound != null && upperBound.getPrecision().ordinal() <= TemporalPrecisionEnum.MONTH.ordinal()) {
genericUpperBound = Integer.parseInt(DateUtils.getCompletedDate(upperBound.getValueAsString())
.getRight()
.replace("-", ""));
}
} else {
lowValueField = DatePredicateBuilder.ColumnEnum.LOW;
highValueField = DatePredicateBuilder.ColumnEnum.HIGH;
genericLowerBound = lowerBoundInstant;
genericUpperBound = upperBoundInstant;
if (upperBound != null && upperBound.getPrecision().ordinal() <= TemporalPrecisionEnum.MONTH.ordinal()) {
String theCompleteDateStr = DateUtils.getCompletedDate(upperBound.getValueAsString())
.getRight()
.replace("-", "");
genericUpperBound = DateUtils.parseDate(theCompleteDateStr);
}
}
if (theOperation == SearchFilterParser.CompareOperation.lt
|| theOperation == SearchFilterParser.CompareOperation.le) {
// use lower bound first
if (lowerBoundInstant != null) {
lb = this.createPredicate(lowValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, genericLowerBound);
if (myStorageSettings.isAccountForDateIndexNulls()) {
lb = ComboCondition.or(
lb,
this.createPredicate(
highValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, genericLowerBound));
}
} else if (upperBoundInstant != null) {
ub = this.createPredicate(lowValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, genericUpperBound);
if (myStorageSettings.isAccountForDateIndexNulls()) {
ub = ComboCondition.or(
ub,
this.createPredicate(
highValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, genericUpperBound));
}
} else {
throw new InvalidRequestException(Msg.code(1252)
+ "lowerBound and upperBound value not correctly specified for comparing " + theOperation);
}
} else if (theOperation == SearchFilterParser.CompareOperation.gt
|| theOperation == SearchFilterParser.CompareOperation.ge) {
// use upper bound first, e.g value between 6 and 10
if (upperBoundInstant != null) {
ub = this.createPredicate(highValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericUpperBound);
if (myStorageSettings.isAccountForDateIndexNulls()) {
ub = ComboCondition.or(
ub,
this.createPredicate(
lowValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericUpperBound));
}
} else if (lowerBoundInstant != null) {
lb = this.createPredicate(highValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericLowerBound);
if (myStorageSettings.isAccountForDateIndexNulls()) {
lb = ComboCondition.or(
lb,
this.createPredicate(
lowValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericLowerBound));
}
} else {
throw new InvalidRequestException(Msg.code(1253)
+ "upperBound and lowerBound value not correctly specified for compare theOperation");
}
} else if (theOperation == SearchFilterParser.CompareOperation.ne) {
if ((lowerBoundInstant == null) || (upperBoundInstant == null)) {
throw new InvalidRequestException(Msg.code(1254)
+ "lowerBound and/or upperBound value not correctly specified for compare theOperation");
}
lt = this.createPredicate(lowValueField, ParamPrefixEnum.LESSTHAN, genericLowerBound);
gt = this.createPredicate(highValueField, ParamPrefixEnum.GREATERTHAN, genericUpperBound);
lb = ComboCondition.or(lt, gt);
} else if ((theOperation == SearchFilterParser.CompareOperation.eq)
|| (theOperation == SearchFilterParser.CompareOperation.sa)
|| (theOperation == SearchFilterParser.CompareOperation.eb)
|| (theOperation == null)) {
if (lowerBoundInstant != null) {
gt = this.createPredicate(lowValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericLowerBound);
lt = this.createPredicate(highValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericLowerBound);
if (lowerBound.getPrefix() == ParamPrefixEnum.STARTS_AFTER
|| lowerBound.getPrefix() == ParamPrefixEnum.EQUAL) {
lb = gt;
} else {
lb = ComboCondition.or(gt, lt);
}
}
if (upperBoundInstant != null) {
gt = this.createPredicate(lowValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, genericUpperBound);
lt = this.createPredicate(highValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, genericUpperBound);
if (theRange.getUpperBound().getPrefix() == ParamPrefixEnum.ENDS_BEFORE
|| theRange.getUpperBound().getPrefix() == ParamPrefixEnum.EQUAL) {
ub = lt;
} else {
ub = ComboCondition.or(gt, lt);
}
}
} else {
throw new InvalidRequestException(
Msg.code(1255) + String.format("Unsupported operator specified, operator=%s", theOperation.name()));
}
if (isOrdinalComparison) {
ourLog.trace("Ordinal date range is {} - {} ", lowerBoundAsOrdinal, upperBoundAsOrdinal);
} else {
ourLog.trace("Date range is {} - {}", lowerBoundInstant, upperBoundInstant);
}
if (lb != null && ub != null) {
return (ComboCondition.and(lb, ub));
} else if (lb != null) {
return (lb);
} else {
return (ub);
}
return datePredicateBounds.calculate(theOperation);
}
public DbColumn getColumnValueLow() {
@ -282,4 +139,226 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
HIGH,
HIGH_DATE_ORDINAL
}
public class DatePredicateBounds {
private DatePredicateBuilder.ColumnEnum myLowValueField;
private DatePredicateBuilder.ColumnEnum myHighValueField;
private Condition myLowerBoundCondition = null;
private Condition myUpperBoundCondition = null;
private final Date myLowerBoundInstant;
private final Date myUpperBoundInstant;
private final DateParam myLowerBound;
private final DateParam myUpperBound;
private final Integer myLowerBoundAsOrdinal;
private final Integer myUpperBoundAsOrdinal;
private Comparable<?> myGenericLowerBound;
private Comparable<?> myGenericUpperBound;
public DatePredicateBounds(DateRangeParam theRange) {
myLowerBoundInstant = theRange.getLowerBoundAsInstant();
myUpperBoundInstant = theRange.getUpperBoundAsInstant();
myLowerBound = theRange.getLowerBound();
myUpperBound = theRange.getUpperBound();
myLowerBoundAsOrdinal = theRange.getLowerBoundAsDateInteger();
myUpperBoundAsOrdinal = theRange.getUpperBoundAsDateInteger();
init();
}
public Condition calculate(SearchFilterParser.CompareOperation theOperation) {
if (theOperation == SearchFilterParser.CompareOperation.lt
|| theOperation == SearchFilterParser.CompareOperation.le) {
// use lower bound first
handleLessThanAndLessThanOrEqualTo();
} else if (theOperation == SearchFilterParser.CompareOperation.gt
|| theOperation == SearchFilterParser.CompareOperation.ge) {
// use upper bound first, e.g value between 6 and 10
handleGreaterThanAndGreaterThanOrEqualTo();
} else if (theOperation == SearchFilterParser.CompareOperation.ne) {
if ((myLowerBoundInstant == null) || (myUpperBoundInstant == null)) {
throw new InvalidRequestException(Msg.code(1254)
+ "lowerBound and/or upperBound value not correctly specified for compare theOperation");
}
Condition lessThan = DatePredicateBuilder.this.createPredicate(
myLowValueField, ParamPrefixEnum.LESSTHAN, myGenericLowerBound);
Condition greaterThan = DatePredicateBuilder.this.createPredicate(
myHighValueField, ParamPrefixEnum.GREATERTHAN, myGenericUpperBound);
myLowerBoundCondition = ComboCondition.or(lessThan, greaterThan);
} else if ((theOperation == SearchFilterParser.CompareOperation.eq)
|| (theOperation == SearchFilterParser.CompareOperation.sa)
|| (theOperation == SearchFilterParser.CompareOperation.eb)
|| (theOperation == null)) {
handleEqualToCompareOperator();
} else {
throw new InvalidRequestException(Msg.code(1255)
+ String.format("Unsupported operator specified, operator=%s", theOperation.name()));
}
if (isOrdinalComparison()) {
ourLog.trace("Ordinal date range is {} - {} ", myLowerBoundAsOrdinal, myUpperBoundAsOrdinal);
} else {
ourLog.trace("Date range is {} - {}", myLowerBoundInstant, myUpperBoundInstant);
}
if (myLowerBoundCondition != null && myUpperBoundCondition != null) {
return (ComboCondition.and(myLowerBoundCondition, myUpperBoundCondition));
} else if (myLowerBoundCondition != null) {
return (myLowerBoundCondition);
} else {
return (myUpperBoundCondition);
}
}
private void handleEqualToCompareOperator() {
Condition lessThan;
Condition greaterThan;
if (myLowerBoundInstant != null && myUpperBoundInstant != null) {
// both upper and lower bound
// lowerbound; :lowerbound <= low_field <= :upperbound
greaterThan = ComboCondition.and(
DatePredicateBuilder.this.createPredicate(
myLowValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, myGenericLowerBound),
DatePredicateBuilder.this.createPredicate(
myLowValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, myGenericUpperBound));
// upperbound; :lowerbound <= high_field <= :upperbound
lessThan = ComboCondition.and(
DatePredicateBuilder.this.createPredicate(
myHighValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, myGenericUpperBound),
DatePredicateBuilder.this.createPredicate(
myHighValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, myGenericLowerBound));
myLowerBoundCondition = greaterThan;
myUpperBoundCondition = lessThan;
} else if (myLowerBoundInstant != null) {
// lower bound only
greaterThan = DatePredicateBuilder.this.createPredicate(
myLowValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, myGenericLowerBound);
lessThan = DatePredicateBuilder.this.createPredicate(
myHighValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, myGenericLowerBound);
if (myLowerBound.getPrefix() == ParamPrefixEnum.STARTS_AFTER
|| myLowerBound.getPrefix() == ParamPrefixEnum.EQUAL) {
myLowerBoundCondition = greaterThan;
} else {
myLowerBoundCondition = ComboCondition.or(greaterThan, lessThan);
}
} else {
// only upper bound provided
greaterThan = DatePredicateBuilder.this.createPredicate(
myLowValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, myGenericUpperBound);
lessThan = DatePredicateBuilder.this.createPredicate(
myHighValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, myGenericUpperBound);
if (myUpperBound.getPrefix() == ParamPrefixEnum.ENDS_BEFORE
|| myUpperBound.getPrefix() == ParamPrefixEnum.EQUAL) {
myUpperBoundCondition = lessThan;
} else {
myUpperBoundCondition = ComboCondition.or(greaterThan, lessThan);
}
}
}
private void handleGreaterThanAndGreaterThanOrEqualTo() {
if (myUpperBoundInstant != null) {
// upper bound only
myUpperBoundCondition = DatePredicateBuilder.this.createPredicate(
myHighValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, myGenericUpperBound);
if (myStorageSettings.isAccountForDateIndexNulls()) {
myUpperBoundCondition = ComboCondition.or(
myUpperBoundCondition,
DatePredicateBuilder.this.createPredicate(
myLowValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, myGenericUpperBound));
}
} else if (myLowerBoundInstant != null) {
// lower bound only
myLowerBoundCondition = DatePredicateBuilder.this.createPredicate(
myHighValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, myGenericLowerBound);
if (myStorageSettings.isAccountForDateIndexNulls()) {
myLowerBoundCondition = ComboCondition.or(
myLowerBoundCondition,
DatePredicateBuilder.this.createPredicate(
myLowValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, myGenericLowerBound));
}
} else {
throw new InvalidRequestException(
Msg.code(1253)
+ "upperBound and lowerBound value not correctly specified for greater than (or equal to) compare operator");
}
}
/**
* Handle (LOW|HIGH)_FIELD <(=) value
*/
private void handleLessThanAndLessThanOrEqualTo() {
if (myLowerBoundInstant != null) {
// lower bound only provided
myLowerBoundCondition = DatePredicateBuilder.this.createPredicate(
myLowValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, myGenericLowerBound);
if (myStorageSettings.isAccountForDateIndexNulls()) {
myLowerBoundCondition = ComboCondition.or(
myLowerBoundCondition,
DatePredicateBuilder.this.createPredicate(
myHighValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, myGenericLowerBound));
}
} else if (myUpperBoundInstant != null) {
// upper bound only provided
myUpperBoundCondition = DatePredicateBuilder.this.createPredicate(
myLowValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, myGenericUpperBound);
if (myStorageSettings.isAccountForDateIndexNulls()) {
myUpperBoundCondition = ComboCondition.or(
myUpperBoundCondition,
DatePredicateBuilder.this.createPredicate(
myHighValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, myGenericUpperBound));
}
} else {
throw new InvalidRequestException(
Msg.code(1252)
+ "lowerBound and upperBound value not correctly specified for comparing using lower than (or equal to) compare operator");
}
}
private void init() {
if (isOrdinalComparison()) {
myLowValueField = DatePredicateBuilder.ColumnEnum.LOW_DATE_ORDINAL;
myHighValueField = DatePredicateBuilder.ColumnEnum.HIGH_DATE_ORDINAL;
myGenericLowerBound = myLowerBoundAsOrdinal;
myGenericUpperBound = myUpperBoundAsOrdinal;
if (myUpperBound != null
&& myUpperBound.getPrecision().ordinal() <= TemporalPrecisionEnum.MONTH.ordinal()) {
myGenericUpperBound = Integer.parseInt(DateUtils.getCompletedDate(myUpperBound.getValueAsString())
.getRight()
.replace("-", ""));
}
} else {
myLowValueField = DatePredicateBuilder.ColumnEnum.LOW;
myHighValueField = DatePredicateBuilder.ColumnEnum.HIGH;
myGenericLowerBound = myLowerBoundInstant;
myGenericUpperBound = myUpperBoundInstant;
if (myUpperBound != null
&& myUpperBound.getPrecision().ordinal() <= TemporalPrecisionEnum.MONTH.ordinal()) {
String theCompleteDateStr = DateUtils.getCompletedDate(myUpperBound.getValueAsString())
.getRight()
.replace("-", "");
myGenericUpperBound = DateUtils.parseDate(theCompleteDateStr);
}
}
}
/**
* If all present search parameters are of DAY precision, and {@link ca.uhn.fhir.jpa.model.entity.StorageSettings#getUseOrdinalDatesForDayPrecisionSearches()} is true,
* then we attempt to use the ordinal field for date comparisons instead of the date field.
*/
private boolean isOrdinalComparison() {
return isNullOrDatePrecision(myLowerBound)
&& isNullOrDatePrecision(myUpperBound)
&& myStorageSettings.getUseOrdinalDatesForDayPrecisionSearches();
}
}
}

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java
View File

@ -793,10 +793,12 @@ public class SearchQueryBuilder {
return BinaryCondition.greaterThanOrEq(theColumn, generatePlaceholder(theValue));
case NOT_EQUAL:
return BinaryCondition.notEqualTo(theColumn, generatePlaceholder(theValue));
case EQUAL:
// NB: fhir searches are always range searches;
// which is why we do not use "EQUAL"
case STARTS_AFTER:
case APPROXIMATE:
case ENDS_BEFORE:
case EQUAL:
default:
throw new IllegalArgumentException(Msg.code(1263));
}

7
hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4ElasticsearchIT.java
View File

@ -277,8 +277,11 @@ public class ValueSetExpansionR4ElasticsearchIT extends BaseJpaTest implements I
myTermCodeSystemStorageSvc.storeNewCodeSystemVersion(codeSystem, codeSystemVersion,
new SystemRequestDetails(), Collections.singletonList(valueSet), Collections.emptyList());
myTerminologyDeferredStorageSvc.saveAllDeferred();
await().atMost(10, SECONDS).until(() -> myTerminologyDeferredStorageSvc.isStorageQueueEmpty(true));
// myTerminologyDeferredStorageSvc.saveAllDeferred();
await().atMost(10, SECONDS).until(() -> {
myTerminologyDeferredStorageSvc.saveDeferred();
return myTerminologyDeferredStorageSvc.isStorageQueueEmpty(true);
});
myTermSvc.preExpandDeferredValueSetsToTerminologyTables();

5
hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java
View File

@ -158,6 +158,7 @@ public class SearchParameterMap implements Serializable {
return this;
}
@SuppressWarnings("unchecked")
public SearchParameterMap add(String theName, IQueryParameterAnd<?> theAnd) {
if (theAnd == null) {
return this;
@ -166,12 +167,14 @@ public class SearchParameterMap implements Serializable {
put(theName, new ArrayList<>());
}
List<List<IQueryParameterType>> paramList = get(theName);
for (IQueryParameterOr<?> next : theAnd.getValuesAsQueryTokens()) {
if (next == null) {
continue;
}
get(theName).add((List<IQueryParameterType>) next.getValuesAsQueryTokens());
paramList.add((List<IQueryParameterType>) next.getValuesAsQueryTokens());
}
return this;
}

2
hapi-fhir-jpaserver-searchparam/src/test/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMapTest.java
View File

@ -198,10 +198,8 @@ class SearchParameterMapTest {
assertEquals(orig.get("int"), clone.get("int"));
}
@Test
public void testCompareParameters() {
// Missing
assertEquals(0, compare(ourFhirContext, new StringParam().setMissing(true), new StringParam().setMissing(true)));
assertEquals(-1, compare(ourFhirContext, new StringParam("A"), new StringParam().setMissing(true)));

2
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java
View File

@ -291,7 +291,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue());
String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false);
String expected = "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.HASH_COMPLETE = '-2634469377090377342') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND ((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202'))))";
String expected = "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.HASH_COMPLETE = '-2634469377090377342') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND (((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_LOW_DATE_ORDINAL <= '20210202')) AND ((t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL >= '20210202')))))";
assertEquals(expected, sql);
logCapturedMessages();

3
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java
View File

@ -2446,6 +2446,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
SearchParameterMap params;
List<Encounter> encs;
// only upper bound -> should find encounters with period values that are
params = new SearchParameterMap();
params.add(Encounter.SP_DATE, new DateRangeParam(null, "2001-01-03"));
params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02"));
@ -2453,6 +2454,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
assertThat(encs).hasSize(1);
params = new SearchParameterMap();
params.setLoadSynchronous(true);
params.add(Encounter.SP_DATE, new DateRangeParam("2001-01-01", "2001-01-03"));
params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02"));
encs = toList(myEncounterDao.search(params));
@ -2475,7 +2477,6 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
params.add(Encounter.SP_IDENTIFIER, new TokenParam("testDatePeriodParam", "02"));
encs = toList(myEncounterDao.search(params));
assertThat(encs).isEmpty();
}
@Test

43
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirSearchDaoR4Test.java
View File

@ -1,5 +1,7 @@
package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.interceptor.api.IInterceptorService;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
import ca.uhn.fhir.jpa.model.dao.JpaPid;
import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
@ -15,24 +17,53 @@ import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenAndListParam;
import ca.uhn.fhir.rest.param.TokenOrListParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.storage.test.BaseDateSearchDaoTests;
import org.hl7.fhir.r4.model.Organization;
import org.hl7.fhir.r4.model.Patient;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.InvalidDataAccessApiUsageException;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import javax.sql.DataSource;
import java.util.ArrayList;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class FhirSearchDaoR4Test extends BaseJpaR4Test {
public class FhirSearchDaoR4Test extends BaseJpaR4Test implements IR4SearchIndexTests {
private static final Logger ourLog = LoggerFactory.getLogger(FhirSearchDaoR4Test.class);
@Autowired
private IFulltextSearchSvc mySearchDao;
@Autowired
private DataSource myDataSource;
@Override
public IInterceptorService getInterceptorService() {
return myInterceptorRegistry;
}
@Override
public Logger getLogger() {
return ourLog;
}
@Override
public DaoRegistry getDaoRegistry() {
return myDaoRegistry;
}
@Override
public DataSource getDataSource() {
return myDataSource;
}
@Test
public void testDaoCallRequiresTransaction() {
@ -267,8 +298,7 @@ public class FhirSearchDaoR4Test extends BaseJpaR4Test {
final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
for (int i = 0; i < numberOfPatientsToCreate; i++)
{
for (int i = 0; i < numberOfPatientsToCreate; i++) {
Patient patient = new Patient();
patient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC </div>");
expectedActivePatientIds.add(myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless().getIdPart());
@ -300,8 +330,7 @@ public class FhirSearchDaoR4Test extends BaseJpaR4Test {
List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
// create active and non-active patients with the same narrative
for (int i = 0; i < numberOfPatientsToCreate; i++)
{
for (int i = 0; i < numberOfPatientsToCreate; i++) {
Patient activePatient = new Patient();
activePatient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC </div>");
activePatient.setActive(true);
@ -335,8 +364,7 @@ public class FhirSearchDaoR4Test extends BaseJpaR4Test {
List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
// create active and non-active patients with the same narrative
for (int i = 0; i < numberOfPatientsToCreate; i++)
{
for (int i = 0; i < numberOfPatientsToCreate; i++) {
Patient activePatient = new Patient();
activePatient.addName().setFamily(patientFamilyName);
activePatient.setActive(true);
@ -362,5 +390,4 @@ public class FhirSearchDaoR4Test extends BaseJpaR4Test {
assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
}
}

154
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/IR4SearchIndexTests.java Normal file
View File

@ -0,0 +1,154 @@
package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.interceptor.api.Hook;
import ca.uhn.fhir.interceptor.api.IInterceptorService;
import ca.uhn.fhir.interceptor.api.Pointcut;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.util.SqlQuery;
import ca.uhn.fhir.jpa.util.SqlQueryList;
import ca.uhn.fhir.rest.api.server.IBundleProvider;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
import ca.uhn.fhir.rest.param.DateParam;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.r4.model.DateTimeType;
import org.hl7.fhir.r4.model.Observation;
import org.hl7.fhir.r4.model.Patient;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
public interface IR4SearchIndexTests {
IInterceptorService getInterceptorService();
DaoRegistry getDaoRegistry();
DataSource getDataSource();
Logger getLogger();
@SuppressWarnings("unchecked")
private <T extends IBaseResource> IFhirResourceDao<T> getResourceDao(String theResourceType) {
return getDaoRegistry()
.getResourceDao(theResourceType);
}
@Test
default void search_dateValue_withEquals() {
// setup
RequestDetails rd = new SystemRequestDetails();
DateTimeType birthdayDateTime = new DateTimeType();
birthdayDateTime.setValueAsString("1999-12-31");
IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
IFhirResourceDao<Observation> observationDao = getResourceDao("Observation");
// create some patients (a few so we have a few to scan through)
int birthYear = birthdayDateTime.getYear();
for (int i = 0; i < 10; i++) {
Patient patient = new Patient();
patient.setActive(true);
patient.addName()
.setFamily("simpson")
.addGiven("homer" + i);
// i = 0 will give us the resource we're looking for
// all other dates will be a new resource
Date d = birthdayDateTime.getValue();
int adjustment = -i;
d.setYear((birthYear + adjustment) - 1900);
patient.setBirthDate(d);
patientDao.create(patient, rd);
}
// add a bunch of things with recent dates
Date now = new Date();
for (int i = 0; i < 200; i++) {
Observation obs = new Observation();
now.setDate(i % 28); // 28 because that's the shortest month
obs.setIssued(now);
obs.setStatus(Observation.ObservationStatus.CORRECTED);
observationDao.create(obs, rd);
}
SearchParameterMap searchParameterMap = new SearchParameterMap();
searchParameterMap.setLoadSynchronous(true);
DateParam birthdayParam = new DateParam();
birthdayParam.setValueAsString("1999-12-31");
searchParameterMap.add("birthdate", birthdayParam);
/*
* the searches are very fast, regardless.
* so we'll be checking the actual query plan instead
*/
Object interceptor = new Object() {
@Hook(Pointcut.JPA_PERFTRACE_RAW_SQL)
public void captureSql(ServletRequestDetails theRequestDetails, SqlQueryList theQueries) {
for (SqlQuery q : theQueries) {
String sql = q.getSql(true, false);
StringBuilder sb = new StringBuilder();
try (Connection connection = getDataSource().getConnection()) {
try (Statement stmt = connection.createStatement()) {
ResultSet results = stmt.executeQuery("explain analyze " + sql);
while (results.next()) {
sb.append(results.getString(1));
}
}
} catch (SQLException theE) {
throw new RuntimeException(theE);
}
log(theRequestDetails, sb.toString());
}
}
@Hook(Pointcut.JPA_PERFTRACE_SEARCH_FIRST_RESULT_LOADED)
public void firstResultLoaded(ServletRequestDetails theRequestDetails, SearchRuntimeDetails theSearchRuntimeDetails) {
String msg = "SQL statement returned first result in "
+ theSearchRuntimeDetails.getQueryStopwatch().toString();
log(theRequestDetails, msg);
}
@Hook(Pointcut.JPA_PERFTRACE_SEARCH_SELECT_COMPLETE)
public void selectComplete(ServletRequestDetails theRequestDetails, SearchRuntimeDetails theSearchRuntimeDetails) {
String msg = "SQL statement execution complete in "
+ theSearchRuntimeDetails.getQueryStopwatch().toString() + " - Returned "
+ theSearchRuntimeDetails.getFoundMatchesCount() + " results";
log(theRequestDetails, msg);
}
private void log(ServletRequestDetails theRequestDetails, String theMsg) {
getLogger().info(theMsg);
}
};
try {
getInterceptorService().registerInterceptor(interceptor);
// test
IBundleProvider results = patientDao.search(searchParameterMap, rd);
// verify
assertNotNull(results);
assertEquals(1, results.size());
} finally {
// remove the interceptor
getInterceptorService().unregisterInterceptor(interceptor);
}
}
}

21
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java
View File

@ -1815,7 +1815,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(0, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date OR param
@ -1831,7 +1831,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(0, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date AND param
@ -1847,7 +1847,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(0, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// DateRangeParam
@ -1900,7 +1900,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql);
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "SP_VALUE_LOW")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "SP_VALUE_LOW")).as(searchSql).isEqualTo(2);
// Date OR param
@ -1916,7 +1916,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date AND param
@ -1932,7 +1932,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// DateRangeParam
@ -1980,7 +1980,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date OR param
@ -1996,7 +1996,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date AND param
@ -2012,7 +2012,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// DateRangeParam
@ -2047,7 +2047,6 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
createPatient(withPartition(2), withBirthdate("2021-04-20"));
// Date param
addReadDefaultPartition();
myCaptureQueriesListener.clear();
SearchParameterMap map = new SearchParameterMap();
@ -2060,7 +2059,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID = '-1'"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
}

11
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java
View File

@ -3924,8 +3924,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
myClient.update().resource(enc).execute().getId().toUnqualifiedVersionless();
HttpGet get = new HttpGet(myServerBase + "/Encounter?patient=P2&date=ge2017-01-01&_include:recurse=Encounter:practitioner&_lastUpdated=ge2017-11-10");
CloseableHttpResponse response = ourHttpClient.execute(get);
try {
try (CloseableHttpResponse response = ourHttpClient.execute(get)) {
assertEquals(200, response.getStatusLine().getStatusCode());
String output = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
response.getEntity().getContent().close();
@ -3933,13 +3932,10 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
List<String> ids = toUnqualifiedVersionlessIdValues(myFhirContext.newXmlParser().parseResource(Bundle.class, output));
ourLog.info(ids.toString());
assertThat(ids).containsExactlyInAnyOrder("Practitioner/PRAC", "Encounter/E2");
} finally {
response.close();
}
get = new HttpGet(myServerBase + "/Encounter?patient=P2&date=ge2017-01-01&_include:recurse=Encounter:practitioner&_lastUpdated=ge2099-11-10");
response = ourHttpClient.execute(get);
try {
try (CloseableHttpResponse response = ourHttpClient.execute(get)) {
assertEquals(200, response.getStatusLine().getStatusCode());
String output = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
response.getEntity().getContent().close();
@ -3947,10 +3943,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
List<String> ids = toUnqualifiedVersionlessIdValues(myFhirContext.newXmlParser().parseResource(Bundle.class, output));
ourLog.info(ids.toString());
assertThat(ids).isEmpty();
} finally {
response.close();
}
}
@Test

9
hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilderDialectPostgresTest.java
View File

@ -44,15 +44,16 @@ public class SearchQueryBuilderDialectPostgresTest extends BaseSearchQueryBuilde
GeneratedSql generatedSql = searchQueryBuilder.generate(0, 500);
logSql(generatedSql);
String expected = "SELECT t0.RES_ID FROM HFJ_SPIDX_DATE t0 WHERE ((t0.HASH_IDENTITY = ?) AND (((t0.SP_VALUE_LOW_DATE_ORDINAL >= ?) AND (t0.SP_VALUE_LOW_DATE_ORDINAL <= ?)) AND ((t0.SP_VALUE_HIGH_DATE_ORDINAL <= ?) AND (t0.SP_VALUE_HIGH_DATE_ORDINAL >= ?)))) fetch first ? rows only";
String sql = generatedSql.getSql();
assertEquals("SELECT t0.RES_ID FROM HFJ_SPIDX_DATE t0 WHERE ((t0.HASH_IDENTITY = ?) AND ((t0.SP_VALUE_LOW_DATE_ORDINAL >= ?) AND (t0.SP_VALUE_HIGH_DATE_ORDINAL <= ?))) fetch first ? rows only", sql);
assertEquals(expected, sql);
assertEquals(4, StringUtils.countMatches(sql, "?"));
assertThat(generatedSql.getBindVariables()).hasSize(4);
assertEquals(6, StringUtils.countMatches(sql, "?"));
assertThat(generatedSql.getBindVariables()).hasSize(6);
assertEquals(123682819940570799L, generatedSql.getBindVariables().get(0));
assertEquals(20220101, generatedSql.getBindVariables().get(1));
assertEquals(20221231, generatedSql.getBindVariables().get(2));
assertEquals(500, generatedSql.getBindVariables().get(3));
assertEquals(500, generatedSql.getBindVariables().get(5));
}
@Nonnull