From bd83bc17cd72c2ac5062bcf55ad99e4566d66c30 Mon Sep 17 00:00:00 2001 From: Tadgh Date: Wed, 26 Jun 2024 16:20:15 -0700 Subject: [PATCH] 6046 text does not work with r5 (#6051) * Initial commit of failing test * wip * Hack a side-route to not use analyzed versions of the field * spotless * Add concept of contains for special * Changelog, docs * spotless * another test * wip * refactor * spotless --- .../ca/uhn/fhir/rest/param/SpecialParam.java | 70 ++++++- .../uhn/fhir/rest/param/SpecialParamTest.java | 67 ++++++ .../uhn/fhir/rest/param/StringParamTest.java | 5 +- ...-failed-text-and-content-search-in-r5.yaml | 4 + .../changelog/7_4_0/6046-text-contains.yaml | 5 + .../uhn/hapi/fhir/docs/server_jpa/elastic.md | 75 +++++-- .../search/ExtendedHSearchClauseBuilder.java | 123 ++++++++--- ...esourceDaoR4SearchWithElasticSearchIT.java | 197 +++++++++++++++++- .../src/test/resources/logback-test.xml | 12 ++ 9 files changed, 509 insertions(+), 49 deletions(-) create mode 100644 hapi-fhir-base/src/test/java/ca/uhn/fhir/rest/param/SpecialParamTest.java create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-failed-text-and-content-search-in-r5.yaml create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-text-contains.yaml diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/param/SpecialParam.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/param/SpecialParam.java index 70a47bca0f7..1c7087ccd74 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/param/SpecialParam.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/param/SpecialParam.java @@ -21,15 +21,22 @@ package ca.uhn.fhir.rest.param; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.model.primitive.UriDt; +import ca.uhn.fhir.rest.api.Constants; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.apache.commons.lang3.StringUtils.defaultString; public class SpecialParam extends BaseParam /*implements IQueryParameterType*/ { + private static final Logger ourLog = LoggerFactory.getLogger(StringParam.class); private String myValue; + private boolean myContains; /** * Constructor @@ -40,7 +47,11 @@ public class SpecialParam extends BaseParam /*implements IQueryParameterType*/ { @Override String doGetQueryParameterQualifier() { - return null; + if (myContains) { + return Constants.PARAMQUALIFIER_STRING_CONTAINS; + } else { + return null; + } } /** @@ -56,6 +67,15 @@ public class SpecialParam extends BaseParam /*implements IQueryParameterType*/ { */ @Override void doSetValueAsQueryToken(FhirContext theContext, String theParamName, String theQualifier, String theParameter) { + if (Constants.PARAMQUALIFIER_STRING_CONTAINS.equals(theQualifier)) { + if (theParamName.equalsIgnoreCase(Constants.PARAM_TEXT) + || theParamName.equalsIgnoreCase(Constants.PARAM_CONTENT)) { + setContains(true); + } else { + ourLog.debug( + "Attempted to set the :contains modifier on a special search parameter that was not `_text` or `_content`. This is not supported."); + } + } setValue(ParameterUtil.unescape(theParameter)); } @@ -93,4 +113,52 @@ public class SpecialParam extends BaseParam /*implements IQueryParameterType*/ { private static String toSystemValue(UriDt theSystem) { return theSystem.getValueAsString(); } + /** + * Special parameter modifier :contains for _text and _content + */ + public boolean isContains() { + return myContains; + } + + /** + * Special parameter modifier :contains for _text and _content + */ + public SpecialParam setContains(boolean theContains) { + myContains = theContains; + if (myContains) { + setMissing(null); + } + return this; + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(isContains()) + .append(getValue()) + .append(getMissing()) + .toHashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof SpecialParam)) { + return false; + } + + SpecialParam other = (SpecialParam) obj; + + EqualsBuilder eb = new EqualsBuilder(); + eb.append(myContains, other.myContains); + eb.append(myValue, other.myValue); + eb.append(getMissing(), other.getMissing()); + + return eb.isEquals(); + } } diff --git a/hapi-fhir-base/src/test/java/ca/uhn/fhir/rest/param/SpecialParamTest.java b/hapi-fhir-base/src/test/java/ca/uhn/fhir/rest/param/SpecialParamTest.java new file mode 100644 index 00000000000..1143aeebaf9 --- /dev/null +++ b/hapi-fhir-base/src/test/java/ca/uhn/fhir/rest/param/SpecialParamTest.java @@ -0,0 +1,67 @@ +package ca.uhn.fhir.rest.param; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.api.Constants; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.LoggerFactory; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(MockitoExtension.class) +public class SpecialParamTest { + + private static final Logger ourLog = (Logger) LoggerFactory.getLogger(StringParam.class); + private ListAppender myListAppender = new ListAppender<>(); + + @Mock + private FhirContext myContext; + + @BeforeEach + public void beforeEach(){ + myListAppender = new ListAppender<>(); + myListAppender.start(); + ourLog.addAppender(myListAppender); + } + + @AfterEach + public void afterEach(){ + myListAppender.stop(); + } + + @Test + public void testEquals() { + SpecialParam specialParam = new SpecialParam(); + specialParam.setValueAsQueryToken(myContext, Constants.PARAM_TEXT, Constants.PARAMQUALIFIER_STRING_CONTAINS, "my-test-value"); + + SpecialParam specialParam2 = new SpecialParam(); + specialParam2.setValueAsQueryToken(myContext, Constants.PARAM_TEXT, Constants.PARAMQUALIFIER_STRING_CONTAINS, "my-test-value"); + assertThat(specialParam).isEqualTo(specialParam2); + } + + @Test + public void testContainsOnlyWorksForSpecificParams() { + SpecialParam specialParamText = new SpecialParam(); + specialParamText.setValueAsQueryToken(myContext, Constants.PARAM_TEXT, Constants.PARAMQUALIFIER_STRING_CONTAINS, "my-test-value"); + assertTrue(specialParamText.isContains()); + + SpecialParam specialParamContent = new SpecialParam(); + specialParamContent.setValueAsQueryToken(myContext, Constants.PARAM_CONTENT, Constants.PARAMQUALIFIER_STRING_CONTAINS, "my-test-value"); + assertTrue(specialParamContent.isContains()); + + SpecialParam nonTextSpecialParam = new SpecialParam(); + nonTextSpecialParam.setValueAsQueryToken(myContext, "name", Constants.PARAMQUALIFIER_STRING_CONTAINS, "my-test-value"); + assertFalse(nonTextSpecialParam.isContains()); + } + + +} diff --git a/hapi-fhir-base/src/test/java/ca/uhn/fhir/rest/param/StringParamTest.java b/hapi-fhir-base/src/test/java/ca/uhn/fhir/rest/param/StringParamTest.java index f5234b4ab75..902cd11b38e 100644 --- a/hapi-fhir-base/src/test/java/ca/uhn/fhir/rest/param/StringParamTest.java +++ b/hapi-fhir-base/src/test/java/ca/uhn/fhir/rest/param/StringParamTest.java @@ -23,10 +23,7 @@ import java.util.stream.Collectors; import static ca.uhn.fhir.rest.api.Constants.PARAMQUALIFIER_STRING_TEXT; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; @ExtendWith(MockitoExtension.class) public class StringParamTest { diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-failed-text-and-content-search-in-r5.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-failed-text-and-content-search-in-r5.yaml new file mode 100644 index 00000000000..13dfe63ecde --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-failed-text-and-content-search-in-r5.yaml @@ -0,0 +1,4 @@ +--- +type: fix +issue: 6046 +title: "Previously, using `_text` and `_content` searches in Hibernate Search in R5 was not supported. This issue has been fixed." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-text-contains.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-text-contains.yaml new file mode 100644 index 00000000000..1dbf3931cec --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6046-text-contains.yaml @@ -0,0 +1,5 @@ +--- +type: add +issue: 6046 +title: "Added support for `:contains` parameter qualifier on the `_text` and `_content` Search Parameters. When using Hibernate Search, this will cause +the search to perform an substring match on the provided value. Documentation can be found [here](/hapi-fhir/docs/server_jpa/elastic.html#performing-fulltext-search-in-luceneelasticsearch)." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/elastic.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/elastic.md index b32a029766a..750a525b5ca 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/elastic.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/elastic.md @@ -3,6 +3,55 @@ The HAPI JPA Server supports optional indexing via Hibernate Search when configured to use Lucene or Elasticsearch. This is required to support the `_content`, or `_text` search parameters. +# Performing Fulltext Search in Lucene/Elasticsearch + +When enabled, searches for `_text` and `_content` are forwarded to the underlying Hibernate Search engine, which can be backed by either Elasticsearch or Lucene. +By default, search is supported in the way indicated in the [FHIR Specification on _text/_content Search](https://www.hl7.org/fhir/search.html#_text). This means that +queries like the following can be evaluated: + +```http request +GET [base]/Observation?_content=cancer OR metastases OR tumor +``` +To understand how this works, look at the following example. During ingestion, the fields required for `_content` and `_text` searches are stored in the backing engine, after undergoing normalization and analysis. For example consider this Observation: + +```json +{ + "resourceType" : "Observation", + "code" : { + "coding" : [{ + "system" : "http://loinc.org", + "code" : "15074-8", + "display" : "Glucose [Moles/volume] in Blood Found during patient's visit!" + }] + } + "valueQuantity" : { + "value" : 6.3, + "unit" : "mmol/l", + "system" : "http://unitsofmeasure.org", + "code" : "mmol/L" + } +} +``` + +In the display section, once parsed and analyzed, will result in the followings tokens being generated to be able to be searched on: + +```json +["glucose", "mole", "volume", "blood", "found", "during", "patient", "visit"] +``` + +You will notice that plurality is removed, and the text has been normalized, and special characters removed. When searched for, the search terms will be normalized in the same fashion. + +However, the default implementation will not allow you to search for an exact match over a long string that contains special characters or other characters which could be broken apart during tokenization. E.g. an exact match for `_content=[Moles/volume]` would not return this result. + +In order to perform such an exact string match in Lucene/Elasticsearch, you should modify the `_text` or `_content` Search Parameter with the `:contains` modifier, as follows: + +```http request +GET [base]/Observation?_content:contains=[Moles/volume] +``` + +Using `:contains` on the `_text` or `_content` modifies the search engine to perform a direct substring match anywhere within the field. + + # Experimental Extended Lucene/Elasticsearch Indexing Additional indexing is implemented for simple search parameters of type token, string, and reference. @@ -68,19 +117,19 @@ The `:text` modifier provides the same [modified Simple Query Syntax](#modified- See https://www.hl7.org/fhir/search.html#token. ## Supported Common and Special Search Parameters -| Parameter | Supported | type | -|--------------|-----------|--------| -| _id | no | | -| _lastUpdated | yes | date | -| _tag | yes | token | -| _profile | yes | URI | -| _security | yes | token | -| _text | yes | string | -| _content | yes | string | -| _list | no | | -| _has | no | | -| _type | no | | -| _source | yes | URI | +| Parameter | Supported | type | +|--------------|-----------|------------------------| +| _id | no | | +| _lastUpdated | yes | date | +| _tag | yes | token | +| _profile | yes | URI | +| _security | yes | token | +| _text | yes | string(R4) special(R5) | +| _content | yes | string(R4) special(R5) | +| _list | no | | +| _has | no | | +| _type | no | | +| _source | yes | URI | ## ValueSet autocomplete extension diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchClauseBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchClauseBuilder.java index e7e72bdbd67..37a2a8830ef 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchClauseBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchClauseBuilder.java @@ -36,6 +36,7 @@ import ca.uhn.fhir.rest.param.NumberParam; import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.QuantityParam; import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.SpecialParam; import ca.uhn.fhir.rest.param.StringParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.rest.param.UriParam; @@ -122,14 +123,12 @@ public class ExtendedHSearchClauseBuilder { } @Nonnull - private Set extractOrStringParams(List nextAnd) { + private Set extractOrStringParams(String theSearchParamName, List nextAnd) { Set terms = new HashSet<>(); for (IQueryParameterType nextOr : nextAnd) { String nextValueTrimmed; - if (nextOr instanceof StringParam) { - StringParam nextOrString = (StringParam) nextOr; - nextValueTrimmed = - StringUtils.defaultString(nextOrString.getValue()).trim(); + if (isStringParamOrEquivalent(theSearchParamName, nextOr)) { + nextValueTrimmed = getTrimmedStringValue(nextOr); } else if (nextOr instanceof TokenParam) { TokenParam nextOrToken = (TokenParam) nextOr; nextValueTrimmed = nextOrToken.getValue(); @@ -150,6 +149,34 @@ public class ExtendedHSearchClauseBuilder { return terms; } + private String getTrimmedStringValue(IQueryParameterType nextOr) { + String value; + if (nextOr instanceof StringParam) { + value = ((StringParam) nextOr).getValue(); + } else if (nextOr instanceof SpecialParam) { + value = ((SpecialParam) nextOr).getValue(); + } else { + throw new IllegalArgumentException(Msg.code(2535) + + "Failed to extract value for fulltext search from parameter. Needs to be a `string` parameter, or `_text` or `_content` special parameter." + + nextOr); + } + return StringUtils.defaultString(value).trim(); + } + + /** + * String Search params are valid, so are two special params, _content and _text. + * + * @param theSearchParamName The name of the SP + * @param nextOr the or values of the query parameter. + * + * @return a boolean indicating whether we can treat this as a string. + */ + private static boolean isStringParamOrEquivalent(String theSearchParamName, IQueryParameterType nextOr) { + List specialSearchParamsToTreatAsStrings = List.of(Constants.PARAM_TEXT, Constants.PARAM_CONTENT); + return (nextOr instanceof StringParam) + || (nextOr instanceof SpecialParam && specialSearchParamsToTreatAsStrings.contains(theSearchParamName)); + } + public void addTokenUnmodifiedSearch(String theSearchParamName, List> theAndOrTerms) { if (CollectionUtils.isEmpty(theAndOrTerms)) { return; @@ -229,22 +256,57 @@ public class ExtendedHSearchClauseBuilder { break; } - for (List nextOrList : stringAndOrTerms) { - Set orTerms = TermHelper.makePrefixSearchTerm(extractOrStringParams(nextOrList)); - ourLog.debug("addStringTextSearch {}, {}", theSearchParamName, orTerms); - if (!orTerms.isEmpty()) { - String query = orTerms.stream().map(s -> "( " + s + " )").collect(Collectors.joining(" | ")); - myRootClause.must(myRootContext - .simpleQueryString() - .field(fieldName) - .matching(query) - .defaultOperator( - BooleanOperator - .AND)); // term value may contain multiple tokens. Require all of them to be - // present. - } else { - ourLog.warn("No Terms found in query parameter {}", nextOrList); + if (isContainsSearch(theSearchParamName, stringAndOrTerms)) { + for (List nextOrList : stringAndOrTerms) { + addPreciseMatchClauses(theSearchParamName, nextOrList, fieldName); } + } else { + for (List nextOrList : stringAndOrTerms) { + addSimpleQueryMatchClauses(theSearchParamName, nextOrList, fieldName); + } + } + } + + /** + * This route is used for standard string searches, or `_text` or `_content`. For each term, we build a `simpleQueryString `element which allows hibernate search to search on normalized, analyzed, indexed fields. + * + * @param theSearchParamName The name of the search parameter + * @param nextOrList the list of query parameters + * @param fieldName the field name in the index document to compare with. + */ + private void addSimpleQueryMatchClauses( + String theSearchParamName, List nextOrList, String fieldName) { + Set orTerms = TermHelper.makePrefixSearchTerm(extractOrStringParams(theSearchParamName, nextOrList)); + ourLog.debug("addStringTextSearch {}, {}", theSearchParamName, orTerms); + if (!orTerms.isEmpty()) { + String query = orTerms.stream().map(s -> "( " + s + " )").collect(Collectors.joining(" | ")); + myRootClause.must(myRootContext + .simpleQueryString() + .field(fieldName) + .matching(query) + .defaultOperator( + BooleanOperator.AND)); // term value may contain multiple tokens. Require all of them to + // be + // present. + + } else { + ourLog.warn("No Terms found in query parameter {}", nextOrList); + } + } + + /** + * Note that this `match()` operation is different from out standard behaviour, which uses simpleQueryString(). This `match()` forces a precise string match, Whereas `simpleQueryString()` uses a more nebulous + * and loose check against a collection of terms. We only use this when we see ` _text:contains=` or `_content:contains=` search. + * + * @param theSearchParamName the Name of the search parameter + * @param nextOrList the list of query parameters + * @param fieldName the field name in the index document to compare with. + */ + private void addPreciseMatchClauses( + String theSearchParamName, List nextOrList, String fieldName) { + Set orTerms = TermHelper.makePrefixSearchTerm(extractOrStringParams(theSearchParamName, nextOrList)); + for (String orTerm : orTerms) { + myRootClause.must(myRootContext.match().field(fieldName).matching(orTerm)); } } @@ -252,7 +314,7 @@ public class ExtendedHSearchClauseBuilder { String fieldPath = joinPath(SEARCH_PARAM_ROOT, theSearchParamName, INDEX_TYPE_STRING, IDX_STRING_EXACT); for (List nextAnd : theStringAndOrTerms) { - Set terms = extractOrStringParams(nextAnd); + Set terms = extractOrStringParams(theSearchParamName, nextAnd); ourLog.debug("addStringExactSearch {} {}", theSearchParamName, terms); List orTerms = terms.stream() .map(s -> myRootContext.match().field(fieldPath).matching(s)) @@ -266,7 +328,7 @@ public class ExtendedHSearchClauseBuilder { String theSearchParamName, List> theStringAndOrTerms) { String fieldPath = joinPath(SEARCH_PARAM_ROOT, theSearchParamName, INDEX_TYPE_STRING, IDX_STRING_NORMALIZED); for (List nextAnd : theStringAndOrTerms) { - Set terms = extractOrStringParams(nextAnd); + Set terms = extractOrStringParams(theSearchParamName, nextAnd); ourLog.debug("addStringContainsSearch {} {}", theSearchParamName, terms); List orTerms = terms.stream() // wildcard is a term-level query, so queries aren't analyzed. Do our own normalization first. @@ -294,7 +356,7 @@ public class ExtendedHSearchClauseBuilder { String theSearchParamName, List> theStringAndOrTerms) { PathContext context = contextForFlatSP(theSearchParamName); for (List nextOrList : theStringAndOrTerms) { - Set terms = extractOrStringParams(nextOrList); + Set terms = extractOrStringParams(theSearchParamName, nextOrList); ourLog.debug("addStringUnmodifiedSearch {} {}", theSearchParamName, terms); List orTerms = terms.stream() .map(s -> buildStringUnmodifiedClause(s, context)) @@ -317,7 +379,7 @@ public class ExtendedHSearchClauseBuilder { String theSearchParamName, List> theReferenceAndOrTerms) { String fieldPath = joinPath(SEARCH_PARAM_ROOT, theSearchParamName, "reference", "value"); for (List nextAnd : theReferenceAndOrTerms) { - Set terms = extractOrStringParams(nextAnd); + Set terms = extractOrStringParams(theSearchParamName, nextAnd); ourLog.trace("reference unchained search {}", terms); List orTerms = terms.stream() @@ -832,4 +894,17 @@ public class ExtendedHSearchClauseBuilder { return compositeClause; } + + private boolean hasAContainsModifier(List> stringAndOrTerms) { + return stringAndOrTerms.stream() + .flatMap(List::stream) + .anyMatch(next -> + Constants.PARAMQUALIFIER_STRING_CONTAINS.equalsIgnoreCase(next.getQueryParameterQualifier())); + } + + private boolean isContainsSearch(String theSearchParamName, List> stringAndOrTerms) { + return (Constants.PARAM_TEXT.equalsIgnoreCase(theSearchParamName) + || Constants.PARAM_CONTENT.equalsIgnoreCase(theSearchParamName)) + && hasAContainsModifier(stringAndOrTerms); + } } diff --git a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java index 1a8d4178c75..1b254013c95 100644 --- a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java +++ b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java @@ -27,6 +27,9 @@ import ca.uhn.fhir.jpa.search.BaseSourceSearchParameterTestCases; import ca.uhn.fhir.jpa.search.CompositeSearchParameterTestCases; import ca.uhn.fhir.jpa.search.QuantitySearchParameterTestCases; import ca.uhn.fhir.jpa.search.builder.SearchBuilder; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchRestClientFactory; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc; @@ -42,6 +45,7 @@ import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.SystemRequestDetails; import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; import ca.uhn.fhir.rest.param.ReferenceParam; +import ca.uhn.fhir.rest.param.SpecialParam; import ca.uhn.fhir.rest.param.StringOrListParam; import ca.uhn.fhir.rest.param.StringParam; import ca.uhn.fhir.rest.param.TokenParam; @@ -58,8 +62,15 @@ import ca.uhn.fhir.validation.ValidationResult; import ca.uhn.test.util.LogbackTestExtension; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.annotation.Nonnull; +import jakarta.json.JsonValue; import jakarta.persistence.EntityManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.elasticsearch.client.RequestOptions; import org.hl7.fhir.instance.model.api.IBaseCoding; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; @@ -107,6 +118,7 @@ import org.springframework.test.context.support.DirtiesContextTestExecutionListe import org.springframework.transaction.PlatformTransactionManager; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; +import org.testcontainers.elasticsearch.ElasticsearchContainer; import java.io.IOException; import java.net.URLEncoder; @@ -211,9 +223,13 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl private IFhirResourceDao myQuestionnaireResponseDao; @Autowired private TestHSearchEventDispatcher myHSearchEventDispatcher; + @Autowired + ElasticsearchContainer myElasticsearchContainer; @Mock private IHSearchEventListener mySearchEventListener; + @Autowired + private ElasticsearchSvcImpl myElasticsearchSvc; @BeforeEach @@ -302,7 +318,7 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl } @Test - public void testResourceTextSearch() { + public void testResourceContentSearch() { Observation obs1 = new Observation(); obs1.getCode().setText("Systolic Blood Pressure"); @@ -319,15 +335,182 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl SearchParameterMap map; - map = new SearchParameterMap(); - map.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, new StringParam("systolic")); - assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1)); + { //Content works as String Param + map = new SearchParameterMap(); + map.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, new StringParam("systolic")); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1)); - map = new SearchParameterMap(); - map.add(Constants.PARAM_CONTENT, new StringParam("blood")); - assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1, id2)); + map = new SearchParameterMap(); + map.add(Constants.PARAM_CONTENT, new StringParam("blood")); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1, id2)); + } + + { //_content works as Special Param + map = new SearchParameterMap(); + map.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, new SpecialParam().setValue("systolic")); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1)); + + map = new SearchParameterMap(); + map.add(Constants.PARAM_CONTENT, new SpecialParam().setValue("blood")); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1, id2)); + } } + @Test + public void testResourceTextSearch() { + + Observation obs1 = new Observation(); + obs1.getCode().setText("Systolic Blood Pressure"); + obs1.setStatus(Observation.ObservationStatus.FINAL); + obs1.setValue(new Quantity(123)); + obs1.getNoteFirstRep().setText("obs1"); + obs1.getText().setDivAsString("systolic blood pressure"); + obs1.getText().setStatus(Narrative.NarrativeStatus.ADDITIONAL); + IIdType id1 = myObservationDao.create(obs1, mySrd).getId().toUnqualifiedVersionless(); + + Observation obs2 = new Observation(); + obs2.getCode().setText("Diastolic Blood Pressure"); + obs2.setStatus(Observation.ObservationStatus.FINAL); + obs2.setValue(new Quantity(81)); + obs2.getText().setDivAsString("diastolic blood pressure"); + obs2.getText().setStatus(Narrative.NarrativeStatus.ADDITIONAL); + IIdType id2 = myObservationDao.create(obs2, mySrd).getId().toUnqualifiedVersionless(); + + SearchParameterMap map; + + { //_text works as a string param + + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new StringParam("systolic")); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1)); + + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new StringParam("blood")); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1, id2)); + } + + { //_text works as a special param + + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new SpecialParam().setValue("systolic")); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1)); + + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new SpecialParam().setValue("blood")); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1, id2)); + } + } + + @Test + public void testTextContainsFunctionality() { + String slug = "my-special-@char!"; + Observation obs1 = new Observation(); + obs1.getCode().setText("Systolic Blood Pressure"); + obs1.setStatus(Observation.ObservationStatus.FINAL); + obs1.setValue(new Quantity(123)); + obs1.getNoteFirstRep().setText("obs1"); + obs1.getText().setDivAsString(slug); + obs1.getText().setStatus(Narrative.NarrativeStatus.ADDITIONAL); + IIdType id1 = myObservationDao.create(obs1, mySrd).getId().toUnqualifiedVersionless(); + + Observation obs2 = new Observation(); + obs2.getCode().setText("Diastolic Blood Pressure"); + obs2.setStatus(Observation.ObservationStatus.FINAL); + obs2.setValue(new Quantity(81)); + obs2.getText().setDivAsString("diastolic blood pressure"); + obs2.getText().setStatus(Narrative.NarrativeStatus.ADDITIONAL); + myObservationDao.create(obs2, mySrd).getId().toUnqualifiedVersionless(); + + + SearchParameterMap map; + + { //_text + //With :contains + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new SpecialParam().setValue(slug).setContains(true)); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1)); + + //Without :contains + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new SpecialParam().setValue(slug)); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).isEmpty(); + } + + } + + @Test + public void testLudicrouslyLongNarrative() throws IOException { + String slug = "myveryveryveryveryveryveryveryveryveryeryveryveryveryveryveryveryveryveryeryveryveryveryveryveryveryveryveryeryveryveryveryveryveryveryveryveryeryveryveryveryveryveryveryveryverylongemailaddress@hotmail.com"; + + Observation obs1 = new Observation(); + obs1.getCode().setText("Systolic Blood Pressure"); + obs1.setStatus(Observation.ObservationStatus.FINAL); + obs1.setValue(new Quantity(123)); + obs1.getNoteFirstRep().setText("obs1"); + obs1.getText().setDivAsString(get15000CharacterNarrativeIncludingSlugAtStart(slug)); + obs1.getText().setStatus(Narrative.NarrativeStatus.ADDITIONAL); + IIdType id1 = myObservationDao.create(obs1, mySrd).getId().toUnqualifiedVersionless(); + + + Observation obs2 = new Observation(); + obs2.getCode().setText("Diastolic Blood Pressure"); + obs2.setStatus(Observation.ObservationStatus.FINAL); + obs2.setValue(new Quantity(81)); + obs2.getText().setDivAsString("diastolic blood pressure"); + obs2.getText().setStatus(Narrative.NarrativeStatus.ADDITIONAL); + IIdType id2 = myObservationDao.create(obs2, mySrd).getId().toUnqualifiedVersionless(); + + Observation obs3 = new Observation(); + obs3.getCode().setText("Systolic Blood Pressure"); + obs3.setStatus(Observation.ObservationStatus.FINAL); + obs3.setValue(new Quantity(323)); + obs3.getNoteFirstRep().setText("obs3"); + obs3.getText().setDivAsString(get15000CharacterNarrativeIncludingSlugAtEnd(slug)); + obs3.getText().setStatus(Narrative.NarrativeStatus.ADDITIONAL); + IIdType id3 = myObservationDao.create(obs3, mySrd).getId().toUnqualifiedVersionless(); + + + SearchParameterMap map; + + { //_text works as a special param + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new SpecialParam().setValue(slug).setContains(true)); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1, id3)); + + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new SpecialParam().setValue("blood").setContains(true)); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id2)); + } + + { //_text works as a string param + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new StringParam(slug).setContains(true)); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id1, id3)); + + map = new SearchParameterMap(); + map.add(Constants.PARAM_TEXT, new StringParam("blood").setContains(true)); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map))).containsExactlyInAnyOrder(toValues(id2)); + } + } + + private String get15000CharacterNarrativeIncludingSlugAtEnd(String theSlug) { + StringBuilder builder = new StringBuilder(); + int remainingNarrativeLength = 15000 - theSlug.length(); + builder.append(RandomStringUtils.randomAlphanumeric(remainingNarrativeLength)); + builder.append(" "); + builder.append(theSlug); + return builder.toString(); + } + private String get15000CharacterNarrativeIncludingSlugAtStart(String theSlug) { + StringBuilder builder = new StringBuilder(); + int remainingNarrativeLength = 15000 - theSlug.length(); + builder.append(theSlug); + builder.append(" "); + builder.append(RandomStringUtils.randomAlphanumeric(remainingNarrativeLength)); + return builder.toString(); + } + + @Test public void testResourceReferenceSearch() { IIdType patId, encId, obsId; diff --git a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/resources/logback-test.xml b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/resources/logback-test.xml index 84d0a1fcf5f..d70e58ece76 100644 --- a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/resources/logback-test.xml +++ b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/resources/logback-test.xml @@ -8,5 +8,17 @@ + + + + + + + + + + + +