honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Distinguish multi-coded items during token autocomplete (#3429) 

Add nested indexing to support autocomplete of multi-coded items.
This commit is contained in:
michaelabuckley 2022-02-28 19:59:09 -05:00 committed by GitHub
parent b263c7a595
commit 86f7373f16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 496 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java
View File

@ -33,6 +33,7 @@ import ca.uhn.fhir.jpa.model.search.ExtendedLuceneIndexData;
import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions; import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions;
import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteSearch; import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteSearch;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor;
import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams; import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
import ca.uhn.fhir.model.api.IQueryParameterType; import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.Constants;
@ -75,6 +76,8 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
private ISearchParamRegistry mySearchParamRegistry; private ISearchParamRegistry mySearchParamRegistry;
@Autowired @Autowired
private DaoConfig myDaoConfig; private DaoConfig myDaoConfig;
@Autowired
ISearchParamExtractor mySearchParamExtractor;
final private ExtendedLuceneSearchBuilder myAdvancedIndexQueryBuilder = new ExtendedLuceneSearchBuilder(); final private ExtendedLuceneSearchBuilder myAdvancedIndexQueryBuilder = new ExtendedLuceneSearchBuilder();
private Boolean ourDisabled; private Boolean ourDisabled;
@ -89,8 +92,8 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
public ExtendedLuceneIndexData extractLuceneIndexData(IBaseResource theResource, ResourceIndexedSearchParams theNewParams) { public ExtendedLuceneIndexData extractLuceneIndexData(IBaseResource theResource, ResourceIndexedSearchParams theNewParams) {
String resourceType = myFhirContext.getResourceType(theResource); String resourceType = myFhirContext.getResourceType(theResource);
Map<String, RuntimeSearchParam> activeSearchParams = mySearchParamRegistry.getActiveSearchParams(resourceType); Map<String, RuntimeSearchParam> activeSearchParams = mySearchParamRegistry.getActiveSearchParams(resourceType);
ExtendedLuceneIndexExtractor extractor = new ExtendedLuceneIndexExtractor(myFhirContext, activeSearchParams); ExtendedLuceneIndexExtractor extractor = new ExtendedLuceneIndexExtractor(myFhirContext, activeSearchParams, mySearchParamExtractor);
return extractor.extract(theNewParams); return extractor.extract(theResource,theNewParams);
} }
@Override @Override

28
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedLuceneClauseBuilder.java
View File

@ -20,8 +20,8 @@ package ca.uhn.fhir.jpa.dao.search;
* #L% * #L%
*/ */
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.model.api.IQueryParameterType; import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import ca.uhn.fhir.model.api.TemporalPrecisionEnum;
import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.Constants;
@ -46,7 +46,6 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.time.Instant; import java.time.Instant;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
@ -57,6 +56,7 @@ import java.util.stream.Collectors;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_EXACT; import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_EXACT;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_NORMALIZED; import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_NORMALIZED;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_TEXT; import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_TEXT;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.SEARCH_PARAM_ROOT;
import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank;
public class ExtendedLuceneClauseBuilder { public class ExtendedLuceneClauseBuilder {
@ -144,7 +144,7 @@ public class ExtendedLuceneClauseBuilder {
return myPredicateFactory.match().field(getTokenCodeFieldPath(theSearchParamName)).matching(token.getValue()); return myPredicateFactory.match().field(getTokenCodeFieldPath(theSearchParamName)).matching(token.getValue());
} else if (StringUtils.isBlank(token.getValue())) { } else if (StringUtils.isBlank(token.getValue())) {
// system without value // system without value
return myPredicateFactory.match().field("sp." + theSearchParamName + ".token" + ".system").matching(token.getSystem()); return myPredicateFactory.match().field(SEARCH_PARAM_ROOT + "." + theSearchParamName + ".token" + ".system").matching(token.getSystem());
} else { } else {
// system + value // system + value
return myPredicateFactory.match().field(getTokenSystemCodeFieldPath(theSearchParamName)).matching(token.getValueAsQueryToken(this.myFhirContext)); return myPredicateFactory.match().field(getTokenSystemCodeFieldPath(theSearchParamName)).matching(token.getValueAsQueryToken(this.myFhirContext));
@ -167,12 +167,12 @@ public class ExtendedLuceneClauseBuilder {
@Nonnull @Nonnull
public static String getTokenCodeFieldPath(String theSearchParamName) { public static String getTokenCodeFieldPath(String theSearchParamName) {
return "sp." + theSearchParamName + ".token" + ".code"; return SEARCH_PARAM_ROOT + "." + theSearchParamName + ".token" + ".code";
} }
@Nonnull @Nonnull
public static String getTokenSystemCodeFieldPath(@Nonnull String theSearchParamName) { public static String getTokenSystemCodeFieldPath(@Nonnull String theSearchParamName) {
return "sp." + theSearchParamName + ".token" + ".code-system"; return SEARCH_PARAM_ROOT + "." + theSearchParamName + ".token" + ".code-system";
} }
public void addStringTextSearch(String theSearchParamName, List<List<IQueryParameterType>> stringAndOrTerms) { public void addStringTextSearch(String theSearchParamName, List<List<IQueryParameterType>> stringAndOrTerms) {
@ -190,7 +190,7 @@ public class ExtendedLuceneClauseBuilder {
fieldName = "myNarrativeText"; fieldName = "myNarrativeText";
break; break;
default: default:
fieldName = "sp." + theSearchParamName + ".string." + IDX_STRING_TEXT; fieldName = SEARCH_PARAM_ROOT + "." + theSearchParamName + ".string." + IDX_STRING_TEXT;
break; break;
} }
@ -213,7 +213,7 @@ public class ExtendedLuceneClauseBuilder {
} }
public void addStringExactSearch(String theSearchParamName, List<List<IQueryParameterType>> theStringAndOrTerms) { public void addStringExactSearch(String theSearchParamName, List<List<IQueryParameterType>> theStringAndOrTerms) {
String fieldPath = "sp." + theSearchParamName + ".string." + IDX_STRING_EXACT; String fieldPath = SEARCH_PARAM_ROOT + "." + theSearchParamName + ".string." + IDX_STRING_EXACT;
for (List<? extends IQueryParameterType> nextAnd : theStringAndOrTerms) { for (List<? extends IQueryParameterType> nextAnd : theStringAndOrTerms) {
Set<String> terms = extractOrStringParams(nextAnd); Set<String> terms = extractOrStringParams(nextAnd);
@ -227,7 +227,7 @@ public class ExtendedLuceneClauseBuilder {
} }
public void addStringContainsSearch(String theSearchParamName, List<List<IQueryParameterType>> theStringAndOrTerms) { public void addStringContainsSearch(String theSearchParamName, List<List<IQueryParameterType>> theStringAndOrTerms) {
String fieldPath = "sp." + theSearchParamName + ".string." + IDX_STRING_NORMALIZED; String fieldPath = SEARCH_PARAM_ROOT + "." + theSearchParamName + ".string." + IDX_STRING_NORMALIZED;
for (List<? extends IQueryParameterType> nextAnd : theStringAndOrTerms) { for (List<? extends IQueryParameterType> nextAnd : theStringAndOrTerms) {
Set<String> terms = extractOrStringParams(nextAnd); Set<String> terms = extractOrStringParams(nextAnd);
ourLog.debug("addStringContainsSearch {} {}", theSearchParamName, terms); ourLog.debug("addStringContainsSearch {} {}", theSearchParamName, terms);
@ -241,7 +241,7 @@ public class ExtendedLuceneClauseBuilder {
} }
public void addStringUnmodifiedSearch(String theSearchParamName, List<List<IQueryParameterType>> theStringAndOrTerms) { public void addStringUnmodifiedSearch(String theSearchParamName, List<List<IQueryParameterType>> theStringAndOrTerms) {
String fieldPath = "sp." + theSearchParamName + ".string." + IDX_STRING_NORMALIZED; String fieldPath = SEARCH_PARAM_ROOT + "." + theSearchParamName + ".string." + IDX_STRING_NORMALIZED;
for (List<? extends IQueryParameterType> nextAnd : theStringAndOrTerms) { for (List<? extends IQueryParameterType> nextAnd : theStringAndOrTerms) {
Set<String> terms = extractOrStringParams(nextAnd); Set<String> terms = extractOrStringParams(nextAnd);
ourLog.debug("addStringUnmodifiedSearch {} {}", theSearchParamName, terms); ourLog.debug("addStringUnmodifiedSearch {} {}", theSearchParamName, terms);
@ -255,7 +255,7 @@ public class ExtendedLuceneClauseBuilder {
} }
public void addReferenceUnchainedSearch(String theSearchParamName, List<List<IQueryParameterType>> theReferenceAndOrTerms) { public void addReferenceUnchainedSearch(String theSearchParamName, List<List<IQueryParameterType>> theReferenceAndOrTerms) {
String fieldPath = "sp." + theSearchParamName + ".reference.value"; String fieldPath = SEARCH_PARAM_ROOT + "." + theSearchParamName + ".reference.value";
for (List<? extends IQueryParameterType> nextAnd : theReferenceAndOrTerms) { for (List<? extends IQueryParameterType> nextAnd : theReferenceAndOrTerms) {
Set<String> terms = extractOrStringParams(nextAnd); Set<String> terms = extractOrStringParams(nextAnd);
ourLog.trace("reference unchained search {}", terms); ourLog.trace("reference unchained search {}", terms);
@ -362,8 +362,8 @@ public class ExtendedLuceneClauseBuilder {
} }
private PredicateFinalStep generateDateOrdinalSearchTerms(String theSearchParamName, DateParam theDateParam) { private PredicateFinalStep generateDateOrdinalSearchTerms(String theSearchParamName, DateParam theDateParam) {
String lowerOrdinalField = "sp." + theSearchParamName + ".dt.lower-ord"; String lowerOrdinalField = SEARCH_PARAM_ROOT + "." + theSearchParamName + ".dt.lower-ord";
String upperOrdinalField = "sp." + theSearchParamName + ".dt.upper-ord"; String upperOrdinalField = SEARCH_PARAM_ROOT + "." + theSearchParamName + ".dt.upper-ord";
int lowerBoundAsOrdinal; int lowerBoundAsOrdinal;
int upperBoundAsOrdinal; int upperBoundAsOrdinal;
ParamPrefixEnum prefix = theDateParam.getPrefix(); ParamPrefixEnum prefix = theDateParam.getPrefix();
@ -411,8 +411,8 @@ public class ExtendedLuceneClauseBuilder {
} }
private PredicateFinalStep generateDateInstantSearchTerms(String theSearchParamName, DateParam theDateParam) { private PredicateFinalStep generateDateInstantSearchTerms(String theSearchParamName, DateParam theDateParam) {
String lowerInstantField = "sp." + theSearchParamName + ".dt.lower"; String lowerInstantField = SEARCH_PARAM_ROOT + "." + theSearchParamName + ".dt.lower";
String upperInstantField = "sp." + theSearchParamName + ".dt.upper"; String upperInstantField = SEARCH_PARAM_ROOT + "." + theSearchParamName + ".dt.upper";
ParamPrefixEnum prefix = theDateParam.getPrefix(); ParamPrefixEnum prefix = theDateParam.getPrefix();
if (ParamPrefixEnum.NOT_EQUAL == prefix) { if (ParamPrefixEnum.NOT_EQUAL == prefix) {

64
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedLuceneIndexExtractor.java
View File

@ -24,7 +24,12 @@ import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.RuntimeSearchParam; import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.jpa.model.entity.ResourceLink; import ca.uhn.fhir.jpa.model.entity.ResourceLink;
import ca.uhn.fhir.jpa.model.search.ExtendedLuceneIndexData; import ca.uhn.fhir.jpa.model.search.ExtendedLuceneIndexData;
import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor;
import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams; import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
import org.hl7.fhir.instance.model.api.IBase;
import org.hl7.fhir.instance.model.api.IBaseCoding;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import java.util.ArrayList; import java.util.ArrayList;
@ -43,21 +48,25 @@ public class ExtendedLuceneIndexExtractor {
private final FhirContext myContext; private final FhirContext myContext;
private final Map<String, RuntimeSearchParam> myParams; private final Map<String, RuntimeSearchParam> myParams;
private final ISearchParamExtractor mySearchParamExtractor;
public ExtendedLuceneIndexExtractor(FhirContext theContext, Map<String, RuntimeSearchParam> theActiveParams) { public ExtendedLuceneIndexExtractor(FhirContext theContext, Map<String, RuntimeSearchParam> theActiveParams, ISearchParamExtractor theSearchParamExtractor) {
myContext = theContext; myContext = theContext;
myParams = theActiveParams; myParams = theActiveParams;
mySearchParamExtractor = theSearchParamExtractor;
} }
@NotNull @NotNull
public ExtendedLuceneIndexData extract(ResourceIndexedSearchParams theNewParams) { public ExtendedLuceneIndexData extract(IBaseResource theResource, ResourceIndexedSearchParams theNewParams) {
ExtendedLuceneIndexData retVal = new ExtendedLuceneIndexData(myContext); ExtendedLuceneIndexData retVal = new ExtendedLuceneIndexData(myContext);
extractAutocompleteTokens(theResource, retVal);
theNewParams.myStringParams.forEach(nextParam -> theNewParams.myStringParams.forEach(nextParam ->
retVal.addStringIndexData(nextParam.getParamName(), nextParam.getValueExact())); retVal.addStringIndexData(nextParam.getParamName(), nextParam.getValueExact()));
theNewParams.myTokenParams.forEach(nextParam -> theNewParams.myTokenParams.forEach(nextParam ->
retVal.addTokenIndexData(nextParam.getParamName(), nextParam.getSystem(), nextParam.getValue())); retVal.addTokenIndexDataIfNotPresent(nextParam.getParamName(), nextParam.getSystem(), nextParam.getValue()));
theNewParams.myDateParams.forEach(nextParam -> theNewParams.myDateParams.forEach(nextParam ->
retVal.addDateIndexData(nextParam.getParamName(), nextParam.getValueLow(), nextParam.getValueLowDateOrdinal(), retVal.addDateIndexData(nextParam.getParamName(), nextParam.getValueLow(), nextParam.getValueLowDateOrdinal(),
@ -93,4 +102,53 @@ public class ExtendedLuceneIndexExtractor {
return retVal; return retVal;
} }
/**
* Re-extract token parameters so we can distinguish
*/
private void extractAutocompleteTokens(IBaseResource theResource, ExtendedLuceneIndexData theRetVal) {
// we need to re-index token params to match up display with codes.
myParams.values().stream()
.filter(p->p.getParamType() == RestSearchParameterTypeEnum.TOKEN)
// TODO it would be nice to reuse TokenExtractor
.forEach(p-> mySearchParamExtractor.extractValues(p.getPath(), theResource)
.stream()
.forEach(nextValue->indexTokenValue(theRetVal, p, nextValue)
));
}
private void indexTokenValue(ExtendedLuceneIndexData theRetVal, RuntimeSearchParam p, IBase nextValue) {
String nextType = mySearchParamExtractor.toRootTypeName(nextValue);
String spName = p.getName();
switch (nextType) {
case "CodeableConcept":
addToken_CodeableConcept(theRetVal, spName, nextValue);
break;
case "Coding":
addToken_Coding(theRetVal, spName, (IBaseCoding) nextValue);
break;
// TODO share this with TokenExtractor and introduce a ITokenIndexer interface.
// Ignore unknown types for now.
// This is just for autocomplete, and we are focused on Observation.code, category, combo-code, etc.
// case "Identifier":
// mySearchParamExtractor.addToken_Identifier(myResourceTypeName, params, searchParam, value);
// break;
// case "ContactPoint":
// mySearchParamExtractor.addToken_ContactPoint(myResourceTypeName, params, searchParam, value);
// break;
default:
break;
}
}
private void addToken_CodeableConcept(ExtendedLuceneIndexData theRetVal, String theSpName, IBase theValue) {
List<IBase> codings = mySearchParamExtractor.getCodingsFromCodeableConcept(theValue);
for (IBase nextCoding : codings) {
addToken_Coding(theRetVal, theSpName, (IBaseCoding) nextCoding);
}
}
private void addToken_Coding(ExtendedLuceneIndexData theRetVal, String theSpName, IBaseCoding theNextValue) {
theRetVal.addTokenIndexData(theSpName, theNextValue);
}
} }

9
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/LastNAggregation.java
View File

@ -23,7 +23,6 @@ package ca.uhn.fhir.jpa.dao.search;
import com.google.gson.Gson; import com.google.gson.Gson;
import com.google.gson.JsonArray; import com.google.gson.JsonArray;
import com.google.gson.JsonObject; import com.google.gson.JsonObject;
import org.hibernate.search.engine.search.aggregation.AggregationKey;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.util.List; import java.util.List;
@ -31,13 +30,15 @@ import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.stream.StreamSupport; import java.util.stream.StreamSupport;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.SEARCH_PARAM_ROOT;
/** /**
* Builds lastN aggregation, and parse the results * Builds lastN aggregation, and parse the results
*/ */
public class LastNAggregation { public class LastNAggregation {
static final String SP_SUBJECT = "sp.subject.reference.value"; static final String SP_SUBJECT = SEARCH_PARAM_ROOT + ".subject.reference.value";
private static final String SP_CODE_TOKEN_CODE_AND_SYSTEM = "sp.code.token.code-system"; private static final String SP_CODE_TOKEN_CODE_AND_SYSTEM = SEARCH_PARAM_ROOT + ".code.token.code-system";
private static final String SP_DATE_DT_UPPER = "sp.date.dt.upper"; private static final String SP_DATE_DT_UPPER = SEARCH_PARAM_ROOT + ".date.dt.upper";
private static final String GROUP_BY_CODE_SYSTEM_SUB_AGGREGATION = "group_by_code_system"; private static final String GROUP_BY_CODE_SYSTEM_SUB_AGGREGATION = "group_by_code_system";
private static final String MOST_RECENT_EFFECTIVE_SUB_AGGREGATION = "most_recent_effective"; private static final String MOST_RECENT_EFFECTIVE_SUB_AGGREGATION = "most_recent_effective";

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/package-info.java
View File

@ -7,7 +7,7 @@
* This package extends this search to support token, string, and reference parameters via {@link ca.uhn.fhir.jpa.model.entity.ResourceTable#myLuceneIndexData}. * This package extends this search to support token, string, and reference parameters via {@link ca.uhn.fhir.jpa.model.entity.ResourceTable#myLuceneIndexData}.
* When active, the extracted search parameters which are written to the HFJ_SPIDX_* tables are also written to the Lucene index document. * When active, the extracted search parameters which are written to the HFJ_SPIDX_* tables are also written to the Lucene index document.
* For now, we use the existing JPA index entities to populate the {@link ca.uhn.fhir.jpa.model.search.ExtendedLuceneIndexData} * For now, we use the existing JPA index entities to populate the {@link ca.uhn.fhir.jpa.model.search.ExtendedLuceneIndexData}
* in {@link ca.uhn.fhir.jpa.dao.search.ExtendedLuceneIndexExtractor#extract(ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams)} ()} * in {@link ca.uhn.fhir.jpa.dao.search.ExtendedLuceneIndexExtractor#extract(org.hl7.fhir.instance.model.api.IBaseResource, ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams)} ()}
* *
* <h2>Implementation</h2> * <h2>Implementation</h2>
* Both {@link ca.uhn.fhir.jpa.search.builder.SearchBuilder} and {@link ca.uhn.fhir.jpa.dao.LegacySearchBuilder} delegate the * Both {@link ca.uhn.fhir.jpa.search.builder.SearchBuilder} and {@link ca.uhn.fhir.jpa.dao.LegacySearchBuilder} delegate the

40
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/RawElasticJsonBuilder.java Normal file
View File

@ -0,0 +1,40 @@
package ca.uhn.fhir.jpa.search.autocomplete;
import com.google.gson.JsonObject;
import org.apache.commons.lang3.Validate;
import javax.annotation.Nonnull;
public class RawElasticJsonBuilder {
@Nonnull
static JsonObject makeMatchBoolPrefixPredicate(String theFieldName, String queryText) {
JsonObject matchBoolBody = new JsonObject();
matchBoolBody.addProperty(theFieldName, queryText);
JsonObject predicate = new JsonObject();
predicate.add("match_bool_prefix", matchBoolBody);
return predicate;
}
public static JsonObject makeWildcardPredicate(String theFieldName, String theQueryText) {
Validate.notEmpty(theQueryText);
JsonObject params = new JsonObject();
params.addProperty("value", theQueryText);
JsonObject wildcardBody = new JsonObject();
wildcardBody.add(theFieldName, params);
JsonObject predicate = new JsonObject();
predicate.add("wildcard", wildcardBody);
return predicate;
}
@Nonnull
public static JsonObject makeMatchAllPredicate() {
JsonObject o = new JsonObject();
o.add("match_all", new JsonObject());
return o;
}
}

107
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/TokenAutocompleteAggregation.java
View File

@ -20,10 +20,8 @@ package ca.uhn.fhir.jpa.search.autocomplete;
* #L% * #L%
*/ */
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneClauseBuilder;
import com.google.gson.Gson; import com.google.gson.Gson;
import com.google.gson.JsonArray; import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject; import com.google.gson.JsonObject;
import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.DocumentContext;
@ -31,6 +29,7 @@ import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.ParseContext; import com.jayway.jsonpath.ParseContext;
import com.jayway.jsonpath.spi.json.GsonJsonProvider; import com.jayway.jsonpath.spi.json.GsonJsonProvider;
import com.jayway.jsonpath.spi.mapper.GsonMappingProvider; import com.jayway.jsonpath.spi.mapper.GsonMappingProvider;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.Validate;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
@ -38,33 +37,59 @@ import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.StreamSupport; import java.util.stream.StreamSupport;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_TEXT;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.NESTED_SEARCH_PARAM_ROOT;
/** /**
* Compose the autocomplete aggregation, and parse the results. * Compose the autocomplete aggregation, and parse the results.
*/ */
class TokenAutocompleteAggregation { class TokenAutocompleteAggregation {
static final String NESTED_AGG_NAME = "nestedTopNAgg";
/** /**
* Aggregation template json. * Aggregation template json.
* *
* https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html
*/ */
static final JsonObject AGGREGATION_TEMPLATE = static final JsonObject AGGREGATION_TEMPLATE =
new Gson().fromJson("{\n" + new Gson().fromJson("" +
" \"terms\": {\n" + " {" +
" \"field\": \"sp.TEMPLATE_DUMMY.token.code-system\",\n" + " \"nested\": { \"path\": \"nsp.PLACEHOLDER\" }," +
" \"size\": 30,\n" + " \"aggs\": {" +
" \"min_doc_count\": 1\n" + " \"search\": {" +
" },\n" + " \"filter\": {" +
" \"aggs\": {\n" + " \"bool\": {" +
" \"" + NESTED_AGG_NAME + "\": {\n" + " \"must\": [" +
" \"top_hits\": {\n" + " { \"match_bool_prefix\":" +
" \"_source\": {\n" + " { \"nsp.PLACEHOLDER.string.text\": {" +
" \"includes\": [ \"sp.TEMPLATE_DUMMY\" ]\n" + " \"query\": \"Mors\"}" +
" },\n" + " }" +
" \"size\": 1\n" + " }" +
" }\n" + " ]" +
" }\n" + " }" +
" }}", JsonObject.class); " }," +
" \"aggs\": {" +
" \"group_by_token\": {" +
" \"terms\": {" +
" \"field\": \"nsp.PLACEHOLDER.token.code-system\"," +
" \"size\": 30," +
" \"min_doc_count\": 1," +
" \"shard_min_doc_count\": 0," +
" \"show_term_doc_count_error\": false" +
" }," +
" \"aggs\": {" +
" \"top_tags_hits\": {" +
" \"top_hits\": {" +
" \"_source\": {" +
" \"includes\": [ \"nsp.PLACEHOLDER\" ]" +
" }," +
" \"size\": 1" +
" }" +
" }" +
" }" +
" }" +
" }" +
" }" +
" }" +
" }", JsonObject.class);
static final Configuration configuration = Configuration static final Configuration configuration = Configuration
.builder() .builder()
@ -75,12 +100,28 @@ class TokenAutocompleteAggregation {
private final String mySpName; private final String mySpName;
private final int myCount; private final int myCount;
private final JsonObject mySearch;
public TokenAutocompleteAggregation(String theSpName, int theCount) { public TokenAutocompleteAggregation(String theSpName, int theCount, String theSearchText, String theSearchModifier) {
Validate.notEmpty(theSpName); Validate.notEmpty(theSpName);
Validate.isTrue(theCount>0, "count must be positive"); Validate.isTrue(theCount>0, "count must be positive");
Validate.isTrue("text".equalsIgnoreCase(theSearchModifier) || "".equals(theSearchModifier) || theSearchModifier == null, "Unsupported search modifier " + theSearchModifier);
mySpName = theSpName; mySpName = theSpName;
myCount = theCount; myCount = theCount;
mySearch = makeSearch(theSearchText, theSearchModifier);
}
private JsonObject makeSearch(String theSearchText, String theSearchModifier) {
theSearchText = StringUtils.defaultString(theSearchText);
theSearchModifier = StringUtils.defaultString(theSearchModifier);
if (StringUtils.isEmpty(theSearchText)) {
return RawElasticJsonBuilder.makeMatchAllPredicate();
} else if ("text".equalsIgnoreCase(theSearchModifier)) {
return RawElasticJsonBuilder.makeMatchBoolPrefixPredicate(NESTED_SEARCH_PARAM_ROOT + "." + mySpName + ".string." + IDX_STRING_TEXT, theSearchText);
} else {
return RawElasticJsonBuilder.makeWildcardPredicate(NESTED_SEARCH_PARAM_ROOT + "." + mySpName + ".token.code", theSearchText + "*");
}
} }
/** /**
@ -92,9 +133,12 @@ class TokenAutocompleteAggregation {
// clone and modify the template with the actual field names. // clone and modify the template with the actual field names.
JsonObject result = AGGREGATION_TEMPLATE.deepCopy(); JsonObject result = AGGREGATION_TEMPLATE.deepCopy();
DocumentContext documentContext = parseContext.parse(result); DocumentContext documentContext = parseContext.parse(result);
documentContext.set("terms.field", ExtendedLuceneClauseBuilder.getTokenSystemCodeFieldPath(mySpName)); String nestedSearchParamPath = NESTED_SEARCH_PARAM_ROOT + "." + mySpName;
documentContext.set("terms.size", myCount); documentContext.set("nested.path", nestedSearchParamPath);
documentContext.set("aggs." + NESTED_AGG_NAME + ".top_hits._source.includes[0]","sp." + mySpName); documentContext.set("aggs.search.filter.bool.must[0]", mySearch);
documentContext.set("aggs.search.aggs.group_by_token.terms.field", NESTED_SEARCH_PARAM_ROOT + "." + mySpName + ".token" + ".code-system");
documentContext.set("aggs.search.aggs.group_by_token.terms.size", myCount);
documentContext.set("aggs.search.aggs.group_by_token.aggs.top_tags_hits.top_hits._source.includes[0]", nestedSearchParamPath);
return result; return result;
} }
@ -108,7 +152,11 @@ class TokenAutocompleteAggregation {
List<TokenAutocompleteHit> extractResults(@Nonnull JsonObject theAggregationResult) { List<TokenAutocompleteHit> extractResults(@Nonnull JsonObject theAggregationResult) {
Validate.notNull(theAggregationResult); Validate.notNull(theAggregationResult);
JsonArray buckets = theAggregationResult.getAsJsonArray("buckets"); JsonArray buckets = theAggregationResult
.getAsJsonObject("search")
.getAsJsonObject("group_by_token")
.getAsJsonArray("buckets");
List<TokenAutocompleteHit> result = StreamSupport.stream(buckets.spliterator(), false) List<TokenAutocompleteHit> result = StreamSupport.stream(buckets.spliterator(), false)
.map(b-> bucketToEntry((JsonObject) b)) .map(b-> bucketToEntry((JsonObject) b))
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -129,16 +177,7 @@ class TokenAutocompleteAggregation {
String bucketKey = documentContext.read("key", String.class); String bucketKey = documentContext.read("key", String.class);
// The inner bucket has a hits array, and we only need the first. // The inner bucket has a hits array, and we only need the first.
JsonObject spRootNode = documentContext.read(NESTED_AGG_NAME + ".hits.hits[0]._source.sp"); String displayText = documentContext.read("top_tags_hits.hits.hits[0]._source.string.text", String.class);
// MB - JsonPath doesn't have placeholders, and I don't want to screw-up quoting mySpName, so read the JsonObject explicitly
JsonObject spNode = spRootNode.getAsJsonObject(mySpName);
JsonElement exactNode = spNode.get("string").getAsJsonObject().get("exact");
String displayText;
if (exactNode.isJsonArray()) {
displayText = exactNode.getAsJsonArray().get(0).getAsString();
} else {
displayText = exactNode.getAsString();
}
return new TokenAutocompleteHit(bucketKey,displayText); return new TokenAutocompleteHit(bucketKey,displayText);
} }

9
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/TokenAutocompleteHit.java
View File

@ -21,6 +21,7 @@ package ca.uhn.fhir.jpa.search.autocomplete;
*/ */
import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.builder.ToStringBuilder;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
@ -42,4 +43,12 @@ class TokenAutocompleteHit {
public String getSystemCode() { public String getSystemCode() {
return mySystemCode; return mySystemCode;
} }
@Override
public String toString() {
return new ToStringBuilder(this)
.append("mySystemCode", mySystemCode)
.append("myDisplayText", myDisplayText)
.toString();
}
} }

61
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/TokenAutocompleteSearch.java
View File

@ -21,18 +21,12 @@ package ca.uhn.fhir.jpa.search.autocomplete;
*/ */
import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneClauseBuilder; import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneClauseBuilder;
import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.rest.param.StringParam;
import com.google.gson.JsonObject; import com.google.gson.JsonObject;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension; import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension;
import org.hibernate.search.engine.search.aggregation.AggregationKey; import org.hibernate.search.engine.search.aggregation.AggregationKey;
import org.hibernate.search.engine.search.aggregation.SearchAggregation; import org.hibernate.search.engine.search.aggregation.SearchAggregation;
import org.hibernate.search.engine.search.predicate.dsl.BooleanPredicateClausesStep;
import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory;
import org.hibernate.search.engine.search.query.SearchResult; import org.hibernate.search.engine.search.query.SearchResult;
import org.hibernate.search.engine.search.query.dsl.SearchQueryOptionsStep; import org.hibernate.search.engine.search.query.dsl.SearchQueryOptionsStep;
import org.hibernate.search.mapper.orm.search.loading.dsl.SearchLoadingOptionsStep; import org.hibernate.search.mapper.orm.search.loading.dsl.SearchLoadingOptionsStep;
@ -41,7 +35,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.util.Collections;
import java.util.List; import java.util.List;
import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank;
@ -63,23 +56,28 @@ class TokenAutocompleteSearch {
/** /**
* Search for tokens indexed by theSPName on theResourceType matching theSearchText. * Search for tokens indexed by theSPName on theResourceName matching theSearchText.
* @param theResourceType The resource type (e.g. Observation) * @param theResourceName The resource type (e.g. Observation)
* @param theSPName The search param code (e.g. combo-code) * @param theSPName The search param code (e.g. combo-code)
* @param theSearchText The search test (e.g. "bloo") * @param theSearchText The search test (e.g. "bloo")
* @return A collection of Coding elements * @return A collection of Coding elements
*/ */
@Nonnull @Nonnull
public List<TokenAutocompleteHit> search(String theResourceType, String theSPName, String theSearchText, String theSearchModifier, int theCount) { public List<TokenAutocompleteHit> search(String theResourceName, String theSPName, String theSearchText, String theSearchModifier, int theCount) {
ourLog.trace("search: {}?{}:{}={}", theResourceType,theSPName, theSearchModifier, theSearchText); TokenAutocompleteAggregation tokenAutocompleteAggregation =
new TokenAutocompleteAggregation(theSPName, theCount, theSearchText, theSearchModifier);
TokenAutocompleteAggregation tokenAutocompleteAggregation = new TokenAutocompleteAggregation(theSPName, theCount);
// compose the query json // compose the query json
SearchQueryOptionsStep<?, ?, SearchLoadingOptionsStep, ?, ?> query = mySession.search(ResourceTable.class) SearchQueryOptionsStep<?, ?, SearchLoadingOptionsStep, ?, ?> query = mySession.search(ResourceTable.class)
.where(f -> f.bool(b -> .where(predFactory -> predFactory.bool(boolBuilder -> {
buildQueryPredicate(b, f, theResourceType, theSPName, theSearchModifier, theSearchText))) ExtendedLuceneClauseBuilder clauseBuilder = new ExtendedLuceneClauseBuilder(myFhirContext, boolBuilder, predFactory);
// we apply resource-level predicates here, at the top level
if (isNotBlank(theResourceName)) {
clauseBuilder.addResourceTypeClause(theResourceName);
}
}))
.aggregation(AGGREGATION_KEY, buildAggregation(tokenAutocompleteAggregation)); .aggregation(AGGREGATION_KEY, buildAggregation(tokenAutocompleteAggregation));
// run the query, but with 0 results. We only care about the aggregations. // run the query, but with 0 results. We only care about the aggregations.
@ -92,39 +90,6 @@ class TokenAutocompleteSearch {
return aggEntries; return aggEntries;
} }
void buildQueryPredicate(BooleanPredicateClausesStep<?> b, SearchPredicateFactory f, String theResourceType, String theSPName, String theSearchModifier, String theSearchText) {
ExtendedLuceneClauseBuilder clauseBuilder = new ExtendedLuceneClauseBuilder(myFhirContext, b, f);
if (isNotBlank(theResourceType)) {
clauseBuilder.addResourceTypeClause(theResourceType);
}
String queryText = StringUtils.defaultString(theSearchText, "");
if (StringUtils.isNotEmpty(queryText)) {
switch (StringUtils.defaultString(theSearchModifier)) {
case "text":
// Add a wildcard to act like match_bool_prefix
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-bool-prefix-query.html
queryText = queryText + "*";
StringParam stringParam = new StringParam(queryText);
List<List<IQueryParameterType>> andOrTerms = Collections.singletonList(Collections.singletonList(stringParam));
clauseBuilder.addStringTextSearch(theSPName, andOrTerms);
break;
case "":
b.must(
// use wildcard to allow matching prefix of keyword indexed field.
f.wildcard()
.field(ExtendedLuceneClauseBuilder.getTokenCodeFieldPath(theSPName))
.matching(queryText + "*")
.toPredicate());
break;
default:
throw new IllegalArgumentException(Msg.code(2034) + "Autocomplete only accepts text search for now.");
}
}
}
/** /**
* Hibernate-search doesn't support nested aggregations, so we use an extension to build what we need from raw JSON. * Hibernate-search doesn't support nested aggregations, so we use an extension to build what we need from raw JSON.
*/ */

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/package-info.java
View File

@ -5,13 +5,8 @@
* *
* This work depends on on the Hibernate Search infrastructure in {@link ca.uhn.fhir.jpa.dao.search}. * This work depends on on the Hibernate Search infrastructure in {@link ca.uhn.fhir.jpa.dao.search}.
* *
* NIH sponsored this work to provide an interactive-autocomplete when browsing codes in a research dataset. * Provides an interactive-autocomplete when browsing codes in a dataset.
* *
* https://gitlab.com/simpatico.ai/cdr/-/issues/2452
* wipmb TODO-LIST
* wipmb - docs - no partition support
* wipmb - link to docs
* wipmb what if the sp isn't of type token? do we check, or discard results without tokens?
* *
*/ */
package ca.uhn.fhir.jpa.search.autocomplete; package ca.uhn.fhir.jpa.search.autocomplete;

23
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDataBuilderConfig.java Normal file
View File

@ -0,0 +1,23 @@
package ca.uhn.fhir.jpa.config;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.dao.DaoTestDataBuilder;
import ca.uhn.fhir.jpa.partition.SystemRequestDetails;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class TestDataBuilderConfig {
@Autowired
FhirContext myFhirContext;
@Autowired
DaoRegistry myDaoRegistry;
@Bean
DaoTestDataBuilder testDataBuilder() {
return new DaoTestDataBuilder(myFhirContext, myDaoRegistry, new SystemRequestDetails());
}
}

47
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/autocomplete/RawElasticJsonBuilderTest.java Normal file
View File

@ -0,0 +1,47 @@
package ca.uhn.fhir.jpa.search.autocomplete;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
class RawElasticJsonBuilderTest {
JsonObject myJson;
@Test
public void matchAll() {
myJson = RawElasticJsonBuilder.makeMatchAllPredicate();
assertJson("""
{"match_all": {}}""");
}
@Test
public void wildcard() {
myJson = RawElasticJsonBuilder.makeWildcardPredicate("a.field", "pattern_text");
assertJson("""
{ "wildcard": {
"a.field": {
"value": "pattern_text"
}}}""");
}
@Test
public void matchBoolPrefix() {
myJson = RawElasticJsonBuilder.makeMatchBoolPrefixPredicate("a.field", "query_text");
assertJson("""
{ "match_bool_prefix" : {
"a.field" : "query_text"
}}""");
}
private void assertJson(String expected) {
assertEquals(JsonParser.parseString(expected).getAsJsonObject(), myJson);
}
}

177
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/autocomplete/TokenAutocompleteAggregationTest.java
View File

@ -6,7 +6,6 @@ import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.util.List; import java.util.List;
import java.util.Optional;
import static com.jayway.jsonpath.matchers.JsonPathMatchers.isJson; import static com.jayway.jsonpath.matchers.JsonPathMatchers.isJson;
import static com.jayway.jsonpath.matchers.JsonPathMatchers.withJsonPath; import static com.jayway.jsonpath.matchers.JsonPathMatchers.withJsonPath;
@ -30,8 +29,10 @@ class TokenAutocompleteAggregationTest {
myCode = "combo-code"; myCode = "combo-code";
buildAggregation(); buildAggregation();
assertThat("terms field is sp", myAggJson, isJson(withJsonPath("terms.field", equalTo("sp.combo-code.token.code-system"))));
assertThat("fetched piece is sp", myAggJson, isJson(withJsonPath("aggs.nestedTopNAgg.top_hits._source.includes[0]", equalTo("sp.combo-code")))); assertThat("nested clause includes sp", myAggJson, isJson(withJsonPath("nested.path", equalTo("nsp.combo-code"))));
assertThat("terms field is sp", myAggJson, isJson(withJsonPath("aggs.search.aggs.group_by_token.terms.field", equalTo("nsp.combo-code.token.code-system"))));
assertThat("fetched piece is sp", myAggJson, isJson(withJsonPath("aggs.search.aggs.group_by_token.aggs.top_tags_hits.top_hits._source.includes[0]", equalTo("nsp.combo-code"))));
} }
@Test @Test
@ -40,36 +41,103 @@ class TokenAutocompleteAggregationTest {
myCount = 77; myCount = 77;
buildAggregation(); buildAggregation();
assertThat("terms field is sp", myAggJson, isJson(withJsonPath("terms.size", equalTo(77))));
assertThat("count for top n", myAggJson, isJson(withJsonPath("aggs.search.aggs.group_by_token.terms.size", equalTo(77))));
} }
private void buildAggregation() { private void buildAggregation() {
myAggJson = new TokenAutocompleteAggregation(myCode, myCount).toJsonAggregation().toString(); myAggJson = new TokenAutocompleteAggregation(myCode, myCount, null, null).toJsonAggregation().toString();
} }
} }
@Nested @Nested
public class ResultExtraction { public class ResultExtraction {
// Sample result from elastic for Observation.code // Sample result from elastic for Observation.code
String resultJson = "" + String resultJson = """
"{ \"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0," + {
" \"buckets\": [" + "doc_count": 22770,
" { \"key\": \"http://loinc.org|88262-1\"," + "search": {
" \"doc_count\":3," + "doc_count": 4,
" \"nestedTopNAgg\": " + "group_by_token": {
" { \"hits\":" + "doc_count_error_upper_bound": 0,
" { \"total\":{\"value\":3,\"relation\":\"eq\"}, \"max_score\":1.0," + "sum_other_doc_count": 0,
" \"hits\":[" + "buckets": [
" { \"_index\":\"resourcetable-000001\",\"_type\":\"_doc\",\"_id\":\"13\",\"_score\":1.0," + {
" \"_source\":{\"sp\":{\"code\":" + "key": "http://loinc.org|59460-6",
" { \"string\":{\"exact\":\"Gram positive blood culture panel by Probe in Positive blood culture\",\"text\":\"Gram positive blood culture panel by Probe in Positive blood culture\",\"norm\":\"Gram positive blood culture panel by Probe in Positive blood culture\"}," + "doc_count": 2,
" \"token\":{\"code\":\"88262-1\",\"system\":\"http://loinc.org\",\"code-system\":\"http://loinc.org|88262-1\"}}}}}]}}}," + "top_tags_hits": {
// a second result "hits": {
"{\"key\":\"http://loinc.org|4544-3\",\"doc_count\":1,\"nestedTopNAgg\":{\"hits\":{\"total\":{\"value\":1,\"relation\":\"eq\"},\"max_score\":1.0,\"hits\":[{\"_index\":\"resourcetable-000001\",\"_type\":\"_doc\",\"_id\":\"12\",\"_score\":1.0,\"_source\":{\"sp\":{\"code\":{\"string\":{\"exact\":\"Hematocrit [Volume Fraction] of Blood by Automated count\",\"text\":\"Hematocrit [Volume Fraction] of Blood by Automated count\",\"norm\":\"Hematocrit [Volume Fraction] of Blood by Automated count\"},\"token\":{\"code\":\"4544-3\",\"system\":\"http://loinc.org\",\"code-system\":\"http://loinc.org|4544-3\"}}}}}]}}}," + "total": {
"{\"key\":\"http://loinc.org|4548-4\",\"doc_count\":1,\"nestedTopNAgg\":{\"hits\":{\"total\":{\"value\":1,\"relation\":\"eq\"},\"max_score\":1.0,\"hits\":[{\"_index\":\"resourcetable-000001\",\"_type\":\"_doc\",\"_id\":\"11\",\"_score\":1.0,\"_source\":{\"sp\":{\"code\":{\"string\":{\"exact\":\"Hemoglobin A1c/Hemoglobin.total in Blood\",\"text\":\"Hemoglobin A1c/Hemoglobin.total in Blood\",\"norm\":\"Hemoglobin A1c/Hemoglobin.total in Blood\"},\"token\":{\"code\":\"4548-4\",\"system\":\"http://loinc.org\",\"code-system\":\"http://loinc.org|4548-4\"}}}}}]}}}" + "value": 2,
"]}"; "relation": "eq"
},
"max_score": 4.9845064e-05,
"hits": [
{
"_index": "resourcetable-000001",
"_type": "_doc",
"_id": "1405280",
"_nested": {
"field": "nsp.code",
"offset": 0
},
"_score": 4.9845064e-05,
"_source": {
"string": {
"text": "Fall risk total [Morse Fall Scale]"
},
"token": {
"code": "59460-6",
"system": "http://loinc.org",
"code-system": "http://loinc.org|59460-6"
}
}
}
]
}
}
},
{
"key": "http://loinc.org|59461-4",
"doc_count": 2,
"top_tags_hits": {
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": 4.9845064e-05,
"hits": [
{
"_index": "resourcetable-000001",
"_type": "_doc",
"_id": "1405281",
"_nested": {
"field": "nsp.code",
"offset": 0
},
"_score": 4.9845064e-05,
"_source": {
"string": {
"text": "Fall risk level [Morse Fall Scale]"
},
"token": {
"code": "59461-4",
"system": "http://loinc.org",
"code-system": "http://loinc.org|59461-4"
}
}
}
]
}
}
}
]
}
}
}""";
JsonObject parsedResult = new Gson().fromJson(resultJson, JsonObject.class); JsonObject parsedResult = new Gson().fromJson(resultJson, JsonObject.class);
TokenAutocompleteAggregation myAutocompleteAggregation = new TokenAutocompleteAggregation("code", 22); TokenAutocompleteAggregation myAutocompleteAggregation = new TokenAutocompleteAggregation("code", 22, null, null);
@Test @Test
public void testResultExtraction() { public void testResultExtraction() {
@ -77,69 +145,24 @@ class TokenAutocompleteAggregationTest {
List<TokenAutocompleteHit> hits = myAutocompleteAggregation.extractResults(parsedResult); List<TokenAutocompleteHit> hits = myAutocompleteAggregation.extractResults(parsedResult);
assertThat(hits, is(not(empty()))); assertThat(hits, is(not(empty())));
assertThat(hits, (hasSize(3))); assertThat(hits, (hasSize(2)));
} }
@Test @Test
public void testBucketExtraction() { public void testBucketExtraction() {
JsonObject bucket = (JsonObject) parsedResult.getAsJsonArray("buckets").get(0); JsonObject bucket = parsedResult
.getAsJsonObject("search")
.getAsJsonObject("group_by_token")
.getAsJsonArray("buckets")
.get(0)
.getAsJsonObject();
TokenAutocompleteHit entry = myAutocompleteAggregation.bucketToEntry(bucket); TokenAutocompleteHit entry = myAutocompleteAggregation.bucketToEntry(bucket);
assertThat(entry.mySystemCode, equalTo("http://loinc.org|88262-1")); assertThat(entry.mySystemCode, equalTo("http://loinc.org|59460-6"));
assertThat(entry.myDisplayText, equalTo("Gram positive blood culture panel by Probe in Positive blood culture")); assertThat(entry.myDisplayText, equalTo("Fall risk total [Morse Fall Scale]"));
} }
/**
* Until we move to nested, we may have multiple Coding in a code. This is broken.
*/
@Test
public void testMultiValuedBucketExtraction() {
JsonObject bucket = new Gson().fromJson("{" +
" \"key\": \"http://loinc.org|2708-6\"," +
" \"doc_count\": 14," +
" \"nestedTopNAgg\": {" +
" \"hits\": {" +
" \"total\": {" +
" \"value\": 14," +
" \"relation\": \"eq\"" +
" }," +
" \"max_score\": 1.0000025," +
" \"hits\": [" +
" {" +
" \"_index\": \"resourcetable-000001\"," +
" \"_type\": \"_doc\"," +
" \"_id\": \"1393284\"," +
" \"_score\": 1.0000025," +
" \"_source\": {" +
" \"sp\": {" +
" \"code\": {" +
" \"string\": {" +
" \"exact\": [" +
" \"Oxygen saturation in Arterial blood by Pulse oximetry\"," +
" \"Oxygen saturation in Arterial blood\"" +
" ]" +
" }," +
" \"token\": {" +
" \"code-system\": [" +
" \"http://loinc.org|2708-6\"," +
" \"http://loinc.org|59408-5\"" +
" ]" +
" }" +
" }" +
" }" +
" }" +
" }" +
" ]" +
" }" +
" }" +
"}", JsonObject.class);
TokenAutocompleteHit entry = myAutocompleteAggregation.bucketToEntry(bucket);
assertThat(entry.mySystemCode, equalTo("http://loinc.org|2708-6"));
assertThat(entry.myDisplayText, equalTo("Oxygen saturation in Arterial blood by Pulse oximetry"));
}
} }

53
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/autocomplete/TokenAutocompleteElasticsearchIT.java
View File

@ -6,6 +6,7 @@ import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao; import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportSvc; import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportSvc;
import ca.uhn.fhir.jpa.config.TestDataBuilderConfig;
import ca.uhn.fhir.jpa.config.TestHibernateSearchAddInConfig; import ca.uhn.fhir.jpa.config.TestHibernateSearchAddInConfig;
import ca.uhn.fhir.jpa.config.TestR4Config; import ca.uhn.fhir.jpa.config.TestR4Config;
import ca.uhn.fhir.jpa.dao.BaseJpaTest; import ca.uhn.fhir.jpa.dao.BaseJpaTest;
@ -13,6 +14,7 @@ import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc; import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
import ca.uhn.fhir.test.utilities.ITestDataBuilder;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker; import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import org.hamcrest.Description; import org.hamcrest.Description;
import org.hamcrest.Matcher; import org.hamcrest.Matcher;
@ -35,9 +37,9 @@ import javax.annotation.Nonnull;
import javax.persistence.EntityManager; import javax.persistence.EntityManager;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.stream.Collectors;
import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.hasSize;
@ -46,8 +48,11 @@ import static org.hamcrest.Matchers.not;
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@RequiresDocker @RequiresDocker
@ContextConfiguration(classes = {TestR4Config.class, TestHibernateSearchAddInConfig.Elasticsearch.class}) @ContextConfiguration(classes = {
public class TokenAutocompleteElasticsearchIT extends BaseJpaTest { TestR4Config.class, TestHibernateSearchAddInConfig.Elasticsearch.class, TestDataBuilderConfig.class
})
public class TokenAutocompleteElasticsearchIT extends BaseJpaTest{
public static final Coding erythrocyte_by_volume = new Coding("http://loinc.org", "789-8", "Erythrocytes [#/volume] in Blood by Automated count");
@Autowired @Autowired
protected PlatformTransactionManager myTxManager; protected PlatformTransactionManager myTxManager;
protected ServletRequestDetails mySrd = new ServletRequestDetails(); protected ServletRequestDetails mySrd = new ServletRequestDetails();
@ -72,6 +77,12 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest {
IResourceReindexingSvc myResourceReindexingSvc; IResourceReindexingSvc myResourceReindexingSvc;
@Autowired @Autowired
IBulkDataExportSvc myBulkDataExportSvc; IBulkDataExportSvc myBulkDataExportSvc;
@Autowired
ITestDataBuilder myDataBuilder;
// a few different codes
static final Coding mean_blood_pressure = new Coding("http://loinc.org", "8478-0", "Mean blood pressure");
static final Coding gram_positive_culture = new Coding("http://loinc.org", "88262-1", "Gram positive blood culture panel by Probe in Positive blood culture");
@BeforeEach @BeforeEach
public void beforePurgeDatabase() { public void beforePurgeDatabase() {
@ -88,14 +99,11 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest {
protected PlatformTransactionManager getTxManager() { protected PlatformTransactionManager getTxManager() {
return myTxManager; return myTxManager;
} }
@Test @Test
public void testAutocompleteByCodeDisplay() { public void testAutocompleteByCodeDisplay() {
// a few different codes createObservationWithCode(erythrocyte_by_volume);
Coding mean_blood_pressure = new Coding("http://loinc.org", "8478-0", "Mean blood pressure");
Coding gram_positive_culture = new Coding("http://loinc.org", "88262-1", "Gram positive blood culture panel by Probe in Positive blood culture");
createObservationWithCode(new Coding("http://loinc.org", "789-8", "Erythrocytes [#/volume] in Blood by Automated count"));
createObservationWithCode(mean_blood_pressure); createObservationWithCode(mean_blood_pressure);
createObservationWithCode(mean_blood_pressure); createObservationWithCode(mean_blood_pressure);
createObservationWithCode(new Coding("http://loinc.org", "788-0", "Erythrocyte distribution width [Ratio] by Automated count")); createObservationWithCode(new Coding("http://loinc.org", "788-0", "Erythrocyte distribution width [Ratio] by Automated count"));
@ -146,6 +154,29 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest {
} }
/**
* If an observation has multiple codes, make sure searching by text only matches the right code.
*/
@Test
public void testAutocompleteDistinguishesMultipleCodes() {
createObservationWithCode(erythrocyte_by_volume,mean_blood_pressure,gram_positive_culture);
createObservationWithCode(gram_positive_culture);
createObservationWithCode(mean_blood_pressure);
List<TokenAutocompleteHit> codes = autocompleteSearch("Observation", "code", null, null);
assertThat("null finds all three codes", codes, hasSize(3));
codes = autocompleteSearch("Observation", "code", null, "789");
assertThat("token prefix finds the matching code", codes, hasItem(matchingSystemAndCode(erythrocyte_by_volume)));
assertThat("token prefix finds only the matching code, not all codes on the resource", codes, contains(matchingSystemAndCode(erythrocyte_by_volume)));
codes = autocompleteSearch("Observation", "code", "text", "erythrocyte");
assertThat("text finds the matching code", codes, hasItem(matchingSystemAndCode(erythrocyte_by_volume)));
assertThat("text finds only the matching code, not all codes on the resource", codes, contains(matchingSystemAndCode(erythrocyte_by_volume)));
}
List<TokenAutocompleteHit> autocompleteSearch(String theResourceType, String theSPName, String theModifier, String theSearchText) { List<TokenAutocompleteHit> autocompleteSearch(String theResourceType, String theSPName, String theModifier, String theSearchText) {
return new TransactionTemplate(myTxManager).execute(s -> { return new TransactionTemplate(myTxManager).execute(s -> {
TokenAutocompleteSearch tokenAutocompleteSearch = new TokenAutocompleteSearch(myFhirCtx, Search.session(myEntityManager)); TokenAutocompleteSearch tokenAutocompleteSearch = new TokenAutocompleteSearch(myFhirCtx, Search.session(myEntityManager));
@ -153,9 +184,11 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest {
}); });
} }
private IIdType createObservationWithCode(Coding c) { private IIdType createObservationWithCode(Coding... theCodings) {
Observation obs1 = new Observation(); Observation obs1 = new Observation();
obs1.getCode().addCoding(c); for (Coding coding : theCodings) {
obs1.getCode().addCoding(coding);
}
return myObservationDao.create(obs1, mySrd).getId().toUnqualifiedVersionless(); return myObservationDao.create(obs1, mySrd).getId().toUnqualifiedVersionless();
} }

25
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/ExtendedLuceneIndexData.java
View File

@ -21,16 +21,17 @@ package ca.uhn.fhir.jpa.model.search;
*/ */
import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.model.dstu2.composite.CodingDt;
import com.google.common.collect.HashMultimap; import com.google.common.collect.HashMultimap;
import com.google.common.collect.SetMultimap; import com.google.common.collect.SetMultimap;
import org.hibernate.search.engine.backend.document.DocumentElement; import org.hibernate.search.engine.backend.document.DocumentElement;
import org.hl7.fhir.instance.model.api.IBaseCoding;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.function.BiConsumer;
import java.util.Date; import java.util.Date;
import java.util.Objects;
import java.util.function.BiConsumer;
/** /**
* Collects our lucene extended indexing data. * Collects our lucene extended indexing data.
@ -41,7 +42,7 @@ public class ExtendedLuceneIndexData {
final FhirContext myFhirContext; final FhirContext myFhirContext;
final SetMultimap<String, String> mySearchParamStrings = HashMultimap.create(); final SetMultimap<String, String> mySearchParamStrings = HashMultimap.create();
final SetMultimap<String, TokenParam> mySearchParamTokens = HashMultimap.create(); final SetMultimap<String, IBaseCoding> mySearchParamTokens = HashMultimap.create();
final SetMultimap<String, String> mySearchParamLinks = HashMultimap.create(); final SetMultimap<String, String> mySearchParamLinks = HashMultimap.create();
final SetMultimap<String, DateSearchIndexData> mySearchParamDates = HashMultimap.create(); final SetMultimap<String, DateSearchIndexData> mySearchParamDates = HashMultimap.create();
@ -73,8 +74,19 @@ public class ExtendedLuceneIndexData {
mySearchParamStrings.put(theSpName, theText); mySearchParamStrings.put(theSpName, theText);
} }
public void addTokenIndexData(String theSpName, String theSystem, String theValue) { /**
mySearchParamTokens.put(theSpName, new TokenParam(theSystem, theValue)); * Add if not already present.
*/
public void addTokenIndexDataIfNotPresent(String theSpName, String theSystem, String theValue) {
boolean isPresent = mySearchParamTokens.get(theSpName).stream()
.anyMatch(c -> Objects.equals(c.getSystem(), theSystem) && Objects.equals(c.getCode(), theValue));
if (!isPresent) {
addTokenIndexData(theSpName, new CodingDt(theSystem, theValue));
}
}
public void addTokenIndexData(String theSpName, IBaseCoding theNextValue) {
mySearchParamTokens.put(theSpName, theNextValue);
} }
public void addResourceLinkIndexData(String theSpName, String theTargetResourceId) { public void addResourceLinkIndexData(String theSpName, String theTargetResourceId) {
@ -84,4 +96,5 @@ public class ExtendedLuceneIndexData {
public void addDateIndexData(String theSpName, Date theLowerBound, int theLowerBoundOrdinal, Date theUpperBound, int theUpperBoundOrdinal) { public void addDateIndexData(String theSpName, Date theLowerBound, int theLowerBoundOrdinal, Date theUpperBound, int theUpperBoundOrdinal) {
mySearchParamDates.put(theSpName, new DateSearchIndexData(theLowerBound, theLowerBoundOrdinal, theUpperBound, theUpperBoundOrdinal)); mySearchParamDates.put(theSpName, new DateSearchIndexData(theLowerBound, theLowerBoundOrdinal, theUpperBound, theUpperBoundOrdinal));
} }
} }

26
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/HibernateSearchIndexWriter.java
View File

@ -21,8 +21,9 @@ package ca.uhn.fhir.jpa.model.search;
*/ */
import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.rest.param.TokenParam; import org.apache.commons.lang3.StringUtils;
import org.hibernate.search.engine.backend.document.DocumentElement; import org.hibernate.search.engine.backend.document.DocumentElement;
import org.hl7.fhir.instance.model.api.IBaseCoding;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -31,6 +32,8 @@ public class HibernateSearchIndexWriter {
public static final String IDX_STRING_NORMALIZED = "norm"; public static final String IDX_STRING_NORMALIZED = "norm";
public static final String IDX_STRING_EXACT = "exact"; public static final String IDX_STRING_EXACT = "exact";
public static final String IDX_STRING_TEXT = "text"; public static final String IDX_STRING_TEXT = "text";
public static final String NESTED_SEARCH_PARAM_ROOT = "nsp";
public static final String SEARCH_PARAM_ROOT = "sp";
final HibernateSearchElementCache myNodeCache; final HibernateSearchElementCache myNodeCache;
final FhirContext myFhirContext; final FhirContext myFhirContext;
@ -40,7 +43,7 @@ public class HibernateSearchIndexWriter {
} }
public DocumentElement getSearchParamIndexNode(String theSearchParamName, String theIndexType) { public DocumentElement getSearchParamIndexNode(String theSearchParamName, String theIndexType) {
return myNodeCache.getObjectElement("sp", theSearchParamName, theIndexType); return myNodeCache.getObjectElement(SEARCH_PARAM_ROOT, theSearchParamName, theIndexType);
} }
@ -57,14 +60,25 @@ public class HibernateSearchIndexWriter {
ourLog.debug("Adding Search Param Text: {} -- {}", theSearchParam, theValue); ourLog.debug("Adding Search Param Text: {} -- {}", theSearchParam, theValue);
} }
public void writeTokenIndex(String theSearchParam, TokenParam theValue) { public void writeTokenIndex(String theSearchParam, IBaseCoding theValue) {
DocumentElement nestedRoot = myNodeCache.getObjectElement(NESTED_SEARCH_PARAM_ROOT);
DocumentElement nestedSpNode = nestedRoot.addObject(theSearchParam);
DocumentElement nestedTokenNode = nestedSpNode.addObject("token");
nestedTokenNode.addValue("code", theValue.getCode());
nestedTokenNode.addValue("system", theValue.getSystem());
nestedTokenNode.addValue("code-system", theValue.getSystem() + "|" + theValue.getCode());
if (StringUtils.isNotEmpty(theValue.getDisplay())) {
DocumentElement nestedStringNode = nestedSpNode.addObject("string");
nestedStringNode.addValue(IDX_STRING_TEXT, theValue.getDisplay());
}
DocumentElement tokenIndexNode = getSearchParamIndexNode(theSearchParam, "token"); DocumentElement tokenIndexNode = getSearchParamIndexNode(theSearchParam, "token");
// TODO mb we can use a token_filter with pattern_capture to generate all three off a single value. Do this next, after merge. // TODO mb we can use a token_filter with pattern_capture to generate all three off a single value. Do this next, after merge.
tokenIndexNode.addValue("code", theValue.getValue()); tokenIndexNode.addValue("code", theValue.getCode());
tokenIndexNode.addValue("system", theValue.getSystem()); tokenIndexNode.addValue("system", theValue.getSystem());
//This next one returns as system|value tokenIndexNode.addValue("code-system", theValue.getSystem() + "|" + theValue.getCode());
tokenIndexNode.addValue("code-system", theValue.getValueAsQueryToken(myFhirContext));
ourLog.debug("Adding Search Param Token: {} -- {}", theSearchParam, theValue); ourLog.debug("Adding Search Param Token: {} -- {}", theSearchParam, theValue);
// TODO mb should we write the strings here too? Or leave it to the old spidx indexing?
} }
public void writeReferenceIndex(String theSearchParam, String theValue) { public void writeReferenceIndex(String theSearchParam, String theValue) {

19
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchParamTextPropertyBinder.java
View File

@ -38,7 +38,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.time.Instant; import java.time.Instant;
import java.time.LocalDateTime;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_EXACT; import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_EXACT;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_NORMALIZED; import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_NORMALIZED;
@ -108,17 +107,16 @@ public class SearchParamTextPropertyBinder implements PropertyBinder, PropertyBr
// The following section is a bit ugly. We need to enforce order and dependency or the object matches will be too big. // The following section is a bit ugly. We need to enforce order and dependency or the object matches will be too big.
{ {
IndexSchemaObjectField spfield = indexSchemaElement.objectField("sp", ObjectStructure.FLATTENED); IndexSchemaObjectField spfield = indexSchemaElement.objectField(HibernateSearchIndexWriter.SEARCH_PARAM_ROOT, ObjectStructure.FLATTENED);
spfield.toReference(); spfield.toReference();
IndexSchemaObjectField nestedSpField = indexSchemaElement.objectField(HibernateSearchIndexWriter.NESTED_SEARCH_PARAM_ROOT, ObjectStructure.FLATTENED);
nestedSpField.toReference();
// TODO MB: the lucene/elastic independent api is hurting a bit here. // TODO MB: the lucene/elastic independent api is hurting a bit here.
// For lucene, we need a separate field for each analyzer. So we'll add string (for :exact), and text (for :text). // For lucene, we need a separate field for each analyzer. So we'll add string (for :exact), and text (for :text).
// They aren't marked stored, so there's no space cost beyond the index for each. // They aren't marked stored, so there's no space cost beyond the index for each.
// But for elastic, I'd rather have a single field defined, with multi-field sub-fields. The index cost is the same, // But for elastic, I'd rather have a single field defined, with multi-field sub-fields. The index cost is the same,
// but elastic will actually store all fields in the source document. // but elastic will actually store all fields in the source document.
// Something like this. But we'll need two index writers (lucene vs hibernate).
// ElasticsearchNativeIndexFieldTypeMappingStep nativeStep = indexFieldTypeFactory.extension(ElasticsearchExtension.get()).asNative();
// nativeStep.mapping()
// So triplicate the storage for now. :-( // So triplicate the storage for now. :-(
String stringPathGlob = "*.string"; String stringPathGlob = "*.string";
@ -127,6 +125,9 @@ public class SearchParamTextPropertyBinder implements PropertyBinder, PropertyBr
spfield.fieldTemplate("string-exact", exactAnalyzer).matchingPathGlob(stringPathGlob + "." + IDX_STRING_EXACT).multiValued(); spfield.fieldTemplate("string-exact", exactAnalyzer).matchingPathGlob(stringPathGlob + "." + IDX_STRING_EXACT).multiValued();
spfield.fieldTemplate("string-text", standardAnalyzer).matchingPathGlob(stringPathGlob + "." + IDX_STRING_TEXT).multiValued(); spfield.fieldTemplate("string-text", standardAnalyzer).matchingPathGlob(stringPathGlob + "." + IDX_STRING_TEXT).multiValued();
nestedSpField.objectFieldTemplate("nestedStringIndex", ObjectStructure.FLATTENED).matchingPathGlob(stringPathGlob);
nestedSpField.fieldTemplate("string-text", standardAnalyzer).matchingPathGlob(stringPathGlob + "." + IDX_STRING_TEXT).multiValued();
// token // token
// Ideally, we'd store a single code-system string and use a custom tokenizer to // Ideally, we'd store a single code-system string and use a custom tokenizer to
// generate "system|" "|code" and "system|code" tokens to support all three. // generate "system|" "|code" and "system|code" tokens to support all three.
@ -139,6 +140,11 @@ public class SearchParamTextPropertyBinder implements PropertyBinder, PropertyBr
spfield.fieldTemplate("token-code-system", keywordFieldType).matchingPathGlob(tokenPathGlob + ".code-system").multiValued(); spfield.fieldTemplate("token-code-system", keywordFieldType).matchingPathGlob(tokenPathGlob + ".code-system").multiValued();
spfield.fieldTemplate("token-system", keywordFieldType).matchingPathGlob(tokenPathGlob + ".system").multiValued(); spfield.fieldTemplate("token-system", keywordFieldType).matchingPathGlob(tokenPathGlob + ".system").multiValued();
nestedSpField.objectFieldTemplate("nestedTokenIndex", ObjectStructure.FLATTENED).matchingPathGlob(tokenPathGlob);
nestedSpField.fieldTemplate("token-code", keywordFieldType).matchingPathGlob(tokenPathGlob + ".code").multiValued();
nestedSpField.fieldTemplate("token-code-system", keywordFieldType).matchingPathGlob(tokenPathGlob + ".code-system").multiValued();
nestedSpField.fieldTemplate("token-system", keywordFieldType).matchingPathGlob(tokenPathGlob + ".system").multiValued();
// reference // reference
// reference // reference
@ -154,6 +160,9 @@ public class SearchParamTextPropertyBinder implements PropertyBinder, PropertyBr
// last, since the globs are matched in declaration order, and * matches even nested nodes. // last, since the globs are matched in declaration order, and * matches even nested nodes.
spfield.objectFieldTemplate("spObject", ObjectStructure.FLATTENED).matchingPathGlob("*"); spfield.objectFieldTemplate("spObject", ObjectStructure.FLATTENED).matchingPathGlob("*");
// we use nested search params for the autocomplete search.
nestedSpField.objectFieldTemplate("nestedSpObject", ObjectStructure.NESTED).matchingPathGlob("*").multiValued();
} }
} }

9
hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java
View File

@ -730,7 +730,7 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
if ("Coding".equals(nextType)) { if ("Coding".equals(nextType)) {
String system = extractValueAsString(myCodingSystemValueChild, theValue); String system = extractValueAsString(myCodingSystemValueChild, theValue);
String code = extractValueAsString(myCodingCodeValueChild, theValue); String code = extractValueAsString(myCodingCodeValueChild, theValue);
return createTokenIndexIfNotBlank(theResourceType, theSearchParam, system, code); return createTokenIndexIfNotBlank(theResourceType, system, code, theSearchParam.getName());
} else { } else {
return null; return null;
} }
@ -1079,7 +1079,7 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
} }
private void createTokenIndexIfNotBlankAndAdd(String theResourceType, Set<BaseResourceIndexedSearchParam> theParams, RuntimeSearchParam theSearchParam, String theSystem, String theValue) { private void createTokenIndexIfNotBlankAndAdd(String theResourceType, Set<BaseResourceIndexedSearchParam> theParams, RuntimeSearchParam theSearchParam, String theSystem, String theValue) {
ResourceIndexedSearchParamToken nextEntity = createTokenIndexIfNotBlank(theResourceType, theSearchParam, theSystem, theValue); ResourceIndexedSearchParamToken nextEntity = createTokenIndexIfNotBlank(theResourceType, theSystem, theValue, theSearchParam.getName());
if (nextEntity != null) { if (nextEntity != null) {
theParams.add(nextEntity); theParams.add(nextEntity);
} }
@ -1090,11 +1090,6 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
myPartitionSettings = thePartitionSettings; myPartitionSettings = thePartitionSettings;
} }
private ResourceIndexedSearchParamToken createTokenIndexIfNotBlank(String theResourceType, RuntimeSearchParam theSearchParam, String theSystem, String theValue) {
String searchParamName = theSearchParam.getName();
return createTokenIndexIfNotBlank(theResourceType, theSystem, theValue, searchParamName);
}
private ResourceIndexedSearchParamToken createTokenIndexIfNotBlank(String theResourceType, String theSystem, String theValue, String searchParamName) { private ResourceIndexedSearchParamToken createTokenIndexIfNotBlank(String theResourceType, String theSystem, String theValue, String searchParamName) {
String system = theSystem; String system = theSystem;
String value = theValue; String value = theValue;

12
hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/ITestDataBuilder.java
View File

@ -24,6 +24,7 @@ import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition; import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.RuntimeResourceDefinition; import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.util.FhirTerser;
import org.hl7.fhir.instance.model.api.IBase; import org.hl7.fhir.instance.model.api.IBase;
import org.hl7.fhir.instance.model.api.IBaseReference; import org.hl7.fhir.instance.model.api.IBaseReference;
import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IBaseResource;
@ -185,13 +186,10 @@ public interface ITestDataBuilder {
default Consumer<IBaseResource> withObservationCode(@Nullable String theSystem, @Nullable String theCode) { default Consumer<IBaseResource> withObservationCode(@Nullable String theSystem, @Nullable String theCode) {
return t -> { return t -> {
ICompositeType codeableConcept = (ICompositeType) getFhirContext().getElementDefinition("CodeableConcept").newInstance(); FhirTerser terser = getFhirContext().newTerser();
IBase coding = getFhirContext().newTerser().addElement(codeableConcept, "coding"); IBase coding = terser.addElement(t, "code.coding");
getFhirContext().newTerser().addElement(coding, "system", theSystem); terser.addElement(coding, "system", theSystem);
getFhirContext().newTerser().addElement(coding, "code", theCode); terser.addElement(coding, "code", theCode);
RuntimeResourceDefinition resourceDef = getFhirContext().getResourceDefinition(t.getClass());
resourceDef.getChildByName("code").getMutator().addValue(t, codeableConcept);
}; };
} }

2
lgtm.yml
View File

@ -4,5 +4,5 @@ extraction:
index: index:
java_version: 17 java_version: 17
maven: maven:
version: 3.8.1 version: 3.8.4