Skip database query when hibernate search fully satisfies search (#3478)

If a query is completely satisfied by Hibernate Search, skip the database.

Co-authored-by: Jaison Baskaran <jaisonb@gmail.com>
This commit is contained in:
michaelabuckley 2022-03-22 18:41:21 -04:00 committed by GitHub
parent 3eb3014d25
commit 92db526786
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 580 additions and 113 deletions

View File

@ -0,0 +1,6 @@
---
type: perf
issue: 3478
title: "When using JPA persistence with Hibernate Search (Lucene or Elasticsearch), simple FHIR queries that can be
satisfied completely by Hibernate Search no longer query the database. Before, every search involved the database,
even when not needed. This is faster when there are many results."

View File

@ -1750,6 +1750,9 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> extends BaseStora
if (myDaoConfig.isAdvancedLuceneIndexing()) {
ExtendedLuceneIndexData luceneIndexData = myFulltextSearchSvc.extractLuceneIndexData(theResource, theNewParams);
theEntity.setLuceneIndexData(luceneIndexData);
if(myDaoConfig.isStoreResourceInLuceneIndex()) {
theEntity.setRawResourceData(theContext.newJsonParser().encodeResourceToString(theResource));
}
}
}
}

View File

@ -816,7 +816,6 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
IBaseResource oldVersion = toResource(theEntity, false);
List<TagDefinition> tags = toTagList(theMetaDel);
for (TagDefinition nextDef : tags) {

View File

@ -23,9 +23,10 @@ package ca.uhn.fhir.jpa.dao;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneClauseBuilder;
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneIndexExtractor;
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneResourceProjection;
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneSearchBuilder;
import ca.uhn.fhir.jpa.dao.search.LastNOperation;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
@ -36,11 +37,9 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor;
import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.parser.IParser;
import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
import ca.uhn.fhir.rest.server.util.ResourceSearchParams;
import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension;
@ -48,7 +47,6 @@ import org.hibernate.search.mapper.orm.Search;
import org.hibernate.search.mapper.orm.session.SearchSession;
import org.hibernate.search.mapper.orm.work.SearchIndexingPlan;
import org.hibernate.search.util.common.SearchException;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.PlatformTransactionManager;
@ -59,6 +57,7 @@ import javax.annotation.Nonnull;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceContextType;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
@ -66,8 +65,6 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FulltextSearchSvcImpl.class);
@Autowired
protected IForcedIdDao myForcedIdDao;
@PersistenceContext(type = PersistenceContextType.TRANSACTION)
private EntityManager myEntityManager;
@Autowired
@ -80,6 +77,9 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
private DaoConfig myDaoConfig;
@Autowired
ISearchParamExtractor mySearchParamExtractor;
@Autowired
IIdHelperService myIdHelperService;
final private ExtendedLuceneSearchBuilder myAdvancedIndexQueryBuilder = new ExtendedLuceneSearchBuilder();
private Boolean ourDisabled;
@ -140,7 +140,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
/*
* Handle _text parameter (resource narrative content)
*
* Positerity:
* Posterity:
* We do not want the HAPI-FHIR dao's to process the
* _text parameter, so we remove it from the map here
*/
@ -158,7 +158,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
/*
* Handle other supported parameters
*/
if (myDaoConfig.isAdvancedLuceneIndexing()) {
if (myDaoConfig.isAdvancedLuceneIndexing() && theParams.getEverythingMode() == null) {
myAdvancedIndexQueryBuilder.addAndConsumeAdvancedQueryClauses(builder, theResourceType, theParams, mySearchParamRegistry);
}
//DROP EARLY HERE IF BOOL IS EMPTY?
@ -181,26 +181,12 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
}
@Override
public List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, RequestDetails theRequest) {
public List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
ResourcePersistentId pid = null;
if (theParams.get(IAnyResource.SP_RES_ID) != null) {
String idParamValue;
IQueryParameterType idParam = theParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
if (idParam instanceof TokenParam) {
TokenParam idParm = (TokenParam) idParam;
idParamValue = idParm.getValue();
} else {
StringParam idParm = (StringParam) idParam;
idParamValue = idParm.getValue();
}
// pid = myIdHelperService.translateForcedIdToPid_(theResourceName, idParamValue, theRequest);
}
ResourcePersistentId referencingPid = pid;
List<ResourcePersistentId> retVal = doSearch(null, theParams, referencingPid);
if (referencingPid != null) {
retVal.add(referencingPid);
List<ResourcePersistentId> retVal = doSearch(null, theParams, theReferencingPid);
if (theReferencingPid != null) {
retVal.add(theReferencingPid);
}
return retVal;
}
@ -271,4 +257,23 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
return convertLongsToResourcePersistentIds(pidList);
}
@Override
public List<IBaseResource> getResources(Collection<Long> thePids) {
SearchSession session = getSearchSession();
List<ExtendedLuceneResourceProjection> rawResourceDataList = session.search(ResourceTable.class)
.select(
f -> f.composite(
(pid, forcedId, resource)-> new ExtendedLuceneResourceProjection(pid, forcedId, resource),
f.field("myId", Long.class),
f.field("myForcedId", String.class),
f.field("myRawResource", String.class))
)
.where(
f -> f.id().matchingAny(thePids) // matches '_id' from resource index
).fetchAllHits();
IParser parser = myFhirContext.newJsonParser();
return rawResourceDataList.stream()
.map(p -> p.toResource(parser))
.collect(Collectors.toList());
}
}

View File

@ -20,18 +20,17 @@ package ca.uhn.fhir.jpa.dao;
* #L%
*/
import java.util.List;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.search.ExtendedLuceneIndexData;
import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
import org.hl7.fhir.instance.model.api.IBaseResource;
import java.util.Collection;
import java.util.List;
public interface IFulltextSearchSvc {
@ -52,7 +51,7 @@ public interface IFulltextSearchSvc {
*/
IBaseResource tokenAutocompleteValueSetSearch(ValueSetAutocompleteOptions theOptions);
List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, RequestDetails theRequest);
List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, ResourcePersistentId theReferencingPid);
boolean isDisabled();
@ -72,4 +71,12 @@ public interface IFulltextSearchSvc {
List<ResourcePersistentId> lastN(SearchParameterMap theParams, Integer theMaximumResults);
/**
* Returns inlined resource stored along with index mappings for matched identifiers
*
* @param thePids raw pids - we dont support versioned references
* @return Resources list or empty if nothing found
*/
List<IBaseResource> getResources(Collection<Long> thePids);
}

View File

@ -20,12 +20,12 @@ package ca.uhn.fhir.jpa.dao;
* #L%
*/
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.context.ComboSearchParamType;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.FhirVersionEnum;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.interceptor.api.HookParams;
import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
import ca.uhn.fhir.interceptor.api.Pointcut;
@ -35,7 +35,6 @@ import ca.uhn.fhir.jpa.api.dao.IDao;
import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
import ca.uhn.fhir.jpa.dao.index.IdHelperService;
import ca.uhn.fhir.jpa.dao.predicate.PredicateBuilder;
import ca.uhn.fhir.jpa.dao.predicate.PredicateBuilderFactory;
import ca.uhn.fhir.jpa.dao.predicate.SearchBuilderJoinEnum;
@ -54,13 +53,10 @@ import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper;
import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
import ca.uhn.fhir.jpa.util.BaseIterator;
import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
import ca.uhn.fhir.jpa.util.QueryChunker;
import ca.uhn.fhir.jpa.util.ScrollableResultsIterator;
import ca.uhn.fhir.jpa.util.SqlQueryList;
@ -80,8 +76,11 @@ import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
import ca.uhn.fhir.util.StopWatch;
import ca.uhn.fhir.util.UrlUtil;
import com.google.common.annotations.VisibleForTesting;
@ -288,7 +287,7 @@ public class LegacySearchBuilder implements ISearchBuilder {
}
if (myParams.getEverythingMode() != null) {
pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest);
pids = queryHibernateSearchForEverythingPids();
} else {
pids = myFulltextSearchSvc.search(myResourceName, myParams);
}
@ -333,6 +332,25 @@ public class LegacySearchBuilder implements ISearchBuilder {
return myQueries;
}
private List<ResourcePersistentId> queryHibernateSearchForEverythingPids() {
ResourcePersistentId pid = null;
if (myParams.get(IAnyResource.SP_RES_ID) != null) {
String idParamValue;
IQueryParameterType idParam = myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
if (idParam instanceof TokenParam) {
TokenParam idParm = (TokenParam) idParam;
idParamValue = idParm.getValue();
} else {
StringParam idParm = (StringParam) idParam;
idParamValue = idParm.getValue();
}
pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue);
}
List<ResourcePersistentId> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid);
return pids;
}
private void doCreateChunkedQueries(List<Long> thePids, SortSpec sort, Integer theOffset, boolean theCount, RequestDetails theRequest, ArrayList<TypedQuery<Long>> theQueries) {
if(thePids.size() < getMaximumPageSize()) {
normalizeIdListForLastNInClause(thePids);

View File

@ -61,6 +61,8 @@ public class ExtendedLuceneIndexExtractor {
public ExtendedLuceneIndexData extract(IBaseResource theResource, ResourceIndexedSearchParams theNewParams) {
ExtendedLuceneIndexData retVal = new ExtendedLuceneIndexData(myContext);
retVal.setForcedId(theResource.getIdElement().getIdPart());
extractAutocompleteTokens(theResource, retVal);
theNewParams.myStringParams.forEach(nextParam ->

View File

@ -0,0 +1,34 @@
package ca.uhn.fhir.jpa.dao.search;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.parser.IParser;
import org.hl7.fhir.instance.model.api.IBaseResource;
/**
* Query result when fetching full resources from Hibernate Search.
*/
public class ExtendedLuceneResourceProjection {
final long myPid;
final String myForcedId;
final String myResourceString;
public ExtendedLuceneResourceProjection(long thePid, String theForcedId, String theResourceString) {
myPid = thePid;
myForcedId = theForcedId;
myResourceString = theResourceString;
}
public IBaseResource toResource(IParser theParser) {
IBaseResource result = theParser.parseResource(myResourceString);
IdDt id;
if (myForcedId != null) {
id = new IdDt(myForcedId);
} else {
id = new IdDt(myPid);
}
result.setId(id);
return result;
}
}

View File

@ -16,6 +16,12 @@
* This pid list is used as a narrowing where clause against the remaining unprocessed search parameters in a jdbc query.
* The actual queries for the different search types (e.g. token, string, modifiers, etc.) are
* generated in {@link ca.uhn.fhir.jpa.dao.search.ExtendedLuceneSearchBuilder}.
* <p>
* Full resource bodies can be stored in the Hibernate Search index.
* The {@link ca.uhn.fhir.jpa.dao.search.ExtendedLuceneResourceProjection} is used to extract these.
* This is currently restricted to LastN, and misses tag changes from $meta-add and $meta-delete since those don't
* update Hibernate Search.
* </p>
*
* <h2>Operation</h2>
* During startup, Hibernate Search uses {@link ca.uhn.fhir.jpa.model.search.SearchParamTextPropertyBinder} to generate a schema.

View File

@ -24,6 +24,7 @@ import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.rest.param.TokenParam;
import org.hibernate.search.mapper.orm.session.SearchSession;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.r4.model.Enumerations;
import org.hl7.fhir.r4.model.ValueSet;
import java.util.List;
@ -47,6 +48,7 @@ public class ValueSetAutocompleteSearch {
ValueSet result = new ValueSet();
ValueSet.ValueSetExpansionComponent expansion = new ValueSet.ValueSetExpansionComponent();
result.setExpansion(expansion);
result.setStatus(Enumerations.PublicationStatus.ACTIVE);
aggEntries.stream()
.map(this::makeCoding)
.forEach(expansion::addContains);

View File

@ -0,0 +1,12 @@
package ca.uhn.fhir.jpa.search.builder;
import java.io.Closeable;
import java.util.Iterator;
public interface ISearchQueryExecutor extends Iterator<Long>, Closeable {
/**
* Narrow the signature - no IOException allowed.
*/
@Override
void close();
}

View File

@ -43,7 +43,6 @@ import ca.uhn.fhir.jpa.dao.IResultIterator;
import ca.uhn.fhir.jpa.dao.ISearchBuilder;
import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
import ca.uhn.fhir.jpa.dao.index.IdHelperService;
import ca.uhn.fhir.jpa.entity.ResourceSearchView;
import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
import ca.uhn.fhir.jpa.model.config.PartitionSettings;
@ -111,7 +110,6 @@ import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceContextType;
import javax.persistence.Query;
import javax.persistence.SqlResultSetMapping;
import javax.persistence.Tuple;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaBuilder;
@ -278,11 +276,11 @@ public class SearchBuilder implements ISearchBuilder {
init(theParams, theSearchUuid, theRequestPartitionId);
ArrayList<SearchQueryExecutor> queries = createQuery(myParams, null, null, null, true, theRequest, null);
List<ISearchQueryExecutor> queries = createQuery(myParams, null, null, null, true, theRequest, null);
if (queries.isEmpty()) {
return Collections.emptyIterator();
}
try (SearchQueryExecutor queryExecutor = queries.get(0)) {
try (ISearchQueryExecutor queryExecutor = queries.get(0)) {
return Lists.newArrayList(queryExecutor.next()).iterator();
}
}
@ -317,20 +315,24 @@ public class SearchBuilder implements ISearchBuilder {
myRequestPartitionId = theRequestPartitionId;
}
private ArrayList<SearchQueryExecutor> createQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCount, RequestDetails theRequest,
SearchRuntimeDetails theSearchRuntimeDetails) {
private List<ISearchQueryExecutor> createQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCount, RequestDetails theRequest,
SearchRuntimeDetails theSearchRuntimeDetails) {
List<ResourcePersistentId> pids = new ArrayList<>();
ArrayList<ISearchQueryExecutor> queries = new ArrayList<>();
if (checkUseHibernateSearch()) {
// we're going to run at least part of the search against the Fulltext service.
List<ResourcePersistentId> fulltextMatchIds;
if (myParams.isLastN()) {
pids = executeLastNAgainstIndex(theMaximumResults);
fulltextMatchIds = executeLastNAgainstIndex(theMaximumResults);
} else if (myParams.getEverythingMode() != null) {
fulltextMatchIds = queryHibernateSearchForEverythingPids();
} else {
pids = queryLuceneForPIDs(theRequest);
fulltextMatchIds = myFulltextSearchSvc.search(myResourceName, myParams);
}
if (theSearchRuntimeDetails != null) {
theSearchRuntimeDetails.setFoundIndexMatchesCount(pids.size());
theSearchRuntimeDetails.setFoundIndexMatchesCount(fulltextMatchIds.size());
HookParams params = new HookParams()
.add(RequestDetails.class, theRequest)
.addIfMatchesType(ServletRequestDetails.class, theRequest)
@ -338,20 +340,39 @@ public class SearchBuilder implements ISearchBuilder {
CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params);
}
if (pids.isEmpty()) {
// Will never match
pids = Collections.singletonList(new ResourcePersistentId(-1L));
List<Long> rawPids = ResourcePersistentId.toLongList(fulltextMatchIds);
// can we skip the database entirely and return the pid list from here?
boolean canSkipDatabase =
// if we processed an AND clause, and it returned nothing, then nothing can match.
rawPids.isEmpty() ||
// Our hibernate search query doesn't respect partitions yet
(!myPartitionSettings.isPartitioningEnabled() &&
// were there AND terms left? Then we still need the db.
theParams.isEmpty() &&
// not every param is a param. :-(
theParams.getNearDistanceParam() == null &&
theParams.getLastUpdated() == null &&
theParams.getEverythingMode() == null &&
theParams.getOffset() == null &&
// or sorting?
theParams.getSort() == null
// todo MB Ugh - review with someone else
//theParams.toNormalizedQueryString(myContext).length() <= 1 &&
);
if (canSkipDatabase) {
queries.add(ResolvedSearchQueryExecutor.from(rawPids));
} else {
// Finish the query in the database for the rest of the search parameters, sorting, partitioning, etc.
// We break the pids into chunks that fit in the 1k limit for jdbc bind params.
new QueryChunker<Long>()
.chunk(rawPids, t -> doCreateChunkedQueries(theParams, t, theOffset, sort, theCount, theRequest, queries));
}
}
ArrayList<SearchQueryExecutor> queries = new ArrayList<>();
if (!pids.isEmpty()) {
new QueryChunker<Long>().chunk(ResourcePersistentId.toLongList(pids), t -> doCreateChunkedQueries(theParams, t, theOffset, sort, theCount, theRequest, queries));
} else {
// do everything in the database.
Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, theMaximumResults, theCount, theRequest, null);
query.ifPresent(t -> queries.add(t));
query.ifPresent(queries::add);
}
return queries;
@ -371,8 +392,10 @@ public class SearchBuilder implements ISearchBuilder {
failIfUsed(Constants.PARAM_CONTENT);
}
// TODO MB someday we'll want a query planner to figure out if we _should_ use the ft index, not just if we can.
return fulltextEnabled && myFulltextSearchSvc.supportsSomeOf(myParams);
// TODO MB someday we'll want a query planner to figure out if we _should_ or _must_ use the ft index, not just if we can.
return fulltextEnabled &&
myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE &&
myFulltextSearchSvc.supportsSomeOf(myParams);
}
private void failIfUsed(String theParamName) {
@ -401,19 +424,26 @@ public class SearchBuilder implements ISearchBuilder {
}
}
private List<ResourcePersistentId> queryLuceneForPIDs(RequestDetails theRequest) {
validateFullTextSearchIsEnabled();
private List<ResourcePersistentId> queryHibernateSearchForEverythingPids() {
ResourcePersistentId pid = null;
if (myParams.get(IAnyResource.SP_RES_ID) != null) {
String idParamValue;
IQueryParameterType idParam = myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
if (idParam instanceof TokenParam) {
TokenParam idParm = (TokenParam) idParam;
idParamValue = idParm.getValue();
} else {
StringParam idParm = (StringParam) idParam;
idParamValue = idParm.getValue();
}
List<ResourcePersistentId> pids;
if (myParams.getEverythingMode() != null) {
pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest);
} else {
pids = myFulltextSearchSvc.search(myResourceName, myParams);
pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue);
}
List<ResourcePersistentId> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid);
return pids;
}
private void doCreateChunkedQueries(SearchParameterMap theParams, List<Long> thePids, Integer theOffset, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<SearchQueryExecutor> theQueries) {
private void doCreateChunkedQueries(SearchParameterMap theParams, List<Long> thePids, Integer theOffset, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<ISearchQueryExecutor> theQueries) {
if (thePids.size() < getMaximumPageSize()) {
normalizeIdListForLastNInClause(thePids);
}
@ -820,14 +850,19 @@ public class SearchBuilder implements ISearchBuilder {
idList.add(resource.getId());
}
return getPidToTagMap(idList);
}
@Nonnull
private Map<Long, Collection<ResourceTag>> getPidToTagMap(List<Long> thePidList) {
Map<Long, Collection<ResourceTag>> tagMap = new HashMap<>();
//-- no tags
if (idList.size() == 0)
if (thePidList.size() == 0)
return tagMap;
//-- get all tags for the idList
Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(idList);
Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(thePidList);
//-- build the map, key = resourceId, value = list of ResourceTag
ResourcePersistentId resourceId;
@ -865,23 +900,55 @@ public class SearchBuilder implements ISearchBuilder {
theResourceListToPopulate.add(null);
}
List<ResourcePersistentId> pids = new ArrayList<>(thePids);
// Can we fast track this loading by checking elastic search?
if (isLoadingFromElasticSearchSupported(theIncludedPids.isEmpty())) {
theResourceListToPopulate.addAll(loadObservationResourcesFromElasticSearch(thePids));
if (isLoadingFromElasticSearchSupported(theIncludedPids)) {
theResourceListToPopulate.addAll(loadResourcesFromElasticSearch(thePids));
} else {
// We only chunk because some jdbc drivers can't handle long param lists.
new QueryChunker<ResourcePersistentId>().chunk(pids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position));
new QueryChunker<ResourcePersistentId>().chunk(thePids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position));
}
}
private boolean isLoadingFromElasticSearchSupported(boolean noIncludePids) {
return noIncludePids && !Objects.isNull(myParams) && myParams.isLastN() && myDaoConfig.isStoreResourceInLuceneIndex()
&& myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3);
/**
* Check if we can load the resources from Hibernate Search instead of the database.
* We assume this is faster.
*
* Hibernate Search only stores the current version, and only if enabled.
* @param theIncludedPids the _include target to check for versioned ids
* @return can we fetch from Hibernate Search?
*/
private boolean isLoadingFromElasticSearchSupported(Collection<ResourcePersistentId> theIncludedPids) {
// todo mb we can be smarter here.
// todo check if theIncludedPids has any with version not null.
// is storage enabled?
return myDaoConfig.isStoreResourceInLuceneIndex() &&
// only support lastN for now.
myParams.isLastN() &&
// do we need to worry about versions?
theIncludedPids.isEmpty() &&
// skip the complexity for metadata in dstu2
myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3);
}
private List<IBaseResource> loadObservationResourcesFromElasticSearch(Collection<ResourcePersistentId> thePids) {
return myIElasticsearchSvc.getObservationResources(thePids);
private List<IBaseResource> loadResourcesFromElasticSearch(Collection<ResourcePersistentId> thePids) {
// Do we use the fulltextsvc via hibernate-search to load resources or be backwards compatible with older ES only impl
// to handle lastN?
if (myDaoConfig.isAdvancedLuceneIndexing() && myDaoConfig.isStoreResourceInLuceneIndex()) {
List<Long> pidList = thePids.stream().map(ResourcePersistentId::getIdAsLong).collect(Collectors.toList());
// todo need to inject metadata - use profile to build resource, tags, and security labels
//Map<Long, Collection<ResourceTag>> pidToTagMap = getPidToTagMap(pidList);
List<IBaseResource> resources = myFulltextSearchSvc.getResources(pidList);
return resources;
} else if (!Objects.isNull(myParams) && myParams.isLastN()) {
// legacy LastN implementation
return myIElasticsearchSvc.getObservationResources(thePids);
} else {
// TODO I wonder if we should drop this path, and only support the new Hibernate Search path.
Validate.isTrue(false, "Unexpected");
return Collections.emptyList();
}
}
/**
@ -1323,6 +1390,41 @@ public class SearchBuilder implements ISearchBuilder {
myDaoConfig = theDaoConfig;
}
/**
* Adapt simple Iterator to our internal query interface.
*/
static class ResolvedSearchQueryExecutor implements ISearchQueryExecutor {
private final Iterator<Long> myIterator;
ResolvedSearchQueryExecutor(Iterable<Long> theIterable) {
this(theIterable.iterator());
}
ResolvedSearchQueryExecutor(Iterator<Long> theIterator) {
myIterator = theIterator;
}
@Nonnull
public static ResolvedSearchQueryExecutor from(List<Long> rawPids) {
return new ResolvedSearchQueryExecutor(rawPids);
}
@Override
public boolean hasNext() {
return myIterator.hasNext();
}
@Override
public Long next() {
return myIterator.next();
}
@Override
public void close() {
// empty
}
}
public class IncludesIterator extends BaseIterator<ResourcePersistentId> implements Iterator<ResourcePersistentId> {
private final RequestDetails myRequest;
@ -1381,11 +1483,11 @@ public class SearchBuilder implements ISearchBuilder {
private boolean myFirst = true;
private IncludesIterator myIncludesIterator;
private ResourcePersistentId myNext;
private SearchQueryExecutor myResultsIterator;
private ISearchQueryExecutor myResultsIterator;
private boolean myStillNeedToFetchIncludes;
private int mySkipCount = 0;
private int myNonSkipCount = 0;
private ArrayList<SearchQueryExecutor> myQueryList = new ArrayList<>();
private List<ISearchQueryExecutor> myQueryList = new ArrayList<>();
private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
mySearchRuntimeDetails = theSearchRuntimeDetails;
@ -1668,15 +1770,4 @@ public class SearchBuilder implements ISearchBuilder {
return thePredicates.toArray(new Predicate[0]);
}
private void validateFullTextSearchIsEnabled() {
if (myFulltextSearchSvc == null) {
if (myParams.containsKey(Constants.PARAM_TEXT)) {
throw new InvalidRequestException(Msg.code(1200) + "Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT);
} else if (myParams.containsKey(Constants.PARAM_CONTENT)) {
throw new InvalidRequestException(Msg.code(1201) + "Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT);
} else {
throw new InvalidRequestException(Msg.code(1202) + "Fulltext search is not enabled on this service, can not process qualifier :text");
}
}
}
}

View File

@ -21,6 +21,7 @@ package ca.uhn.fhir.jpa.search.builder.sql;
*/
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.jpa.search.builder.ISearchQueryExecutor;
import ca.uhn.fhir.jpa.util.ScrollableResultsIterator;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.util.IoUtil;
@ -35,12 +36,10 @@ import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceContextType;
import javax.persistence.Query;
import java.io.Closeable;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.Iterator;
public class SearchQueryExecutor implements Iterator<Long>, Closeable {
public class SearchQueryExecutor implements ISearchQueryExecutor {
private static final Long NO_MORE = -1L;
private static final SearchQueryExecutor NO_VALUE_EXECUTOR = new SearchQueryExecutor();

View File

@ -10,6 +10,7 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.api.server.IBundleProvider;
import ca.uhn.fhir.rest.server.method.SortParameter;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ -45,6 +46,11 @@ public class TestDaoSearch {
myFhirCtx = theFhirCtx;
}
public List<IBaseResource> searchForResources(String theQueryUrl) {
IBundleProvider result = searchForBundleProvider(theQueryUrl);
return result.getAllResources();
}
public List<String> searchForIds(String theQueryUrl) {
// fake out the server url parsing
IBundleProvider result = searchForBundleProvider(theQueryUrl);
@ -55,19 +61,31 @@ public class TestDaoSearch {
public IBundleProvider searchForBundleProvider(String theQueryUrl) {
ResourceSearch search = myMatchUrlService.getResourceSearch(theQueryUrl);
IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(search.getResourceName());
SearchParameterMap map = search.getSearchParameterMap();
map.setLoadSynchronous(true);
SystemRequestDetails request = fakeRequestDetailsFromUrl(theQueryUrl);
SortSpec sort = (SortSpec) new SortParameter(myFhirCtx).translateQueryParametersIntoServerArgument(request, null);
SortSpec sort = (SortSpec) new SortParameter(myFhirCtx).translateQueryParametersIntoServerArgument(fakeRequestDetailsFromUrl(theQueryUrl), null);
if (sort != null) {
map.setSort(sort);
}
IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(search.getResourceName());
IBundleProvider result = dao.search(map, request);
IBundleProvider result = dao.search(map, fakeRequestDetailsFromUrl(theQueryUrl));
return result;
}
public SearchParameterMap toSearchParameters(String theQueryUrl) {
ResourceSearch search = myMatchUrlService.getResourceSearch(theQueryUrl);
SearchParameterMap map = search.getSearchParameterMap();
map.setLoadSynchronous(true);
SortSpec sort = (SortSpec) new SortParameter(myFhirCtx).translateQueryParametersIntoServerArgument(fakeRequestDetailsFromUrl(theQueryUrl), null);
if (sort != null) {
map.setSort(sort);
}
return map;
}
@Nonnull
private SystemRequestDetails fakeRequestDetailsFromUrl(String theQueryUrl) {
SystemRequestDetails request = new SystemRequestDetails();

View File

@ -24,7 +24,6 @@ import java.util.UUID;
import java.util.stream.Collectors;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.matchesPattern;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -34,7 +33,6 @@ import static org.mockito.Mockito.when;
@ExtendWith(SpringExtension.class)
public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN {
@AfterEach
public void reset() {
SearchBuilder.setMaxPageSize50ForTest(false);
@ -127,16 +125,22 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN {
when(mySrd.getParameters()).thenReturn(requestParameters);
List<String> results = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(), null));
verifyResourcesLoadedFromElastic(observationIds, results);
}
void verifyResourcesLoadedFromElastic(List<IIdType> theObservationIds, List<String> theResults) {
List<ResourcePersistentId> expectedArgumentPids = ResourcePersistentId.fromLongList(
observationIds.stream().map(IIdType::getIdPartAsLong).collect(Collectors.toList())
theObservationIds.stream().map(IIdType::getIdPartAsLong).collect(Collectors.toList())
);
ArgumentCaptor<List<ResourcePersistentId>> actualPids = ArgumentCaptor.forClass(List.class);
verify(myElasticsearchSvc, times(1)).getObservationResources(actualPids.capture());
assertThat(actualPids.getValue(), is(expectedArgumentPids));
List<String> expectedObservationList = observationIds.stream()
List<String> expectedObservationList = theObservationIds.stream()
.map(id -> id.toUnqualifiedVersionless().getValue()).collect(Collectors.toList());
assertEquals(results, expectedObservationList);
assertEquals(expectedObservationList, theResults);
}
}

View File

@ -1,17 +1,37 @@
package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
import org.hl7.fhir.instance.model.api.IIdType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.util.List;
import java.util.stream.Collectors;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
/**
* Run entire @see {@link FhirResourceDaoR4SearchLastNIT} test suite this time
* using Extended Lucene index as search target
* using Extended Lucene index as search target.
*
* The other implementation is obsolete, and we can merge these someday.
*/
@ExtendWith(SpringExtension.class)
public class FhirResourceDaoR4SearchLastNUsingExtendedLuceneIndexIT extends FhirResourceDaoR4SearchLastNIT {
// awkward override so we can spy
@SpyBean
@Autowired(required = false)
IFulltextSearchSvc myFulltestSearchSvc;
@BeforeEach
public void enableAdvancedLuceneIndexing() {
@ -23,4 +43,25 @@ public class FhirResourceDaoR4SearchLastNUsingExtendedLuceneIndexIT extends Fhir
myDaoConfig.setAdvancedLuceneIndexing(new DaoConfig().isAdvancedLuceneIndexing());
}
/**
* We pull the resources from Hibernate Search when LastN uses Hibernate Search.
* Override the test verification
*/
@Override
void verifyResourcesLoadedFromElastic(List<IIdType> theObservationIds, List<String> theResults) {
List<Long> expectedArgumentPids =
theObservationIds.stream().map(IIdType::getIdPartAsLong).collect(Collectors.toList());
ArgumentCaptor<List<Long>> actualPids = ArgumentCaptor.forClass(List.class);
verify(myFulltestSearchSvc, times(1)).getResources(actualPids.capture());
assertThat(actualPids.getValue(), is(expectedArgumentPids));
// we don't include the type in the id returned from Hibernate Search for now.
List<String> expectedObservationList = theObservationIds.stream()
.map(id -> id.toUnqualifiedVersionless().getIdPart()).collect(Collectors.toList());
assertEquals(expectedObservationList, theResults);
}
}

View File

@ -10,12 +10,12 @@ import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportJobSchedulingHelper;
import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportSvc;
import ca.uhn.fhir.jpa.config.TestHibernateSearchAddInConfig;
import ca.uhn.fhir.jpa.config.TestR4Config;
import ca.uhn.fhir.jpa.dao.BaseDateSearchDaoTests;
import ca.uhn.fhir.jpa.dao.BaseJpaTest;
import ca.uhn.fhir.jpa.dao.DaoTestDataBuilder;
import ca.uhn.fhir.jpa.dao.TestDaoSearch;
import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
@ -37,10 +37,14 @@ import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.param.TokenParamModifier;
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
import ca.uhn.fhir.test.utilities.ITestDataBuilder;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import ca.uhn.fhir.validation.FhirValidator;
import ca.uhn.fhir.validation.ValidationResult;
import org.hamcrest.Matchers;
import org.hl7.fhir.instance.model.api.IBaseCoding;
import org.hl7.fhir.instance.model.api.IBaseMetaType;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.Bundle;
import org.hl7.fhir.r4.model.CodeSystem;
@ -75,14 +79,22 @@ import java.util.List;
import java.util.stream.Collectors;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.stringContainsInOrder;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ExtendWith(SpringExtension.class)
@RequiresDocker
@ContextConfiguration(classes = {TestR4Config.class, TestHibernateSearchAddInConfig.Elasticsearch.class})
@ContextConfiguration(classes = {
TestR4Config.class,
TestHibernateSearchAddInConfig.Elasticsearch.class,
DaoTestDataBuilder.Config.class,
TestDaoSearch.Config.class
})
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
public static final String URL_MY_CODE_SYSTEM = "http://example.com/my_code_system";
@ -132,6 +144,11 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
private ITermCodeSystemStorageSvc myTermCodeSystemStorageSvc;
@Autowired
private DaoRegistry myDaoRegistry;
@Autowired
ITestDataBuilder myTestDataBuilder;
@Autowired
TestDaoSearch myTestDaoSearch;
@BeforeEach
public void beforePurgeDatabase() {
@ -159,6 +176,7 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
DaoConfig defaultConfig = new DaoConfig();
myDaoConfig.setAllowContainsSearches(defaultConfig.isAllowContainsSearches());
myDaoConfig.setAdvancedLuceneIndexing(defaultConfig.isAdvancedLuceneIndexing());
myDaoConfig.setStoreResourceInLuceneIndex(defaultConfig.isStoreResourceInLuceneIndex());
}
@Test
@ -422,6 +440,10 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
map.add("code", new TokenParam("Bodum").setModifier(TokenParamModifier.TEXT));
assertObservationSearchMatchesNothing("search with shared prefix does not match", map);
}
{
assertObservationSearchMatches("empty params finds everything", "Observation?", id1, id2, id3, id4);
}
}
@Test
@ -515,6 +537,11 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
assertThat(message, toUnqualifiedVersionlessIdValues(myObservationDao.search(map)), containsInAnyOrder(toValues(iIdTypes)));
}
private void assertObservationSearchMatches(String theMessage, String theSearch, IIdType... theIds) {
SearchParameterMap map = myTestDaoSearch.toSearchParameters(theSearch);
assertObservationSearchMatches(theMessage, map, theIds);
}
@Nested
public class WithContainedIndexingIT {
@BeforeEach
@ -762,4 +789,121 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
}
}
/**
* Some queries can be satisfied directly from Hibernate Search.
* We still need at least one query to fetch the resources.
*/
@Nested
public class FastPath {
@BeforeEach
public void enableResourceStorage() {
myDaoConfig.setStoreResourceInLuceneIndex(false);
}
@AfterEach
public void resetResourceStorage() {
myDaoConfig.setStoreResourceInLuceneIndex(new DaoConfig().isStoreResourceInLuceneIndex());
}
@Test
public void simpleTokenSkipsSql() {
IIdType id = myTestDataBuilder.createObservation(myTestDataBuilder.withObservationCode("http://example.com/", "theCode"));
myCaptureQueriesListener.clear();
List<String> ids = myTestDaoSearch.searchForIds("Observation?code=theCode");
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
assertThat(ids, hasSize(1));
assertThat(ids, contains(id.getIdPart()));
assertEquals(1, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "sql just to fetch resources");
}
@Test
public void sortStillRequiresSql() {
IIdType id = myTestDataBuilder.createObservation(myTestDataBuilder.withObservationCode("http://example.com/", "theCode"));
myCaptureQueriesListener.clear();
List<String> ids = myTestDaoSearch.searchForIds("Observation?code=theCode&_sort=code");
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
assertThat(ids, hasSize(1));
assertThat(ids, contains(id.getIdPart()));
assertEquals(2, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "the pids come from elastic, but we use sql to sort, and fetch resources");
}
@Test
public void deletedResourceNotFound() {
IIdType id = myTestDataBuilder.createObservation(myTestDataBuilder.withObservationCode("http://example.com/", "theCode"));
myObservationDao.delete(id);
myCaptureQueriesListener.clear();
List<String> ids = myTestDaoSearch.searchForIds("Observation?code=theCode&_sort=code");
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
assertThat(ids, hasSize(0));
assertEquals(0, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "the pids come from elastic, and nothing to fetch");
}
@Test
public void forcedIdSurvivesWithNoSql() {
IIdType id = myTestDataBuilder.createObservation(
myTestDataBuilder.withObservationCode("http://example.com/", "theCode"),
myTestDataBuilder.withId("forcedid"));
assertThat(id.getIdPart(), equalTo("forcedid"));
myCaptureQueriesListener.clear();
List<String> ids = myTestDaoSearch.searchForIds("Observation?code=theCode");
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
assertThat(ids, hasSize(1));
assertThat(ids, contains(id.getIdPart()));
assertEquals(1, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "just 1 to fetch the resources");
}
/**
* A paranoid test to make sure tags stay with the resource.
*
* Tags live outside the resource, and can be modified by
* Since we lost the id, also check tags in case someone changes metadata processing during ingestion.
*/
@Test
public void tagsSurvive() {
IIdType id = myTestDataBuilder.createObservation(
myTestDataBuilder.withObservationCode("http://example.com/", "theCode"),
myTestDataBuilder.withTag("http://example.com", "aTag"));
myCaptureQueriesListener.clear();
List<IBaseResource> observations = myTestDaoSearch.searchForResources("Observation?code=theCode");
assertThat(observations, hasSize(1));
List<? extends IBaseCoding> tags = observations.get(0).getMeta().getTag();
assertThat(tags, hasSize(1));
assertThat(tags.get(0).getSystem(), equalTo("http://example.com"));
assertThat(tags.get(0).getCode(), equalTo("aTag"));
Meta meta = new Meta();
meta.addTag().setSystem("tag_scheme1").setCode("tag_code1");
meta.addProfile("http://profile/1");
meta.addSecurity().setSystem("seclabel_sys1").setCode("seclabel_code1");
myObservationDao.metaAddOperation(id, meta, mySrd);
observations = myTestDaoSearch.searchForResources("Observation?code=theCode");
assertThat(observations, hasSize(1));
IBaseMetaType newMeta = observations.get(0).getMeta();
assertThat(newMeta.getProfile(), hasSize(1));
assertThat(newMeta.getSecurity(), hasSize(1));
assertThat(newMeta.getTag(), hasSize(2));
}
}
}

View File

@ -190,7 +190,7 @@ public class FhirResourceDaoR4StandardQueriesNoFTTest extends BaseJpaTest {
withRiskAssessmentWithProbabilty(0.6);
assertNotFind("when gt", "/RiskAssessment?probability=0.5");
// fixme we break the spec here.
// TODO we break the spec here. Default search should be approx
// assertFind("when a little gt - default is approx", "/RiskAssessment?probability=0.599");
// assertFind("when a little lt - default is approx", "/RiskAssessment?probability=0.601");
assertFind("when eq", "/RiskAssessment?probability=0.6");
@ -301,10 +301,10 @@ public class FhirResourceDaoR4StandardQueriesNoFTTest extends BaseJpaTest {
assertNotFind("when gt", "/Observation?value-quantity=0.5||mmHg");
assertNotFind("when gt unitless", "/Observation?value-quantity=0.5");
// fixme we break the spec here.
// TODO we break the spec here. Default search should be approx
// assertFind("when a little gt - default is approx", "/Observation?value-quantity=0.599");
// assertFind("when a little lt - default is approx", "/Observation?value-quantity=0.601");
// fixme we don't seem to support "units", only "code".
// TODO we don't seem to support "units", only "code".
assertFind("when eq with units", "/Observation?value-quantity=0.6||mm[Hg]");
assertFind("when eq unitless", "/Observation?value-quantity=0.6");
assertNotFind("when lt", "/Observation?value-quantity=0.7||mmHg");

View File

@ -0,0 +1,40 @@
package ca.uhn.fhir.jpa.dao.search;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.parser.IParser;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.r4.model.Observation;
import org.junit.jupiter.api.Test;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
class ExtendedLuceneResourceProjectionTest {
final FhirContext myFhirContext = FhirContext.forR4();
final IParser myParser = myFhirContext.newJsonParser();
ExtendedLuceneResourceProjection myProjection;
IBaseResource myResource;
@Test
public void basicBodyReceivesId() {
myProjection = new ExtendedLuceneResourceProjection(22, null, "{ \"resourceType\":\"Observation\"}");
myResource = myProjection.toResource(myParser);
assertThat(myResource, instanceOf(Observation.class));
assertThat(myResource.getIdElement().getIdPart(), equalTo("22"));
}
@Test
public void forcedIdOverridesPid() {
myProjection = new ExtendedLuceneResourceProjection(22, "force-id", "{ \"resourceType\":\"Observation\"}");
myResource = myProjection.toResource(myParser);
assertThat(myResource, instanceOf(Observation.class));
assertThat(myResource.getIdElement().getIdPart(), equalTo("force-id"));
}
}

View File

@ -20,6 +20,7 @@ import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.hibernate.search.mapper.orm.Search;
import org.hl7.fhir.instance.model.api.IBaseCoding;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.Coding;
import org.hl7.fhir.r4.model.Observation;
@ -193,7 +194,7 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest{
}
@Nonnull
private Matcher<TokenAutocompleteHit> matchingSystemAndCode(Coding theCoding) {
private Matcher<TokenAutocompleteHit> matchingSystemAndCode(IBaseCoding theCoding) {
return new TypeSafeDiagnosingMatcher<TokenAutocompleteHit>() {
private final String mySystemAndCode = theCoding.getSystem() + "|" + theCoding.getCode();

View File

@ -140,6 +140,13 @@ public class ResourceTable extends BaseHasResource implements Serializable, IBas
@PropertyBinding(binder = @PropertyBinderRef(type = SearchParamTextPropertyBinder.class))
private ExtendedLuceneIndexData myLuceneIndexData;
// todo mb move this to ExtendedLuceneIndexData
@Transient
@GenericField(name="myRawResource", projectable = Projectable.YES, searchable = Searchable.NO)
@IndexingDependency(derivedFrom = @ObjectPath(@PropertyValue(propertyName = "myVersion")))
@OptimisticLock(excluded = true)
private String myRawResourceData;
@OneToMany(mappedBy = "myResource", cascade = {}, fetch = FetchType.LAZY, orphanRemoval = false)
@OptimisticLock(excluded = true)
private Collection<ResourceIndexedSearchParamCoords> myParamsCoords;
@ -775,4 +782,8 @@ public class ResourceTable extends BaseHasResource implements Serializable, IBas
public void setLuceneIndexData(ExtendedLuceneIndexData theLuceneIndexData) {
myLuceneIndexData = theLuceneIndexData;
}
public void setRawResourceData(String theResourceData) {
myRawResourceData = theResourceData;
}
}

View File

@ -45,6 +45,7 @@ public class ExtendedLuceneIndexData {
final SetMultimap<String, IBaseCoding> mySearchParamTokens = HashMultimap.create();
final SetMultimap<String, String> mySearchParamLinks = HashMultimap.create();
final SetMultimap<String, DateSearchIndexData> mySearchParamDates = HashMultimap.create();
String myForcedId;
public ExtendedLuceneIndexData(FhirContext theFhirContext) {
this.myFhirContext = theFhirContext;
@ -58,11 +59,20 @@ public class ExtendedLuceneIndexData {
}
};
}
/**
* Write the index document.
*
* Keep this in sync with the schema defined in {@link SearchParamTextPropertyBinder}
* @param theDocument
*/
public void writeIndexElements(DocumentElement theDocument) {
HibernateSearchIndexWriter indexWriter = HibernateSearchIndexWriter.forRoot(myFhirContext, theDocument);
ourLog.debug("Writing JPA index to Hibernate Search");
theDocument.addValue("myForcedId", myForcedId);
mySearchParamStrings.forEach(ifNotContained(indexWriter::writeStringIndex));
mySearchParamTokens.forEach(ifNotContained(indexWriter::writeTokenIndex));
mySearchParamLinks.forEach(ifNotContained(indexWriter::writeReferenceIndex));
@ -97,4 +107,11 @@ public class ExtendedLuceneIndexData {
mySearchParamDates.put(theSpName, new DateSearchIndexData(theLowerBound, theLowerBoundOrdinal, theUpperBound, theUpperBoundOrdinal));
}
public void setForcedId(String theForcedId) {
myForcedId = theForcedId;
}
public String getForcedId() {
return myForcedId;
}
}

View File

@ -100,11 +100,18 @@ public class SearchParamTextPropertyBinder implements PropertyBinder, PropertyBr
.projectable(Projectable.NO)
.sortable(Sortable.YES);
StringIndexFieldTypeOptionsStep<?> forcedIdType = indexFieldTypeFactory.asString()
.projectable(Projectable.YES)
.aggregable(Aggregable.NO);
// the old style for _text and _contains
indexSchemaElement
.fieldTemplate("SearchParamText", standardAnalyzer)
.matchingPathGlob(SEARCH_PARAM_TEXT_PREFIX + "*");
indexSchemaElement.field("myForcedId", forcedIdType).toReference();
// The following section is a bit ugly. We need to enforce order and dependency or the object matches will be too big.
{
IndexSchemaObjectField spfield = indexSchemaElement.objectField(HibernateSearchIndexWriter.SEARCH_PARAM_ROOT, ObjectStructure.FLATTENED);