Skip database query when hibernate search fully satisfies search (#3478)
If a query is completely satisfied by Hibernate Search, skip the database. Co-authored-by: Jaison Baskaran <jaisonb@gmail.com>
This commit is contained in:
parent
3eb3014d25
commit
92db526786
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
type: perf
|
||||
issue: 3478
|
||||
title: "When using JPA persistence with Hibernate Search (Lucene or Elasticsearch), simple FHIR queries that can be
|
||||
satisfied completely by Hibernate Search no longer query the database. Before, every search involved the database,
|
||||
even when not needed. This is faster when there are many results."
|
|
@ -1750,6 +1750,9 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> extends BaseStora
|
|||
if (myDaoConfig.isAdvancedLuceneIndexing()) {
|
||||
ExtendedLuceneIndexData luceneIndexData = myFulltextSearchSvc.extractLuceneIndexData(theResource, theNewParams);
|
||||
theEntity.setLuceneIndexData(luceneIndexData);
|
||||
if(myDaoConfig.isStoreResourceInLuceneIndex()) {
|
||||
theEntity.setRawResourceData(theContext.newJsonParser().encodeResourceToString(theResource));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -816,7 +816,6 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
|
|||
|
||||
IBaseResource oldVersion = toResource(theEntity, false);
|
||||
|
||||
|
||||
List<TagDefinition> tags = toTagList(theMetaDel);
|
||||
|
||||
for (TagDefinition nextDef : tags) {
|
||||
|
|
|
@ -23,9 +23,10 @@ package ca.uhn.fhir.jpa.dao;
|
|||
import ca.uhn.fhir.context.FhirContext;
|
||||
import ca.uhn.fhir.i18n.Msg;
|
||||
import ca.uhn.fhir.jpa.api.config.DaoConfig;
|
||||
import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
|
||||
import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
|
||||
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneClauseBuilder;
|
||||
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneIndexExtractor;
|
||||
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneResourceProjection;
|
||||
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneSearchBuilder;
|
||||
import ca.uhn.fhir.jpa.dao.search.LastNOperation;
|
||||
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
|
||||
|
@ -36,11 +37,9 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
|
|||
import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor;
|
||||
import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
|
||||
import ca.uhn.fhir.model.api.IQueryParameterType;
|
||||
import ca.uhn.fhir.parser.IParser;
|
||||
import ca.uhn.fhir.rest.api.Constants;
|
||||
import ca.uhn.fhir.rest.api.server.RequestDetails;
|
||||
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
|
||||
import ca.uhn.fhir.rest.param.StringParam;
|
||||
import ca.uhn.fhir.rest.param.TokenParam;
|
||||
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
|
||||
import ca.uhn.fhir.rest.server.util.ResourceSearchParams;
|
||||
import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension;
|
||||
|
@ -48,7 +47,6 @@ import org.hibernate.search.mapper.orm.Search;
|
|||
import org.hibernate.search.mapper.orm.session.SearchSession;
|
||||
import org.hibernate.search.mapper.orm.work.SearchIndexingPlan;
|
||||
import org.hibernate.search.util.common.SearchException;
|
||||
import org.hl7.fhir.instance.model.api.IAnyResource;
|
||||
import org.hl7.fhir.instance.model.api.IBaseResource;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
@ -59,6 +57,7 @@ import javax.annotation.Nonnull;
|
|||
import javax.persistence.EntityManager;
|
||||
import javax.persistence.PersistenceContext;
|
||||
import javax.persistence.PersistenceContextType;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
@ -66,8 +65,6 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
|
|||
|
||||
public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
|
||||
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FulltextSearchSvcImpl.class);
|
||||
@Autowired
|
||||
protected IForcedIdDao myForcedIdDao;
|
||||
@PersistenceContext(type = PersistenceContextType.TRANSACTION)
|
||||
private EntityManager myEntityManager;
|
||||
@Autowired
|
||||
|
@ -80,6 +77,9 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
|
|||
private DaoConfig myDaoConfig;
|
||||
@Autowired
|
||||
ISearchParamExtractor mySearchParamExtractor;
|
||||
@Autowired
|
||||
IIdHelperService myIdHelperService;
|
||||
|
||||
final private ExtendedLuceneSearchBuilder myAdvancedIndexQueryBuilder = new ExtendedLuceneSearchBuilder();
|
||||
|
||||
private Boolean ourDisabled;
|
||||
|
@ -140,7 +140,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
|
|||
/*
|
||||
* Handle _text parameter (resource narrative content)
|
||||
*
|
||||
* Positerity:
|
||||
* Posterity:
|
||||
* We do not want the HAPI-FHIR dao's to process the
|
||||
* _text parameter, so we remove it from the map here
|
||||
*/
|
||||
|
@ -158,7 +158,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
|
|||
/*
|
||||
* Handle other supported parameters
|
||||
*/
|
||||
if (myDaoConfig.isAdvancedLuceneIndexing()) {
|
||||
if (myDaoConfig.isAdvancedLuceneIndexing() && theParams.getEverythingMode() == null) {
|
||||
myAdvancedIndexQueryBuilder.addAndConsumeAdvancedQueryClauses(builder, theResourceType, theParams, mySearchParamRegistry);
|
||||
}
|
||||
//DROP EARLY HERE IF BOOL IS EMPTY?
|
||||
|
@ -181,26 +181,12 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, RequestDetails theRequest) {
|
||||
public List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
|
||||
|
||||
ResourcePersistentId pid = null;
|
||||
if (theParams.get(IAnyResource.SP_RES_ID) != null) {
|
||||
String idParamValue;
|
||||
IQueryParameterType idParam = theParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
|
||||
if (idParam instanceof TokenParam) {
|
||||
TokenParam idParm = (TokenParam) idParam;
|
||||
idParamValue = idParm.getValue();
|
||||
} else {
|
||||
StringParam idParm = (StringParam) idParam;
|
||||
idParamValue = idParm.getValue();
|
||||
}
|
||||
// pid = myIdHelperService.translateForcedIdToPid_(theResourceName, idParamValue, theRequest);
|
||||
}
|
||||
|
||||
ResourcePersistentId referencingPid = pid;
|
||||
List<ResourcePersistentId> retVal = doSearch(null, theParams, referencingPid);
|
||||
if (referencingPid != null) {
|
||||
retVal.add(referencingPid);
|
||||
List<ResourcePersistentId> retVal = doSearch(null, theParams, theReferencingPid);
|
||||
if (theReferencingPid != null) {
|
||||
retVal.add(theReferencingPid);
|
||||
}
|
||||
return retVal;
|
||||
}
|
||||
|
@ -271,4 +257,23 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
|
|||
return convertLongsToResourcePersistentIds(pidList);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<IBaseResource> getResources(Collection<Long> thePids) {
|
||||
SearchSession session = getSearchSession();
|
||||
List<ExtendedLuceneResourceProjection> rawResourceDataList = session.search(ResourceTable.class)
|
||||
.select(
|
||||
f -> f.composite(
|
||||
(pid, forcedId, resource)-> new ExtendedLuceneResourceProjection(pid, forcedId, resource),
|
||||
f.field("myId", Long.class),
|
||||
f.field("myForcedId", String.class),
|
||||
f.field("myRawResource", String.class))
|
||||
)
|
||||
.where(
|
||||
f -> f.id().matchingAny(thePids) // matches '_id' from resource index
|
||||
).fetchAllHits();
|
||||
IParser parser = myFhirContext.newJsonParser();
|
||||
return rawResourceDataList.stream()
|
||||
.map(p -> p.toResource(parser))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,18 +20,17 @@ package ca.uhn.fhir.jpa.dao;
|
|||
* #L%
|
||||
*/
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import ca.uhn.fhir.context.FhirContext;
|
||||
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
|
||||
import ca.uhn.fhir.jpa.model.search.ExtendedLuceneIndexData;
|
||||
import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions;
|
||||
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
|
||||
import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
|
||||
import ca.uhn.fhir.rest.api.server.RequestDetails;
|
||||
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
|
||||
import org.hl7.fhir.instance.model.api.IBaseResource;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
public interface IFulltextSearchSvc {
|
||||
|
||||
|
||||
|
@ -52,7 +51,7 @@ public interface IFulltextSearchSvc {
|
|||
*/
|
||||
IBaseResource tokenAutocompleteValueSetSearch(ValueSetAutocompleteOptions theOptions);
|
||||
|
||||
List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, RequestDetails theRequest);
|
||||
List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, ResourcePersistentId theReferencingPid);
|
||||
|
||||
boolean isDisabled();
|
||||
|
||||
|
@ -72,4 +71,12 @@ public interface IFulltextSearchSvc {
|
|||
|
||||
List<ResourcePersistentId> lastN(SearchParameterMap theParams, Integer theMaximumResults);
|
||||
|
||||
/**
|
||||
* Returns inlined resource stored along with index mappings for matched identifiers
|
||||
*
|
||||
* @param thePids raw pids - we dont support versioned references
|
||||
* @return Resources list or empty if nothing found
|
||||
*/
|
||||
List<IBaseResource> getResources(Collection<Long> thePids);
|
||||
|
||||
}
|
||||
|
|
|
@ -20,12 +20,12 @@ package ca.uhn.fhir.jpa.dao;
|
|||
* #L%
|
||||
*/
|
||||
|
||||
import ca.uhn.fhir.i18n.Msg;
|
||||
import ca.uhn.fhir.context.ComboSearchParamType;
|
||||
import ca.uhn.fhir.context.FhirContext;
|
||||
import ca.uhn.fhir.context.FhirVersionEnum;
|
||||
import ca.uhn.fhir.context.RuntimeResourceDefinition;
|
||||
import ca.uhn.fhir.context.RuntimeSearchParam;
|
||||
import ca.uhn.fhir.i18n.Msg;
|
||||
import ca.uhn.fhir.interceptor.api.HookParams;
|
||||
import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
|
||||
import ca.uhn.fhir.interceptor.api.Pointcut;
|
||||
|
@ -35,7 +35,6 @@ import ca.uhn.fhir.jpa.api.dao.IDao;
|
|||
import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
|
||||
import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
|
||||
import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
|
||||
import ca.uhn.fhir.jpa.dao.index.IdHelperService;
|
||||
import ca.uhn.fhir.jpa.dao.predicate.PredicateBuilder;
|
||||
import ca.uhn.fhir.jpa.dao.predicate.PredicateBuilderFactory;
|
||||
import ca.uhn.fhir.jpa.dao.predicate.SearchBuilderJoinEnum;
|
||||
|
@ -54,13 +53,10 @@ import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
|
|||
import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
|
||||
import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc;
|
||||
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
|
||||
import ca.uhn.fhir.rest.param.TokenParam;
|
||||
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
|
||||
import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper;
|
||||
import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
|
||||
import ca.uhn.fhir.jpa.util.BaseIterator;
|
||||
import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
|
||||
import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
|
||||
import ca.uhn.fhir.jpa.util.QueryChunker;
|
||||
import ca.uhn.fhir.jpa.util.ScrollableResultsIterator;
|
||||
import ca.uhn.fhir.jpa.util.SqlQueryList;
|
||||
|
@ -80,8 +76,11 @@ import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
|
|||
import ca.uhn.fhir.rest.param.DateRangeParam;
|
||||
import ca.uhn.fhir.rest.param.ReferenceParam;
|
||||
import ca.uhn.fhir.rest.param.StringParam;
|
||||
import ca.uhn.fhir.rest.param.TokenParam;
|
||||
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
|
||||
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
|
||||
import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
|
||||
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
|
||||
import ca.uhn.fhir.util.StopWatch;
|
||||
import ca.uhn.fhir.util.UrlUtil;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
@ -288,7 +287,7 @@ public class LegacySearchBuilder implements ISearchBuilder {
|
|||
}
|
||||
|
||||
if (myParams.getEverythingMode() != null) {
|
||||
pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest);
|
||||
pids = queryHibernateSearchForEverythingPids();
|
||||
} else {
|
||||
pids = myFulltextSearchSvc.search(myResourceName, myParams);
|
||||
}
|
||||
|
@ -333,6 +332,25 @@ public class LegacySearchBuilder implements ISearchBuilder {
|
|||
return myQueries;
|
||||
}
|
||||
|
||||
private List<ResourcePersistentId> queryHibernateSearchForEverythingPids() {
|
||||
ResourcePersistentId pid = null;
|
||||
if (myParams.get(IAnyResource.SP_RES_ID) != null) {
|
||||
String idParamValue;
|
||||
IQueryParameterType idParam = myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
|
||||
if (idParam instanceof TokenParam) {
|
||||
TokenParam idParm = (TokenParam) idParam;
|
||||
idParamValue = idParm.getValue();
|
||||
} else {
|
||||
StringParam idParm = (StringParam) idParam;
|
||||
idParamValue = idParm.getValue();
|
||||
}
|
||||
|
||||
pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue);
|
||||
}
|
||||
List<ResourcePersistentId> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid);
|
||||
return pids;
|
||||
}
|
||||
|
||||
private void doCreateChunkedQueries(List<Long> thePids, SortSpec sort, Integer theOffset, boolean theCount, RequestDetails theRequest, ArrayList<TypedQuery<Long>> theQueries) {
|
||||
if(thePids.size() < getMaximumPageSize()) {
|
||||
normalizeIdListForLastNInClause(thePids);
|
||||
|
|
|
@ -61,6 +61,8 @@ public class ExtendedLuceneIndexExtractor {
|
|||
public ExtendedLuceneIndexData extract(IBaseResource theResource, ResourceIndexedSearchParams theNewParams) {
|
||||
ExtendedLuceneIndexData retVal = new ExtendedLuceneIndexData(myContext);
|
||||
|
||||
retVal.setForcedId(theResource.getIdElement().getIdPart());
|
||||
|
||||
extractAutocompleteTokens(theResource, retVal);
|
||||
|
||||
theNewParams.myStringParams.forEach(nextParam ->
|
||||
|
|
|
@ -0,0 +1,34 @@
|
|||
package ca.uhn.fhir.jpa.dao.search;
|
||||
|
||||
import ca.uhn.fhir.model.primitive.IdDt;
|
||||
import ca.uhn.fhir.parser.IParser;
|
||||
import org.hl7.fhir.instance.model.api.IBaseResource;
|
||||
|
||||
/**
|
||||
* Query result when fetching full resources from Hibernate Search.
|
||||
*/
|
||||
public class ExtendedLuceneResourceProjection {
|
||||
final long myPid;
|
||||
final String myForcedId;
|
||||
final String myResourceString;
|
||||
|
||||
public ExtendedLuceneResourceProjection(long thePid, String theForcedId, String theResourceString) {
|
||||
myPid = thePid;
|
||||
myForcedId = theForcedId;
|
||||
myResourceString = theResourceString;
|
||||
}
|
||||
|
||||
public IBaseResource toResource(IParser theParser) {
|
||||
IBaseResource result = theParser.parseResource(myResourceString);
|
||||
|
||||
IdDt id;
|
||||
if (myForcedId != null) {
|
||||
id = new IdDt(myForcedId);
|
||||
} else {
|
||||
id = new IdDt(myPid);
|
||||
}
|
||||
result.setId(id);
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
|
@ -16,6 +16,12 @@
|
|||
* This pid list is used as a narrowing where clause against the remaining unprocessed search parameters in a jdbc query.
|
||||
* The actual queries for the different search types (e.g. token, string, modifiers, etc.) are
|
||||
* generated in {@link ca.uhn.fhir.jpa.dao.search.ExtendedLuceneSearchBuilder}.
|
||||
* <p>
|
||||
* Full resource bodies can be stored in the Hibernate Search index.
|
||||
* The {@link ca.uhn.fhir.jpa.dao.search.ExtendedLuceneResourceProjection} is used to extract these.
|
||||
* This is currently restricted to LastN, and misses tag changes from $meta-add and $meta-delete since those don't
|
||||
* update Hibernate Search.
|
||||
* </p>
|
||||
*
|
||||
* <h2>Operation</h2>
|
||||
* During startup, Hibernate Search uses {@link ca.uhn.fhir.jpa.model.search.SearchParamTextPropertyBinder} to generate a schema.
|
||||
|
|
|
@ -24,6 +24,7 @@ import ca.uhn.fhir.context.FhirContext;
|
|||
import ca.uhn.fhir.rest.param.TokenParam;
|
||||
import org.hibernate.search.mapper.orm.session.SearchSession;
|
||||
import org.hl7.fhir.instance.model.api.IBaseResource;
|
||||
import org.hl7.fhir.r4.model.Enumerations;
|
||||
import org.hl7.fhir.r4.model.ValueSet;
|
||||
|
||||
import java.util.List;
|
||||
|
@ -47,6 +48,7 @@ public class ValueSetAutocompleteSearch {
|
|||
ValueSet result = new ValueSet();
|
||||
ValueSet.ValueSetExpansionComponent expansion = new ValueSet.ValueSetExpansionComponent();
|
||||
result.setExpansion(expansion);
|
||||
result.setStatus(Enumerations.PublicationStatus.ACTIVE);
|
||||
aggEntries.stream()
|
||||
.map(this::makeCoding)
|
||||
.forEach(expansion::addContains);
|
||||
|
|
|
@ -0,0 +1,12 @@
|
|||
package ca.uhn.fhir.jpa.search.builder;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.util.Iterator;
|
||||
|
||||
public interface ISearchQueryExecutor extends Iterator<Long>, Closeable {
|
||||
/**
|
||||
* Narrow the signature - no IOException allowed.
|
||||
*/
|
||||
@Override
|
||||
void close();
|
||||
}
|
|
@ -43,7 +43,6 @@ import ca.uhn.fhir.jpa.dao.IResultIterator;
|
|||
import ca.uhn.fhir.jpa.dao.ISearchBuilder;
|
||||
import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
|
||||
import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
|
||||
import ca.uhn.fhir.jpa.dao.index.IdHelperService;
|
||||
import ca.uhn.fhir.jpa.entity.ResourceSearchView;
|
||||
import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
|
||||
import ca.uhn.fhir.jpa.model.config.PartitionSettings;
|
||||
|
@ -111,7 +110,6 @@ import javax.persistence.EntityManager;
|
|||
import javax.persistence.PersistenceContext;
|
||||
import javax.persistence.PersistenceContextType;
|
||||
import javax.persistence.Query;
|
||||
import javax.persistence.SqlResultSetMapping;
|
||||
import javax.persistence.Tuple;
|
||||
import javax.persistence.TypedQuery;
|
||||
import javax.persistence.criteria.CriteriaBuilder;
|
||||
|
@ -278,11 +276,11 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
|
||||
init(theParams, theSearchUuid, theRequestPartitionId);
|
||||
|
||||
ArrayList<SearchQueryExecutor> queries = createQuery(myParams, null, null, null, true, theRequest, null);
|
||||
List<ISearchQueryExecutor> queries = createQuery(myParams, null, null, null, true, theRequest, null);
|
||||
if (queries.isEmpty()) {
|
||||
return Collections.emptyIterator();
|
||||
}
|
||||
try (SearchQueryExecutor queryExecutor = queries.get(0)) {
|
||||
try (ISearchQueryExecutor queryExecutor = queries.get(0)) {
|
||||
return Lists.newArrayList(queryExecutor.next()).iterator();
|
||||
}
|
||||
}
|
||||
|
@ -317,20 +315,24 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
myRequestPartitionId = theRequestPartitionId;
|
||||
}
|
||||
|
||||
private ArrayList<SearchQueryExecutor> createQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCount, RequestDetails theRequest,
|
||||
private List<ISearchQueryExecutor> createQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCount, RequestDetails theRequest,
|
||||
SearchRuntimeDetails theSearchRuntimeDetails) {
|
||||
|
||||
List<ResourcePersistentId> pids = new ArrayList<>();
|
||||
ArrayList<ISearchQueryExecutor> queries = new ArrayList<>();
|
||||
|
||||
if (checkUseHibernateSearch()) {
|
||||
// we're going to run at least part of the search against the Fulltext service.
|
||||
List<ResourcePersistentId> fulltextMatchIds;
|
||||
if (myParams.isLastN()) {
|
||||
pids = executeLastNAgainstIndex(theMaximumResults);
|
||||
fulltextMatchIds = executeLastNAgainstIndex(theMaximumResults);
|
||||
} else if (myParams.getEverythingMode() != null) {
|
||||
fulltextMatchIds = queryHibernateSearchForEverythingPids();
|
||||
} else {
|
||||
pids = queryLuceneForPIDs(theRequest);
|
||||
fulltextMatchIds = myFulltextSearchSvc.search(myResourceName, myParams);
|
||||
}
|
||||
|
||||
if (theSearchRuntimeDetails != null) {
|
||||
theSearchRuntimeDetails.setFoundIndexMatchesCount(pids.size());
|
||||
theSearchRuntimeDetails.setFoundIndexMatchesCount(fulltextMatchIds.size());
|
||||
HookParams params = new HookParams()
|
||||
.add(RequestDetails.class, theRequest)
|
||||
.addIfMatchesType(ServletRequestDetails.class, theRequest)
|
||||
|
@ -338,20 +340,39 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params);
|
||||
}
|
||||
|
||||
if (pids.isEmpty()) {
|
||||
// Will never match
|
||||
pids = Collections.singletonList(new ResourcePersistentId(-1L));
|
||||
}
|
||||
List<Long> rawPids = ResourcePersistentId.toLongList(fulltextMatchIds);
|
||||
|
||||
}
|
||||
// can we skip the database entirely and return the pid list from here?
|
||||
boolean canSkipDatabase =
|
||||
// if we processed an AND clause, and it returned nothing, then nothing can match.
|
||||
rawPids.isEmpty() ||
|
||||
// Our hibernate search query doesn't respect partitions yet
|
||||
(!myPartitionSettings.isPartitioningEnabled() &&
|
||||
// were there AND terms left? Then we still need the db.
|
||||
theParams.isEmpty() &&
|
||||
// not every param is a param. :-(
|
||||
theParams.getNearDistanceParam() == null &&
|
||||
theParams.getLastUpdated() == null &&
|
||||
theParams.getEverythingMode() == null &&
|
||||
theParams.getOffset() == null &&
|
||||
// or sorting?
|
||||
theParams.getSort() == null
|
||||
// todo MB Ugh - review with someone else
|
||||
//theParams.toNormalizedQueryString(myContext).length() <= 1 &&
|
||||
);
|
||||
|
||||
ArrayList<SearchQueryExecutor> queries = new ArrayList<>();
|
||||
|
||||
if (!pids.isEmpty()) {
|
||||
new QueryChunker<Long>().chunk(ResourcePersistentId.toLongList(pids), t -> doCreateChunkedQueries(theParams, t, theOffset, sort, theCount, theRequest, queries));
|
||||
if (canSkipDatabase) {
|
||||
queries.add(ResolvedSearchQueryExecutor.from(rawPids));
|
||||
} else {
|
||||
// Finish the query in the database for the rest of the search parameters, sorting, partitioning, etc.
|
||||
// We break the pids into chunks that fit in the 1k limit for jdbc bind params.
|
||||
new QueryChunker<Long>()
|
||||
.chunk(rawPids, t -> doCreateChunkedQueries(theParams, t, theOffset, sort, theCount, theRequest, queries));
|
||||
}
|
||||
} else {
|
||||
// do everything in the database.
|
||||
Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, theMaximumResults, theCount, theRequest, null);
|
||||
query.ifPresent(t -> queries.add(t));
|
||||
query.ifPresent(queries::add);
|
||||
}
|
||||
|
||||
return queries;
|
||||
|
@ -371,8 +392,10 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
failIfUsed(Constants.PARAM_CONTENT);
|
||||
}
|
||||
|
||||
// TODO MB someday we'll want a query planner to figure out if we _should_ use the ft index, not just if we can.
|
||||
return fulltextEnabled && myFulltextSearchSvc.supportsSomeOf(myParams);
|
||||
// TODO MB someday we'll want a query planner to figure out if we _should_ or _must_ use the ft index, not just if we can.
|
||||
return fulltextEnabled &&
|
||||
myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE &&
|
||||
myFulltextSearchSvc.supportsSomeOf(myParams);
|
||||
}
|
||||
|
||||
private void failIfUsed(String theParamName) {
|
||||
|
@ -401,19 +424,26 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
}
|
||||
}
|
||||
|
||||
private List<ResourcePersistentId> queryLuceneForPIDs(RequestDetails theRequest) {
|
||||
validateFullTextSearchIsEnabled();
|
||||
|
||||
List<ResourcePersistentId> pids;
|
||||
if (myParams.getEverythingMode() != null) {
|
||||
pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest);
|
||||
private List<ResourcePersistentId> queryHibernateSearchForEverythingPids() {
|
||||
ResourcePersistentId pid = null;
|
||||
if (myParams.get(IAnyResource.SP_RES_ID) != null) {
|
||||
String idParamValue;
|
||||
IQueryParameterType idParam = myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
|
||||
if (idParam instanceof TokenParam) {
|
||||
TokenParam idParm = (TokenParam) idParam;
|
||||
idParamValue = idParm.getValue();
|
||||
} else {
|
||||
pids = myFulltextSearchSvc.search(myResourceName, myParams);
|
||||
StringParam idParm = (StringParam) idParam;
|
||||
idParamValue = idParm.getValue();
|
||||
}
|
||||
|
||||
pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue);
|
||||
}
|
||||
List<ResourcePersistentId> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid);
|
||||
return pids;
|
||||
}
|
||||
|
||||
private void doCreateChunkedQueries(SearchParameterMap theParams, List<Long> thePids, Integer theOffset, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<SearchQueryExecutor> theQueries) {
|
||||
private void doCreateChunkedQueries(SearchParameterMap theParams, List<Long> thePids, Integer theOffset, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<ISearchQueryExecutor> theQueries) {
|
||||
if (thePids.size() < getMaximumPageSize()) {
|
||||
normalizeIdListForLastNInClause(thePids);
|
||||
}
|
||||
|
@ -820,14 +850,19 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
idList.add(resource.getId());
|
||||
}
|
||||
|
||||
return getPidToTagMap(idList);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Map<Long, Collection<ResourceTag>> getPidToTagMap(List<Long> thePidList) {
|
||||
Map<Long, Collection<ResourceTag>> tagMap = new HashMap<>();
|
||||
|
||||
//-- no tags
|
||||
if (idList.size() == 0)
|
||||
if (thePidList.size() == 0)
|
||||
return tagMap;
|
||||
|
||||
//-- get all tags for the idList
|
||||
Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(idList);
|
||||
Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(thePidList);
|
||||
|
||||
//-- build the map, key = resourceId, value = list of ResourceTag
|
||||
ResourcePersistentId resourceId;
|
||||
|
@ -865,23 +900,55 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
theResourceListToPopulate.add(null);
|
||||
}
|
||||
|
||||
List<ResourcePersistentId> pids = new ArrayList<>(thePids);
|
||||
// Can we fast track this loading by checking elastic search?
|
||||
if (isLoadingFromElasticSearchSupported(theIncludedPids.isEmpty())) {
|
||||
theResourceListToPopulate.addAll(loadObservationResourcesFromElasticSearch(thePids));
|
||||
if (isLoadingFromElasticSearchSupported(theIncludedPids)) {
|
||||
theResourceListToPopulate.addAll(loadResourcesFromElasticSearch(thePids));
|
||||
} else {
|
||||
// We only chunk because some jdbc drivers can't handle long param lists.
|
||||
new QueryChunker<ResourcePersistentId>().chunk(pids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position));
|
||||
new QueryChunker<ResourcePersistentId>().chunk(thePids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position));
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isLoadingFromElasticSearchSupported(boolean noIncludePids) {
|
||||
return noIncludePids && !Objects.isNull(myParams) && myParams.isLastN() && myDaoConfig.isStoreResourceInLuceneIndex()
|
||||
&& myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3);
|
||||
/**
|
||||
* Check if we can load the resources from Hibernate Search instead of the database.
|
||||
* We assume this is faster.
|
||||
*
|
||||
* Hibernate Search only stores the current version, and only if enabled.
|
||||
* @param theIncludedPids the _include target to check for versioned ids
|
||||
* @return can we fetch from Hibernate Search?
|
||||
*/
|
||||
private boolean isLoadingFromElasticSearchSupported(Collection<ResourcePersistentId> theIncludedPids) {
|
||||
// todo mb we can be smarter here.
|
||||
// todo check if theIncludedPids has any with version not null.
|
||||
|
||||
// is storage enabled?
|
||||
return myDaoConfig.isStoreResourceInLuceneIndex() &&
|
||||
// only support lastN for now.
|
||||
myParams.isLastN() &&
|
||||
// do we need to worry about versions?
|
||||
theIncludedPids.isEmpty() &&
|
||||
// skip the complexity for metadata in dstu2
|
||||
myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3);
|
||||
}
|
||||
|
||||
private List<IBaseResource> loadObservationResourcesFromElasticSearch(Collection<ResourcePersistentId> thePids) {
|
||||
private List<IBaseResource> loadResourcesFromElasticSearch(Collection<ResourcePersistentId> thePids) {
|
||||
// Do we use the fulltextsvc via hibernate-search to load resources or be backwards compatible with older ES only impl
|
||||
// to handle lastN?
|
||||
if (myDaoConfig.isAdvancedLuceneIndexing() && myDaoConfig.isStoreResourceInLuceneIndex()) {
|
||||
List<Long> pidList = thePids.stream().map(ResourcePersistentId::getIdAsLong).collect(Collectors.toList());
|
||||
|
||||
// todo need to inject metadata - use profile to build resource, tags, and security labels
|
||||
//Map<Long, Collection<ResourceTag>> pidToTagMap = getPidToTagMap(pidList);
|
||||
List<IBaseResource> resources = myFulltextSearchSvc.getResources(pidList);
|
||||
return resources;
|
||||
} else if (!Objects.isNull(myParams) && myParams.isLastN()) {
|
||||
// legacy LastN implementation
|
||||
return myIElasticsearchSvc.getObservationResources(thePids);
|
||||
} else {
|
||||
// TODO I wonder if we should drop this path, and only support the new Hibernate Search path.
|
||||
Validate.isTrue(false, "Unexpected");
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1323,6 +1390,41 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
myDaoConfig = theDaoConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapt simple Iterator to our internal query interface.
|
||||
*/
|
||||
static class ResolvedSearchQueryExecutor implements ISearchQueryExecutor {
|
||||
private final Iterator<Long> myIterator;
|
||||
|
||||
ResolvedSearchQueryExecutor(Iterable<Long> theIterable) {
|
||||
this(theIterable.iterator());
|
||||
}
|
||||
|
||||
ResolvedSearchQueryExecutor(Iterator<Long> theIterator) {
|
||||
myIterator = theIterator;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
public static ResolvedSearchQueryExecutor from(List<Long> rawPids) {
|
||||
return new ResolvedSearchQueryExecutor(rawPids);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return myIterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long next() {
|
||||
return myIterator.next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
// empty
|
||||
}
|
||||
}
|
||||
|
||||
public class IncludesIterator extends BaseIterator<ResourcePersistentId> implements Iterator<ResourcePersistentId> {
|
||||
|
||||
private final RequestDetails myRequest;
|
||||
|
@ -1381,11 +1483,11 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
private boolean myFirst = true;
|
||||
private IncludesIterator myIncludesIterator;
|
||||
private ResourcePersistentId myNext;
|
||||
private SearchQueryExecutor myResultsIterator;
|
||||
private ISearchQueryExecutor myResultsIterator;
|
||||
private boolean myStillNeedToFetchIncludes;
|
||||
private int mySkipCount = 0;
|
||||
private int myNonSkipCount = 0;
|
||||
private ArrayList<SearchQueryExecutor> myQueryList = new ArrayList<>();
|
||||
private List<ISearchQueryExecutor> myQueryList = new ArrayList<>();
|
||||
|
||||
private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
|
||||
mySearchRuntimeDetails = theSearchRuntimeDetails;
|
||||
|
@ -1668,15 +1770,4 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
return thePredicates.toArray(new Predicate[0]);
|
||||
}
|
||||
|
||||
private void validateFullTextSearchIsEnabled() {
|
||||
if (myFulltextSearchSvc == null) {
|
||||
if (myParams.containsKey(Constants.PARAM_TEXT)) {
|
||||
throw new InvalidRequestException(Msg.code(1200) + "Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT);
|
||||
} else if (myParams.containsKey(Constants.PARAM_CONTENT)) {
|
||||
throw new InvalidRequestException(Msg.code(1201) + "Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT);
|
||||
} else {
|
||||
throw new InvalidRequestException(Msg.code(1202) + "Fulltext search is not enabled on this service, can not process qualifier :text");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@ package ca.uhn.fhir.jpa.search.builder.sql;
|
|||
*/
|
||||
|
||||
import ca.uhn.fhir.i18n.Msg;
|
||||
import ca.uhn.fhir.jpa.search.builder.ISearchQueryExecutor;
|
||||
import ca.uhn.fhir.jpa.util.ScrollableResultsIterator;
|
||||
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
|
||||
import ca.uhn.fhir.util.IoUtil;
|
||||
|
@ -35,12 +36,10 @@ import javax.persistence.EntityManager;
|
|||
import javax.persistence.PersistenceContext;
|
||||
import javax.persistence.PersistenceContextType;
|
||||
import javax.persistence.Query;
|
||||
import java.io.Closeable;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.util.Iterator;
|
||||
|
||||
public class SearchQueryExecutor implements Iterator<Long>, Closeable {
|
||||
public class SearchQueryExecutor implements ISearchQueryExecutor {
|
||||
|
||||
private static final Long NO_MORE = -1L;
|
||||
private static final SearchQueryExecutor NO_VALUE_EXECUTOR = new SearchQueryExecutor();
|
||||
|
|
|
@ -10,6 +10,7 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
|
|||
import ca.uhn.fhir.rest.api.SortSpec;
|
||||
import ca.uhn.fhir.rest.api.server.IBundleProvider;
|
||||
import ca.uhn.fhir.rest.server.method.SortParameter;
|
||||
import org.hl7.fhir.instance.model.api.IBaseResource;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
@ -45,6 +46,11 @@ public class TestDaoSearch {
|
|||
myFhirCtx = theFhirCtx;
|
||||
}
|
||||
|
||||
public List<IBaseResource> searchForResources(String theQueryUrl) {
|
||||
IBundleProvider result = searchForBundleProvider(theQueryUrl);
|
||||
return result.getAllResources();
|
||||
}
|
||||
|
||||
public List<String> searchForIds(String theQueryUrl) {
|
||||
// fake out the server url parsing
|
||||
IBundleProvider result = searchForBundleProvider(theQueryUrl);
|
||||
|
@ -55,19 +61,31 @@ public class TestDaoSearch {
|
|||
|
||||
public IBundleProvider searchForBundleProvider(String theQueryUrl) {
|
||||
ResourceSearch search = myMatchUrlService.getResourceSearch(theQueryUrl);
|
||||
IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(search.getResourceName());
|
||||
|
||||
SearchParameterMap map = search.getSearchParameterMap();
|
||||
map.setLoadSynchronous(true);
|
||||
SystemRequestDetails request = fakeRequestDetailsFromUrl(theQueryUrl);
|
||||
SortSpec sort = (SortSpec) new SortParameter(myFhirCtx).translateQueryParametersIntoServerArgument(request, null);
|
||||
SortSpec sort = (SortSpec) new SortParameter(myFhirCtx).translateQueryParametersIntoServerArgument(fakeRequestDetailsFromUrl(theQueryUrl), null);
|
||||
if (sort != null) {
|
||||
map.setSort(sort);
|
||||
}
|
||||
|
||||
IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(search.getResourceName());
|
||||
IBundleProvider result = dao.search(map, request);
|
||||
IBundleProvider result = dao.search(map, fakeRequestDetailsFromUrl(theQueryUrl));
|
||||
return result;
|
||||
}
|
||||
|
||||
public SearchParameterMap toSearchParameters(String theQueryUrl) {
|
||||
ResourceSearch search = myMatchUrlService.getResourceSearch(theQueryUrl);
|
||||
|
||||
SearchParameterMap map = search.getSearchParameterMap();
|
||||
map.setLoadSynchronous(true);
|
||||
SortSpec sort = (SortSpec) new SortParameter(myFhirCtx).translateQueryParametersIntoServerArgument(fakeRequestDetailsFromUrl(theQueryUrl), null);
|
||||
if (sort != null) {
|
||||
map.setSort(sort);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private SystemRequestDetails fakeRequestDetailsFromUrl(String theQueryUrl) {
|
||||
SystemRequestDetails request = new SystemRequestDetails();
|
||||
|
|
|
@ -24,7 +24,6 @@ import java.util.UUID;
|
|||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.contains;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
import static org.hamcrest.Matchers.matchesPattern;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
@ -34,7 +33,6 @@ import static org.mockito.Mockito.when;
|
|||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN {
|
||||
|
||||
@AfterEach
|
||||
public void reset() {
|
||||
SearchBuilder.setMaxPageSize50ForTest(false);
|
||||
|
@ -127,16 +125,22 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseR4SearchLastN {
|
|||
when(mySrd.getParameters()).thenReturn(requestParameters);
|
||||
|
||||
List<String> results = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(), null));
|
||||
verifyResourcesLoadedFromElastic(observationIds, results);
|
||||
|
||||
}
|
||||
|
||||
void verifyResourcesLoadedFromElastic(List<IIdType> theObservationIds, List<String> theResults) {
|
||||
List<ResourcePersistentId> expectedArgumentPids = ResourcePersistentId.fromLongList(
|
||||
observationIds.stream().map(IIdType::getIdPartAsLong).collect(Collectors.toList())
|
||||
theObservationIds.stream().map(IIdType::getIdPartAsLong).collect(Collectors.toList())
|
||||
);
|
||||
ArgumentCaptor<List<ResourcePersistentId>> actualPids = ArgumentCaptor.forClass(List.class);
|
||||
verify(myElasticsearchSvc, times(1)).getObservationResources(actualPids.capture());
|
||||
assertThat(actualPids.getValue(), is(expectedArgumentPids));
|
||||
|
||||
List<String> expectedObservationList = observationIds.stream()
|
||||
List<String> expectedObservationList = theObservationIds.stream()
|
||||
.map(id -> id.toUnqualifiedVersionless().getValue()).collect(Collectors.toList());
|
||||
assertEquals(results, expectedObservationList);
|
||||
assertEquals(expectedObservationList, theResults);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1,17 +1,37 @@
|
|||
package ca.uhn.fhir.jpa.dao.r4;
|
||||
|
||||
import ca.uhn.fhir.jpa.api.config.DaoConfig;
|
||||
import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
|
||||
import org.hl7.fhir.instance.model.api.IIdType;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.mock.mockito.SpyBean;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
/**
|
||||
* Run entire @see {@link FhirResourceDaoR4SearchLastNIT} test suite this time
|
||||
* using Extended Lucene index as search target
|
||||
* using Extended Lucene index as search target.
|
||||
*
|
||||
* The other implementation is obsolete, and we can merge these someday.
|
||||
*/
|
||||
@ExtendWith(SpringExtension.class)
|
||||
public class FhirResourceDaoR4SearchLastNUsingExtendedLuceneIndexIT extends FhirResourceDaoR4SearchLastNIT {
|
||||
// awkward override so we can spy
|
||||
@SpyBean
|
||||
@Autowired(required = false)
|
||||
IFulltextSearchSvc myFulltestSearchSvc;
|
||||
|
||||
@BeforeEach
|
||||
public void enableAdvancedLuceneIndexing() {
|
||||
|
@ -23,4 +43,25 @@ public class FhirResourceDaoR4SearchLastNUsingExtendedLuceneIndexIT extends Fhir
|
|||
myDaoConfig.setAdvancedLuceneIndexing(new DaoConfig().isAdvancedLuceneIndexing());
|
||||
}
|
||||
|
||||
/**
|
||||
* We pull the resources from Hibernate Search when LastN uses Hibernate Search.
|
||||
* Override the test verification
|
||||
*/
|
||||
@Override
|
||||
void verifyResourcesLoadedFromElastic(List<IIdType> theObservationIds, List<String> theResults) {
|
||||
List<Long> expectedArgumentPids =
|
||||
theObservationIds.stream().map(IIdType::getIdPartAsLong).collect(Collectors.toList());
|
||||
|
||||
ArgumentCaptor<List<Long>> actualPids = ArgumentCaptor.forClass(List.class);
|
||||
|
||||
verify(myFulltestSearchSvc, times(1)).getResources(actualPids.capture());
|
||||
assertThat(actualPids.getValue(), is(expectedArgumentPids));
|
||||
|
||||
// we don't include the type in the id returned from Hibernate Search for now.
|
||||
List<String> expectedObservationList = theObservationIds.stream()
|
||||
.map(id -> id.toUnqualifiedVersionless().getIdPart()).collect(Collectors.toList());
|
||||
assertEquals(expectedObservationList, theResults);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -10,12 +10,12 @@ import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
|
|||
import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
|
||||
import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
|
||||
import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportJobSchedulingHelper;
|
||||
import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportSvc;
|
||||
import ca.uhn.fhir.jpa.config.TestHibernateSearchAddInConfig;
|
||||
import ca.uhn.fhir.jpa.config.TestR4Config;
|
||||
import ca.uhn.fhir.jpa.dao.BaseDateSearchDaoTests;
|
||||
import ca.uhn.fhir.jpa.dao.BaseJpaTest;
|
||||
import ca.uhn.fhir.jpa.dao.DaoTestDataBuilder;
|
||||
import ca.uhn.fhir.jpa.dao.TestDaoSearch;
|
||||
import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
|
||||
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
|
||||
import ca.uhn.fhir.jpa.entity.TermConcept;
|
||||
|
@ -37,10 +37,14 @@ import ca.uhn.fhir.rest.param.StringParam;
|
|||
import ca.uhn.fhir.rest.param.TokenParam;
|
||||
import ca.uhn.fhir.rest.param.TokenParamModifier;
|
||||
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
|
||||
import ca.uhn.fhir.test.utilities.ITestDataBuilder;
|
||||
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
|
||||
import ca.uhn.fhir.validation.FhirValidator;
|
||||
import ca.uhn.fhir.validation.ValidationResult;
|
||||
import org.hamcrest.Matchers;
|
||||
import org.hl7.fhir.instance.model.api.IBaseCoding;
|
||||
import org.hl7.fhir.instance.model.api.IBaseMetaType;
|
||||
import org.hl7.fhir.instance.model.api.IBaseResource;
|
||||
import org.hl7.fhir.instance.model.api.IIdType;
|
||||
import org.hl7.fhir.r4.model.Bundle;
|
||||
import org.hl7.fhir.r4.model.CodeSystem;
|
||||
|
@ -75,14 +79,22 @@ import java.util.List;
|
|||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.contains;
|
||||
import static org.hamcrest.Matchers.containsInAnyOrder;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.hasSize;
|
||||
import static org.hamcrest.Matchers.not;
|
||||
import static org.hamcrest.Matchers.stringContainsInOrder;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
@RequiresDocker
|
||||
@ContextConfiguration(classes = {TestR4Config.class, TestHibernateSearchAddInConfig.Elasticsearch.class})
|
||||
@ContextConfiguration(classes = {
|
||||
TestR4Config.class,
|
||||
TestHibernateSearchAddInConfig.Elasticsearch.class,
|
||||
DaoTestDataBuilder.Config.class,
|
||||
TestDaoSearch.Config.class
|
||||
})
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
|
||||
public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
|
||||
public static final String URL_MY_CODE_SYSTEM = "http://example.com/my_code_system";
|
||||
|
@ -132,6 +144,11 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
|
|||
private ITermCodeSystemStorageSvc myTermCodeSystemStorageSvc;
|
||||
@Autowired
|
||||
private DaoRegistry myDaoRegistry;
|
||||
@Autowired
|
||||
ITestDataBuilder myTestDataBuilder;
|
||||
@Autowired
|
||||
TestDaoSearch myTestDaoSearch;
|
||||
|
||||
|
||||
@BeforeEach
|
||||
public void beforePurgeDatabase() {
|
||||
|
@ -159,6 +176,7 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
|
|||
DaoConfig defaultConfig = new DaoConfig();
|
||||
myDaoConfig.setAllowContainsSearches(defaultConfig.isAllowContainsSearches());
|
||||
myDaoConfig.setAdvancedLuceneIndexing(defaultConfig.isAdvancedLuceneIndexing());
|
||||
myDaoConfig.setStoreResourceInLuceneIndex(defaultConfig.isStoreResourceInLuceneIndex());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -422,6 +440,10 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
|
|||
map.add("code", new TokenParam("Bodum").setModifier(TokenParamModifier.TEXT));
|
||||
assertObservationSearchMatchesNothing("search with shared prefix does not match", map);
|
||||
}
|
||||
|
||||
{
|
||||
assertObservationSearchMatches("empty params finds everything", "Observation?", id1, id2, id3, id4);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -515,6 +537,11 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
|
|||
assertThat(message, toUnqualifiedVersionlessIdValues(myObservationDao.search(map)), containsInAnyOrder(toValues(iIdTypes)));
|
||||
}
|
||||
|
||||
private void assertObservationSearchMatches(String theMessage, String theSearch, IIdType... theIds) {
|
||||
SearchParameterMap map = myTestDaoSearch.toSearchParameters(theSearch);
|
||||
assertObservationSearchMatches(theMessage, map, theIds);
|
||||
}
|
||||
|
||||
@Nested
|
||||
public class WithContainedIndexingIT {
|
||||
@BeforeEach
|
||||
|
@ -762,4 +789,121 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Some queries can be satisfied directly from Hibernate Search.
|
||||
* We still need at least one query to fetch the resources.
|
||||
*/
|
||||
@Nested
|
||||
public class FastPath {
|
||||
@BeforeEach
|
||||
public void enableResourceStorage() {
|
||||
myDaoConfig.setStoreResourceInLuceneIndex(false);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void resetResourceStorage() {
|
||||
myDaoConfig.setStoreResourceInLuceneIndex(new DaoConfig().isStoreResourceInLuceneIndex());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void simpleTokenSkipsSql() {
|
||||
|
||||
IIdType id = myTestDataBuilder.createObservation(myTestDataBuilder.withObservationCode("http://example.com/", "theCode"));
|
||||
myCaptureQueriesListener.clear();
|
||||
|
||||
List<String> ids = myTestDaoSearch.searchForIds("Observation?code=theCode");
|
||||
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
|
||||
|
||||
assertThat(ids, hasSize(1));
|
||||
assertThat(ids, contains(id.getIdPart()));
|
||||
assertEquals(1, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "sql just to fetch resources");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void sortStillRequiresSql() {
|
||||
|
||||
IIdType id = myTestDataBuilder.createObservation(myTestDataBuilder.withObservationCode("http://example.com/", "theCode"));
|
||||
myCaptureQueriesListener.clear();
|
||||
|
||||
List<String> ids = myTestDaoSearch.searchForIds("Observation?code=theCode&_sort=code");
|
||||
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
|
||||
|
||||
assertThat(ids, hasSize(1));
|
||||
assertThat(ids, contains(id.getIdPart()));
|
||||
|
||||
assertEquals(2, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "the pids come from elastic, but we use sql to sort, and fetch resources");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deletedResourceNotFound() {
|
||||
|
||||
IIdType id = myTestDataBuilder.createObservation(myTestDataBuilder.withObservationCode("http://example.com/", "theCode"));
|
||||
myObservationDao.delete(id);
|
||||
myCaptureQueriesListener.clear();
|
||||
|
||||
List<String> ids = myTestDaoSearch.searchForIds("Observation?code=theCode&_sort=code");
|
||||
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
|
||||
|
||||
assertThat(ids, hasSize(0));
|
||||
|
||||
assertEquals(0, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "the pids come from elastic, and nothing to fetch");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void forcedIdSurvivesWithNoSql() {
|
||||
IIdType id = myTestDataBuilder.createObservation(
|
||||
myTestDataBuilder.withObservationCode("http://example.com/", "theCode"),
|
||||
myTestDataBuilder.withId("forcedid"));
|
||||
assertThat(id.getIdPart(), equalTo("forcedid"));
|
||||
myCaptureQueriesListener.clear();
|
||||
|
||||
List<String> ids = myTestDaoSearch.searchForIds("Observation?code=theCode");
|
||||
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
|
||||
|
||||
assertThat(ids, hasSize(1));
|
||||
assertThat(ids, contains(id.getIdPart()));
|
||||
|
||||
assertEquals(1, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "just 1 to fetch the resources");
|
||||
}
|
||||
|
||||
/**
|
||||
* A paranoid test to make sure tags stay with the resource.
|
||||
*
|
||||
* Tags live outside the resource, and can be modified by
|
||||
* Since we lost the id, also check tags in case someone changes metadata processing during ingestion.
|
||||
*/
|
||||
@Test
|
||||
public void tagsSurvive() {
|
||||
IIdType id = myTestDataBuilder.createObservation(
|
||||
myTestDataBuilder.withObservationCode("http://example.com/", "theCode"),
|
||||
myTestDataBuilder.withTag("http://example.com", "aTag"));
|
||||
|
||||
myCaptureQueriesListener.clear();
|
||||
List<IBaseResource> observations = myTestDaoSearch.searchForResources("Observation?code=theCode");
|
||||
|
||||
assertThat(observations, hasSize(1));
|
||||
List<? extends IBaseCoding> tags = observations.get(0).getMeta().getTag();
|
||||
assertThat(tags, hasSize(1));
|
||||
assertThat(tags.get(0).getSystem(), equalTo("http://example.com"));
|
||||
assertThat(tags.get(0).getCode(), equalTo("aTag"));
|
||||
|
||||
Meta meta = new Meta();
|
||||
meta.addTag().setSystem("tag_scheme1").setCode("tag_code1");
|
||||
meta.addProfile("http://profile/1");
|
||||
meta.addSecurity().setSystem("seclabel_sys1").setCode("seclabel_code1");
|
||||
myObservationDao.metaAddOperation(id, meta, mySrd);
|
||||
|
||||
observations = myTestDaoSearch.searchForResources("Observation?code=theCode");
|
||||
|
||||
assertThat(observations, hasSize(1));
|
||||
IBaseMetaType newMeta = observations.get(0).getMeta();
|
||||
assertThat(newMeta.getProfile(), hasSize(1));
|
||||
assertThat(newMeta.getSecurity(), hasSize(1));
|
||||
assertThat(newMeta.getTag(), hasSize(2));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -190,7 +190,7 @@ public class FhirResourceDaoR4StandardQueriesNoFTTest extends BaseJpaTest {
|
|||
withRiskAssessmentWithProbabilty(0.6);
|
||||
|
||||
assertNotFind("when gt", "/RiskAssessment?probability=0.5");
|
||||
// fixme we break the spec here.
|
||||
// TODO we break the spec here. Default search should be approx
|
||||
// assertFind("when a little gt - default is approx", "/RiskAssessment?probability=0.599");
|
||||
// assertFind("when a little lt - default is approx", "/RiskAssessment?probability=0.601");
|
||||
assertFind("when eq", "/RiskAssessment?probability=0.6");
|
||||
|
@ -301,10 +301,10 @@ public class FhirResourceDaoR4StandardQueriesNoFTTest extends BaseJpaTest {
|
|||
|
||||
assertNotFind("when gt", "/Observation?value-quantity=0.5||mmHg");
|
||||
assertNotFind("when gt unitless", "/Observation?value-quantity=0.5");
|
||||
// fixme we break the spec here.
|
||||
// TODO we break the spec here. Default search should be approx
|
||||
// assertFind("when a little gt - default is approx", "/Observation?value-quantity=0.599");
|
||||
// assertFind("when a little lt - default is approx", "/Observation?value-quantity=0.601");
|
||||
// fixme we don't seem to support "units", only "code".
|
||||
// TODO we don't seem to support "units", only "code".
|
||||
assertFind("when eq with units", "/Observation?value-quantity=0.6||mm[Hg]");
|
||||
assertFind("when eq unitless", "/Observation?value-quantity=0.6");
|
||||
assertNotFind("when lt", "/Observation?value-quantity=0.7||mmHg");
|
||||
|
|
|
@ -0,0 +1,40 @@
|
|||
package ca.uhn.fhir.jpa.dao.search;
|
||||
|
||||
import ca.uhn.fhir.context.FhirContext;
|
||||
import ca.uhn.fhir.parser.IParser;
|
||||
import org.hl7.fhir.instance.model.api.IBaseResource;
|
||||
import org.hl7.fhir.r4.model.Observation;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
|
||||
class ExtendedLuceneResourceProjectionTest {
|
||||
final FhirContext myFhirContext = FhirContext.forR4();
|
||||
final IParser myParser = myFhirContext.newJsonParser();
|
||||
ExtendedLuceneResourceProjection myProjection;
|
||||
IBaseResource myResource;
|
||||
|
||||
@Test
|
||||
public void basicBodyReceivesId() {
|
||||
myProjection = new ExtendedLuceneResourceProjection(22, null, "{ \"resourceType\":\"Observation\"}");
|
||||
|
||||
myResource = myProjection.toResource(myParser);
|
||||
|
||||
assertThat(myResource, instanceOf(Observation.class));
|
||||
assertThat(myResource.getIdElement().getIdPart(), equalTo("22"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void forcedIdOverridesPid() {
|
||||
myProjection = new ExtendedLuceneResourceProjection(22, "force-id", "{ \"resourceType\":\"Observation\"}");
|
||||
|
||||
myResource = myProjection.toResource(myParser);
|
||||
|
||||
assertThat(myResource, instanceOf(Observation.class));
|
||||
assertThat(myResource.getIdElement().getIdPart(), equalTo("force-id"));
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -20,6 +20,7 @@ import org.hamcrest.Description;
|
|||
import org.hamcrest.Matcher;
|
||||
import org.hamcrest.TypeSafeDiagnosingMatcher;
|
||||
import org.hibernate.search.mapper.orm.Search;
|
||||
import org.hl7.fhir.instance.model.api.IBaseCoding;
|
||||
import org.hl7.fhir.instance.model.api.IIdType;
|
||||
import org.hl7.fhir.r4.model.Coding;
|
||||
import org.hl7.fhir.r4.model.Observation;
|
||||
|
@ -193,7 +194,7 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest{
|
|||
}
|
||||
|
||||
@Nonnull
|
||||
private Matcher<TokenAutocompleteHit> matchingSystemAndCode(Coding theCoding) {
|
||||
private Matcher<TokenAutocompleteHit> matchingSystemAndCode(IBaseCoding theCoding) {
|
||||
return new TypeSafeDiagnosingMatcher<TokenAutocompleteHit>() {
|
||||
private final String mySystemAndCode = theCoding.getSystem() + "|" + theCoding.getCode();
|
||||
|
||||
|
|
|
@ -140,6 +140,13 @@ public class ResourceTable extends BaseHasResource implements Serializable, IBas
|
|||
@PropertyBinding(binder = @PropertyBinderRef(type = SearchParamTextPropertyBinder.class))
|
||||
private ExtendedLuceneIndexData myLuceneIndexData;
|
||||
|
||||
// todo mb move this to ExtendedLuceneIndexData
|
||||
@Transient
|
||||
@GenericField(name="myRawResource", projectable = Projectable.YES, searchable = Searchable.NO)
|
||||
@IndexingDependency(derivedFrom = @ObjectPath(@PropertyValue(propertyName = "myVersion")))
|
||||
@OptimisticLock(excluded = true)
|
||||
private String myRawResourceData;
|
||||
|
||||
@OneToMany(mappedBy = "myResource", cascade = {}, fetch = FetchType.LAZY, orphanRemoval = false)
|
||||
@OptimisticLock(excluded = true)
|
||||
private Collection<ResourceIndexedSearchParamCoords> myParamsCoords;
|
||||
|
@ -775,4 +782,8 @@ public class ResourceTable extends BaseHasResource implements Serializable, IBas
|
|||
public void setLuceneIndexData(ExtendedLuceneIndexData theLuceneIndexData) {
|
||||
myLuceneIndexData = theLuceneIndexData;
|
||||
}
|
||||
|
||||
public void setRawResourceData(String theResourceData) {
|
||||
myRawResourceData = theResourceData;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -45,6 +45,7 @@ public class ExtendedLuceneIndexData {
|
|||
final SetMultimap<String, IBaseCoding> mySearchParamTokens = HashMultimap.create();
|
||||
final SetMultimap<String, String> mySearchParamLinks = HashMultimap.create();
|
||||
final SetMultimap<String, DateSearchIndexData> mySearchParamDates = HashMultimap.create();
|
||||
String myForcedId;
|
||||
|
||||
public ExtendedLuceneIndexData(FhirContext theFhirContext) {
|
||||
this.myFhirContext = theFhirContext;
|
||||
|
@ -58,11 +59,20 @@ public class ExtendedLuceneIndexData {
|
|||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the index document.
|
||||
*
|
||||
* Keep this in sync with the schema defined in {@link SearchParamTextPropertyBinder}
|
||||
* @param theDocument
|
||||
*/
|
||||
public void writeIndexElements(DocumentElement theDocument) {
|
||||
HibernateSearchIndexWriter indexWriter = HibernateSearchIndexWriter.forRoot(myFhirContext, theDocument);
|
||||
|
||||
ourLog.debug("Writing JPA index to Hibernate Search");
|
||||
|
||||
theDocument.addValue("myForcedId", myForcedId);
|
||||
|
||||
mySearchParamStrings.forEach(ifNotContained(indexWriter::writeStringIndex));
|
||||
mySearchParamTokens.forEach(ifNotContained(indexWriter::writeTokenIndex));
|
||||
mySearchParamLinks.forEach(ifNotContained(indexWriter::writeReferenceIndex));
|
||||
|
@ -97,4 +107,11 @@ public class ExtendedLuceneIndexData {
|
|||
mySearchParamDates.put(theSpName, new DateSearchIndexData(theLowerBound, theLowerBoundOrdinal, theUpperBound, theUpperBoundOrdinal));
|
||||
}
|
||||
|
||||
public void setForcedId(String theForcedId) {
|
||||
myForcedId = theForcedId;
|
||||
}
|
||||
|
||||
public String getForcedId() {
|
||||
return myForcedId;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -100,11 +100,18 @@ public class SearchParamTextPropertyBinder implements PropertyBinder, PropertyBr
|
|||
.projectable(Projectable.NO)
|
||||
.sortable(Sortable.YES);
|
||||
|
||||
StringIndexFieldTypeOptionsStep<?> forcedIdType = indexFieldTypeFactory.asString()
|
||||
.projectable(Projectable.YES)
|
||||
.aggregable(Aggregable.NO);
|
||||
|
||||
// the old style for _text and _contains
|
||||
indexSchemaElement
|
||||
.fieldTemplate("SearchParamText", standardAnalyzer)
|
||||
.matchingPathGlob(SEARCH_PARAM_TEXT_PREFIX + "*");
|
||||
|
||||
|
||||
indexSchemaElement.field("myForcedId", forcedIdType).toReference();
|
||||
|
||||
// The following section is a bit ugly. We need to enforce order and dependency or the object matches will be too big.
|
||||
{
|
||||
IndexSchemaObjectField spfield = indexSchemaElement.objectField(HibernateSearchIndexWriter.SEARCH_PARAM_ROOT, ObjectStructure.FLATTENED);
|
||||
|
|
Loading…
Reference in New Issue