Hsearch no scroll queries (#3799)

* Avoid scrolling result for not-asynchronous searches

* Adjust deprecated properties

* Add changelog

Co-authored-by: juan.marchionatto <juan.marchionatto@smilecdr.com>
This commit is contained in:
jmarchionatto 2022-07-15 14:04:12 -04:00 committed by GitHub
parent 0c644271ce
commit 398ed99f86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 43 additions and 37 deletions

View File

@ -0,0 +1,5 @@
---
type: fix
issue: 3801
title: "For Lucene/Elastic previously we were using scrolled results even when performing synchronous searches.
That has now been fixed."

View File

@ -30,7 +30,6 @@ import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchResourceProjection;
import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder; import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder;
import ca.uhn.fhir.jpa.dao.search.IHSearchSortHelper; import ca.uhn.fhir.jpa.dao.search.IHSearchSortHelper;
import ca.uhn.fhir.jpa.dao.search.LastNOperation; import ca.uhn.fhir.jpa.dao.search.LastNOperation;
import ca.uhn.fhir.jpa.dao.search.SearchScrollQueryExecutorAdaptor;
import ca.uhn.fhir.jpa.model.entity.ModelConfig; import ca.uhn.fhir.jpa.model.entity.ModelConfig;
import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData; import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
@ -53,7 +52,6 @@ import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep;
import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory; import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory;
import org.hibernate.search.engine.search.projection.dsl.CompositeProjectionOptionsStep; import org.hibernate.search.engine.search.projection.dsl.CompositeProjectionOptionsStep;
import org.hibernate.search.engine.search.projection.dsl.SearchProjectionFactory; import org.hibernate.search.engine.search.projection.dsl.SearchProjectionFactory;
import org.hibernate.search.engine.search.query.SearchScroll;
import org.hibernate.search.engine.search.query.dsl.SearchQueryOptionsStep; import org.hibernate.search.engine.search.query.dsl.SearchQueryOptionsStep;
import org.hibernate.search.mapper.orm.Search; import org.hibernate.search.mapper.orm.Search;
import org.hibernate.search.mapper.orm.common.EntityReference; import org.hibernate.search.mapper.orm.common.EntityReference;
@ -84,6 +82,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
public class FulltextSearchSvcImpl implements IFulltextSearchSvc { public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FulltextSearchSvcImpl.class); private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FulltextSearchSvcImpl.class);
private static final int DEFAULT_MAX_NON_PAGED_SIZE = 500;
@PersistenceContext(type = PersistenceContextType.TRANSACTION) @PersistenceContext(type = PersistenceContextType.TRANSACTION)
private EntityManager myEntityManager; private EntityManager myEntityManager;
@ -145,34 +144,37 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
} }
@Override @Override
public ISearchQueryExecutor searchAsync(String theResourceName, SearchParameterMap theParams) { public ISearchQueryExecutor searchNotScrolled(String theResourceName, SearchParameterMap theParams, Integer theMaxResultsToFetch) {
return doSearch(theResourceName, theParams, null); return doSearch(theResourceName, theParams, null, theMaxResultsToFetch);
} }
private ISearchQueryExecutor doSearch(String theResourceType, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
// keep this in sync with supportsSomeOf(); // keep this in sync with supportsSomeOf();
if (theParams.getOffset() != null && theParams.getOffset() != 0) { private ISearchQueryExecutor doSearch(String theResourceType, SearchParameterMap theParams,
ResourcePersistentId theReferencingPid, Integer theMaxResultsToFetch) {
int offset = theParams.getOffset() == null ? 0 : theParams.getOffset();
int count = getMaxFetchSize(theParams, theMaxResultsToFetch);
// perform an offset search instead of a scroll one, which doesn't allow for offset // perform an offset search instead of a scroll one, which doesn't allow for offset
List<Long> queryFetchResult = getSearchQueryOptionsStep( List<Long> queryFetchResult = getSearchQueryOptionsStep(theResourceType, theParams, theReferencingPid).fetchHits(offset, count);
theResourceType, theParams, theReferencingPid).fetchHits(theParams.getOffset(), theParams.getCount());
// indicate param was already processed, otherwise queries DB to process it // indicate param was already processed, otherwise queries DB to process it
theParams.setOffset(null); theParams.setOffset(null);
return SearchQueryExecutors.from(queryFetchResult); return SearchQueryExecutors.from(queryFetchResult);
} }
SearchScroll<Long> esResult = getSearchScroll(theResourceType, theParams, theReferencingPid);
return new SearchScrollQueryExecutorAdaptor(esResult);
}
private int getMaxFetchSize(SearchParameterMap theParams, Integer theMax) {
private SearchScroll<Long> getSearchScroll(String theResourceType, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
// disallow scroll size lees than 50 until we fix scrolling performance
int scrollSize = theParams.getCount() == null ? 50 : Math.max(50, theParams.getCount());
if (theParams.getCount() != null) { if (theParams.getCount() != null) {
scrollSize = theParams.getCount(); return theParams.getCount();
} }
return getSearchQueryOptionsStep(theResourceType, theParams, theReferencingPid).scroll(scrollSize); if (theMax != null) {
return theMax;
}
return DEFAULT_MAX_NON_PAGED_SIZE;
} }
@ -261,10 +263,8 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
@Override @Override
public List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) { public List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
// wipmb what about max results here? // wipmb what about max results here?
List<ResourcePersistentId> retVal = toList(doSearch(null, theParams, theReferencingPid), 10000); List<ResourcePersistentId> retVal = toList(doSearch(null, theParams, theReferencingPid, 10_000), 10_000);
if (theReferencingPid != null) { if (theReferencingPid != null) {
retVal.add(theReferencingPid); retVal.add(theReferencingPid);
} }
@ -297,7 +297,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
@Transactional() @Transactional()
@Override @Override
public List<ResourcePersistentId> search(String theResourceName, SearchParameterMap theParams) { public List<ResourcePersistentId> search(String theResourceName, SearchParameterMap theParams) {
return toList(doSearch(theResourceName, theParams, null), 500); return toList(doSearch(theResourceName, theParams, null, DEFAULT_MAX_NON_PAGED_SIZE), DEFAULT_MAX_NON_PAGED_SIZE);
} }
/** /**

View File

@ -47,14 +47,14 @@ public interface IFulltextSearchSvc {
/** /**
* Query the index for a scrollable iterator of results. * Query the index for a plain list (non-scrollable) iterator of results.
* No max size to the result iterator.
* *
* @param theResourceName e.g. Patient * @param theResourceName e.g. Patient
* @param theParams The search query * @param theParams The search query
* @param theMaxResultsToFetch maximum results to fetch
* @return Iterator of result PIDs * @return Iterator of result PIDs
*/ */
ISearchQueryExecutor searchAsync(String theResourceName, SearchParameterMap theParams); ISearchQueryExecutor searchNotScrolled(String theResourceName, SearchParameterMap theParams, Integer theMaxResultsToFetch);
/** /**
* Autocomplete search for NIH $expand contextDirection=existing * Autocomplete search for NIH $expand contextDirection=existing

View File

@ -26,6 +26,7 @@ import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory; import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilterFactory; import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilterFactory;
import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilterFactory;
import org.apache.lucene.analysis.ngram.EdgeNGramFilterFactory; import org.apache.lucene.analysis.ngram.EdgeNGramFilterFactory;
import org.apache.lucene.analysis.ngram.NGramFilterFactory; import org.apache.lucene.analysis.ngram.NGramFilterFactory;
import org.apache.lucene.analysis.pattern.PatternTokenizerFactory; import org.apache.lucene.analysis.pattern.PatternTokenizerFactory;
@ -74,7 +75,7 @@ public class HapiHSearchAnalysisConfigurers {
theLuceneCtx.analyzer("autocompleteNGramAnalyzer").custom() theLuceneCtx.analyzer("autocompleteNGramAnalyzer").custom()
.tokenizer(StandardTokenizerFactory.class) .tokenizer(StandardTokenizerFactory.class)
.tokenFilter(WordDelimiterFilterFactory.class) .tokenFilter(WordDelimiterGraphFilterFactory.class)
.tokenFilter(LowerCaseFilterFactory.class) .tokenFilter(LowerCaseFilterFactory.class)
.tokenFilter(NGramFilterFactory.class) .tokenFilter(NGramFilterFactory.class)
.param("minGramSize", "3") .param("minGramSize", "3")
@ -130,7 +131,7 @@ public class HapiHSearchAnalysisConfigurers {
.param("group", "1"); .param("group", "1");
theConfigCtx.tokenFilter("edgengram_3_50") theConfigCtx.tokenFilter("edgengram_3_50")
.type("edgeNGram") .type("edge_ngram")
.param("min_gram", "3") .param("min_gram", "3")
.param("max_gram", "50"); .param("max_gram", "50");
@ -140,7 +141,7 @@ public class HapiHSearchAnalysisConfigurers {
.tokenFilters("lowercase", "stop", "wordedgengram_3_50"); .tokenFilters("lowercase", "stop", "wordedgengram_3_50");
theConfigCtx.tokenFilter("wordedgengram_3_50") theConfigCtx.tokenFilter("wordedgengram_3_50")
.type("edgeNGram") .type("edge_ngram")
.param("min_gram", "3") .param("min_gram", "3")
.param("max_gram", "20"); .param("max_gram", "20");
@ -157,7 +158,7 @@ public class HapiHSearchAnalysisConfigurers {
.tokenFilters("word_delimiter", "lowercase", "ngram_3_20"); .tokenFilters("word_delimiter", "lowercase", "ngram_3_20");
theConfigCtx.tokenFilter("ngram_3_20") theConfigCtx.tokenFilter("ngram_3_20")
.type("nGram") .type("ngram")
.param("min_gram", "3") .param("min_gram", "3")
.param("max_gram", "20"); .param("max_gram", "20");

View File

@ -341,7 +341,7 @@ public class SearchBuilder implements ISearchBuilder {
fulltextMatchIds = queryHibernateSearchForEverythingPids(); fulltextMatchIds = queryHibernateSearchForEverythingPids();
resultCount = fulltextMatchIds.size(); resultCount = fulltextMatchIds.size();
} else { } else {
fulltextExecutor = myFulltextSearchSvc.searchAsync(myResourceName, myParams); fulltextExecutor = myFulltextSearchSvc.searchNotScrolled(myResourceName, myParams, myMaxResultsToFetch);
} }
if (fulltextExecutor == null) { if (fulltextExecutor == null) {