honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Hsearch no scroll queries (#3799) 

* Avoid scrolling result for not-asynchronous searches

* Adjust deprecated properties

* Add changelog

Co-authored-by: juan.marchionatto <juan.marchionatto@smilecdr.com>
This commit is contained in:
jmarchionatto 2022-07-15 14:04:12 -04:00 committed by GitHub
parent 0c644271ce
commit 398ed99f86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 43 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3801-hsearch-avoid-scrolled-results-for-synchronous-searches.yaml Normal file
View File

@ -0,0 +1,5 @@
---
type: fix
issue: 3801
title: "For Lucene/Elastic previously we were using scrolled results even when performing synchronous searches.
That has now been fixed."

58
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java
View File

@ -30,7 +30,6 @@ import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchResourceProjection;
import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder;
import ca.uhn.fhir.jpa.dao.search.IHSearchSortHelper;
import ca.uhn.fhir.jpa.dao.search.LastNOperation;
import ca.uhn.fhir.jpa.dao.search.SearchScrollQueryExecutorAdaptor;
import ca.uhn.fhir.jpa.model.entity.ModelConfig;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
@ -53,7 +52,6 @@ import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep;
import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory;
import org.hibernate.search.engine.search.projection.dsl.CompositeProjectionOptionsStep;
import org.hibernate.search.engine.search.projection.dsl.SearchProjectionFactory;
import org.hibernate.search.engine.search.query.SearchScroll;
import org.hibernate.search.engine.search.query.dsl.SearchQueryOptionsStep;
import org.hibernate.search.mapper.orm.Search;
import org.hibernate.search.mapper.orm.common.EntityReference;
@ -84,6 +82,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FulltextSearchSvcImpl.class);
private static final int DEFAULT_MAX_NON_PAGED_SIZE = 500;
@PersistenceContext(type = PersistenceContextType.TRANSACTION)
private EntityManager myEntityManager;
@ -145,34 +144,37 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
}
@Override
public ISearchQueryExecutor searchAsync(String theResourceName, SearchParameterMap theParams) {
return doSearch(theResourceName, theParams, null);
}
private ISearchQueryExecutor doSearch(String theResourceType, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
// keep this in sync with supportsSomeOf();
if (theParams.getOffset() != null && theParams.getOffset() != 0) {
// perform an offset search instead of a scroll one, which doesn't allow for offset
List<Long> queryFetchResult = getSearchQueryOptionsStep(
theResourceType, theParams, theReferencingPid).fetchHits(theParams.getOffset(), theParams.getCount());
// indicate param was already processed, otherwise queries DB to process it
theParams.setOffset(null);
return SearchQueryExecutors.from(queryFetchResult);
}
SearchScroll<Long> esResult = getSearchScroll(theResourceType, theParams, theReferencingPid);
return new SearchScrollQueryExecutorAdaptor(esResult);
public ISearchQueryExecutor searchNotScrolled(String theResourceName, SearchParameterMap theParams, Integer theMaxResultsToFetch) {
return doSearch(theResourceName, theParams, null, theMaxResultsToFetch);
}
private SearchScroll<Long> getSearchScroll(String theResourceType, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
// disallow scroll size lees than 50 until we fix scrolling performance
int scrollSize = theParams.getCount() == null ? 50 : Math.max(50, theParams.getCount());
if (theParams.getCount()!=null) {
scrollSize = theParams.getCount();
// keep this in sync with supportsSomeOf();
private ISearchQueryExecutor doSearch(String theResourceType, SearchParameterMap theParams,
ResourcePersistentId theReferencingPid, Integer theMaxResultsToFetch) {
int offset = theParams.getOffset() == null ? 0 : theParams.getOffset();
int count = getMaxFetchSize(theParams, theMaxResultsToFetch);
// perform an offset search instead of a scroll one, which doesn't allow for offset
List<Long> queryFetchResult = getSearchQueryOptionsStep(theResourceType, theParams, theReferencingPid).fetchHits(offset, count);
// indicate param was already processed, otherwise queries DB to process it
theParams.setOffset(null);
return SearchQueryExecutors.from(queryFetchResult);
}
private int getMaxFetchSize(SearchParameterMap theParams, Integer theMax) {
if (theParams.getCount() != null) {
return theParams.getCount();
}
return getSearchQueryOptionsStep(theResourceType, theParams, theReferencingPid).scroll(scrollSize);
if (theMax != null) {
return theMax;
}
return DEFAULT_MAX_NON_PAGED_SIZE;
}
@ -261,10 +263,8 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
@Override
public List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
// wipmb what about max results here?
List<ResourcePersistentId> retVal = toList(doSearch(null, theParams, theReferencingPid), 10000);
List<ResourcePersistentId> retVal = toList(doSearch(null, theParams, theReferencingPid, 10_000), 10_000);
if (theReferencingPid != null) {
retVal.add(theReferencingPid);
}
@ -297,7 +297,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
@Transactional()
@Override
public List<ResourcePersistentId> search(String theResourceName, SearchParameterMap theParams) {
return toList(doSearch(theResourceName, theParams, null), 500);
return toList(doSearch(theResourceName, theParams, null, DEFAULT_MAX_NON_PAGED_SIZE), DEFAULT_MAX_NON_PAGED_SIZE);
}
/**

6
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java
View File

@ -47,14 +47,14 @@ public interface IFulltextSearchSvc {
/**
* Query the index for a scrollable iterator of results.
* No max size to the result iterator.
* Query the index for a plain list (non-scrollable) iterator of results.
*
* @param theResourceName e.g. Patient
* @param theParams The search query
* @param theMaxResultsToFetch maximum results to fetch
* @return Iterator of result PIDs
*/
ISearchQueryExecutor searchAsync(String theResourceName, SearchParameterMap theParams);
ISearchQueryExecutor searchNotScrolled(String theResourceName, SearchParameterMap theParams, Integer theMaxResultsToFetch);
/**
* Autocomplete search for NIH $expand contextDirection=existing

9
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/HapiHSearchAnalysisConfigurers.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilterFactory;
import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilterFactory;
import org.apache.lucene.analysis.ngram.EdgeNGramFilterFactory;
import org.apache.lucene.analysis.ngram.NGramFilterFactory;
import org.apache.lucene.analysis.pattern.PatternTokenizerFactory;
@ -74,7 +75,7 @@ public class HapiHSearchAnalysisConfigurers {
theLuceneCtx.analyzer("autocompleteNGramAnalyzer").custom()
.tokenizer(StandardTokenizerFactory.class)
.tokenFilter(WordDelimiterFilterFactory.class)
.tokenFilter(WordDelimiterGraphFilterFactory.class)
.tokenFilter(LowerCaseFilterFactory.class)
.tokenFilter(NGramFilterFactory.class)
.param("minGramSize", "3")
@ -130,7 +131,7 @@ public class HapiHSearchAnalysisConfigurers {
.param("group", "1");
theConfigCtx.tokenFilter("edgengram_3_50")
.type("edgeNGram")
.type("edge_ngram")
.param("min_gram", "3")
.param("max_gram", "50");
@ -140,7 +141,7 @@ public class HapiHSearchAnalysisConfigurers {
.tokenFilters("lowercase", "stop", "wordedgengram_3_50");
theConfigCtx.tokenFilter("wordedgengram_3_50")
.type("edgeNGram")
.type("edge_ngram")
.param("min_gram", "3")
.param("max_gram", "20");
@ -157,7 +158,7 @@ public class HapiHSearchAnalysisConfigurers {
.tokenFilters("word_delimiter", "lowercase", "ngram_3_20");
theConfigCtx.tokenFilter("ngram_3_20")
.type("nGram")
.type("ngram")
.param("min_gram", "3")
.param("max_gram", "20");

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java
View File

@ -341,7 +341,7 @@ public class SearchBuilder implements ISearchBuilder {
fulltextMatchIds = queryHibernateSearchForEverythingPids();
resultCount = fulltextMatchIds.size();
} else {
fulltextExecutor = myFulltextSearchSvc.searchAsync(myResourceName, myParams);
fulltextExecutor = myFulltextSearchSvc.searchNotScrolled(myResourceName, myParams, myMaxResultsToFetch);
}
if (fulltextExecutor == null) {