Hsearch no scroll queries (#3799)

* Avoid scrolling result for not-asynchronous searches

* Adjust deprecated properties

* Add changelog

Co-authored-by: juan.marchionatto <juan.marchionatto@smilecdr.com>

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3801-hsearch-avoid-scrolled-results-for-synchronous-searches.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 3801
+title: "For Lucene/Elastic previously we were using scrolled results even when performing synchronous searches.
+   That has now been fixed."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java

@@ -30,7 +30,6 @@ import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchResourceProjection;
 import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder;
 import ca.uhn.fhir.jpa.dao.search.IHSearchSortHelper;
 import ca.uhn.fhir.jpa.dao.search.LastNOperation;
-import ca.uhn.fhir.jpa.dao.search.SearchScrollQueryExecutorAdaptor;
 import ca.uhn.fhir.jpa.model.entity.ModelConfig;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
@@ -53,7 +52,6 @@ import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep;
 import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory;
 import org.hibernate.search.engine.search.projection.dsl.CompositeProjectionOptionsStep;
 import org.hibernate.search.engine.search.projection.dsl.SearchProjectionFactory;
-import org.hibernate.search.engine.search.query.SearchScroll;
 import org.hibernate.search.engine.search.query.dsl.SearchQueryOptionsStep;
 import org.hibernate.search.mapper.orm.Search;
 import org.hibernate.search.mapper.orm.common.EntityReference;
@@ -84,6 +82,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FulltextSearchSvcImpl.class);
+	private static final int DEFAULT_MAX_NON_PAGED_SIZE = 500;
 
 	@PersistenceContext(type = PersistenceContextType.TRANSACTION)
 	private EntityManager myEntityManager;
@@ -145,34 +144,37 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 	}
 
 	@Override
-	public ISearchQueryExecutor searchAsync(String theResourceName, SearchParameterMap theParams) {
+	public ISearchQueryExecutor searchNotScrolled(String theResourceName, SearchParameterMap theParams, Integer theMaxResultsToFetch) {
-		return doSearch(theResourceName, theParams, null);
+		return doSearch(theResourceName, theParams, null, theMaxResultsToFetch);
-	}
-
-	private ISearchQueryExecutor doSearch(String theResourceType, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
-		// keep this in sync with supportsSomeOf();
-		if (theParams.getOffset() != null && theParams.getOffset() != 0) {
-			// perform an offset search instead of a scroll one, which doesn't allow for offset
-			List<Long> queryFetchResult = getSearchQueryOptionsStep(
-				theResourceType, theParams, theReferencingPid).fetchHits(theParams.getOffset(), theParams.getCount());
-			// indicate param was already processed, otherwise queries DB to process it
-			theParams.setOffset(null);
-			return SearchQueryExecutors.from(queryFetchResult);
-		}
-
-		SearchScroll<Long> esResult = getSearchScroll(theResourceType, theParams, theReferencingPid);
-		return new SearchScrollQueryExecutorAdaptor(esResult);
 	}
 
 
-	private SearchScroll<Long> getSearchScroll(String theResourceType, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
+	// keep this in sync with supportsSomeOf();
-		// disallow scroll size lees than 50 until we fix scrolling performance
+	private ISearchQueryExecutor doSearch(String theResourceType, SearchParameterMap theParams,
-		int scrollSize = theParams.getCount() == null ? 50 : Math.max(50, theParams.getCount());
+				ResourcePersistentId theReferencingPid, Integer theMaxResultsToFetch) {
-		if (theParams.getCount()!=null) {
+
-			scrollSize = theParams.getCount();
+		int offset = theParams.getOffset() == null ? 0 : theParams.getOffset();
+		int count = getMaxFetchSize(theParams, theMaxResultsToFetch);
+
+		// perform an offset search instead of a scroll one, which doesn't allow for offset
+		List<Long> queryFetchResult = getSearchQueryOptionsStep(theResourceType, theParams, theReferencingPid).fetchHits(offset, count);
+
+		// indicate param was already processed, otherwise queries DB to process it
+		theParams.setOffset(null);
+		return SearchQueryExecutors.from(queryFetchResult);
+	}
+
+
+	private int getMaxFetchSize(SearchParameterMap theParams, Integer theMax) {
+		if (theParams.getCount() != null) {
+			return theParams.getCount();
 		}
 
-		return getSearchQueryOptionsStep(theResourceType, theParams, theReferencingPid).scroll(scrollSize);
+		if (theMax != null) {
+			return theMax;
+		}
+
+		return DEFAULT_MAX_NON_PAGED_SIZE;
 	}
 
 
@@ -261,10 +263,8 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 
 	@Override
 	public List<ResourcePersistentId> everything(String theResourceName, SearchParameterMap theParams, ResourcePersistentId theReferencingPid) {
-
-
 		// wipmb what about max results here?
-		List<ResourcePersistentId> retVal = toList(doSearch(null, theParams, theReferencingPid), 10000);
+		List<ResourcePersistentId> retVal = toList(doSearch(null, theParams, theReferencingPid, 10_000), 10_000);
 		if (theReferencingPid != null) {
 			retVal.add(theReferencingPid);
 		}
@@ -297,7 +297,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 	@Transactional()
 	@Override
 	public List<ResourcePersistentId> search(String theResourceName, SearchParameterMap theParams) {
-		return toList(doSearch(theResourceName, theParams, null), 500);
+		return toList(doSearch(theResourceName, theParams, null, DEFAULT_MAX_NON_PAGED_SIZE), DEFAULT_MAX_NON_PAGED_SIZE);
 	}
 
 	/**

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java

@@ -47,14 +47,14 @@ public interface IFulltextSearchSvc {
 
 
 	/**
-	 * Query the index for a scrollable iterator of results.
+	 * Query the index for a plain list (non-scrollable) iterator of results.
-	 * No max size to the result iterator.
 	 *
 	 * @param theResourceName e.g. Patient
 	 * @param theParams The search query
+	 * @param theMaxResultsToFetch maximum results to fetch
 	 * @return Iterator of result PIDs
 	 */
-	ISearchQueryExecutor searchAsync(String theResourceName, SearchParameterMap theParams);
+	ISearchQueryExecutor searchNotScrolled(String theResourceName, SearchParameterMap theParams, Integer theMaxResultsToFetch);
 
 	/**
 	 * Autocomplete search for NIH $expand contextDirection=existing

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/HapiHSearchAnalysisConfigurers.java

@@ -26,6 +26,7 @@ import org.apache.lucene.analysis.core.StopFilterFactory;
 import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
 import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
 import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilterFactory;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilterFactory;
 import org.apache.lucene.analysis.ngram.EdgeNGramFilterFactory;
 import org.apache.lucene.analysis.ngram.NGramFilterFactory;
 import org.apache.lucene.analysis.pattern.PatternTokenizerFactory;
@@ -74,7 +75,7 @@ public class HapiHSearchAnalysisConfigurers {
 
 			theLuceneCtx.analyzer("autocompleteNGramAnalyzer").custom()
 				.tokenizer(StandardTokenizerFactory.class)
-				.tokenFilter(WordDelimiterFilterFactory.class)
+				.tokenFilter(WordDelimiterGraphFilterFactory.class)
 				.tokenFilter(LowerCaseFilterFactory.class)
 				.tokenFilter(NGramFilterFactory.class)
 				.param("minGramSize", "3")
@@ -130,7 +131,7 @@ public class HapiHSearchAnalysisConfigurers {
 				.param("group", "1");
 
 			theConfigCtx.tokenFilter("edgengram_3_50")
-				.type("edgeNGram")
+				.type("edge_ngram")
 				.param("min_gram", "3")
 				.param("max_gram", "50");
 
@@ -140,7 +141,7 @@ public class HapiHSearchAnalysisConfigurers {
 				.tokenFilters("lowercase", "stop", "wordedgengram_3_50");
 
 			theConfigCtx.tokenFilter("wordedgengram_3_50")
-				.type("edgeNGram")
+				.type("edge_ngram")
 				.param("min_gram", "3")
 				.param("max_gram", "20");
 
@@ -157,7 +158,7 @@ public class HapiHSearchAnalysisConfigurers {
 				.tokenFilters("word_delimiter", "lowercase", "ngram_3_20");
 
 			theConfigCtx.tokenFilter("ngram_3_20")
-				.type("nGram")
+				.type("ngram")
 				.param("min_gram", "3")
 				.param("max_gram", "20");
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java

@@ -341,7 +341,7 @@ public class SearchBuilder implements ISearchBuilder {
 				fulltextMatchIds = queryHibernateSearchForEverythingPids();
 				resultCount = fulltextMatchIds.size();
 			} else {
-				fulltextExecutor = myFulltextSearchSvc.searchAsync(myResourceName, myParams);
+				fulltextExecutor = myFulltextSearchSvc.searchNotScrolled(myResourceName, myParams, myMaxResultsToFetch);
 			}
 
 			if (fulltextExecutor == null) {