From 3ed12ce944f652c67e94b37a835ee4e74dbfc8bb Mon Sep 17 00:00:00 2001 From: James Agnew Date: Wed, 28 Jun 2017 18:13:44 -0400 Subject: [PATCH] Improve performance of JPA searches by using Hibernate ScrollableResults --- .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 143 ++++++++---------- src/changes/changes.xml | 4 + 2 files changed, 63 insertions(+), 84 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 36f99d9368e..f39edd259a8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -27,86 +27,34 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import java.math.BigDecimal; import java.math.MathContext; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Set; import javax.persistence.EntityManager; import javax.persistence.TypedQuery; -import javax.persistence.criteria.AbstractQuery; -import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.*; import javax.persistence.criteria.CriteriaBuilder.In; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.Expression; -import javax.persistence.criteria.From; -import javax.persistence.criteria.Join; -import javax.persistence.criteria.JoinType; -import javax.persistence.criteria.Order; -import javax.persistence.criteria.Path; -import javax.persistence.criteria.Predicate; -import javax.persistence.criteria.Root; -import javax.persistence.criteria.Subquery; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.*; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.tuple.Pair; -import org.hl7.fhir.instance.model.api.IAnyResource; -import org.hl7.fhir.instance.model.api.IBaseResource; -import org.hl7.fhir.instance.model.api.IIdType; +import org.hibernate.ScrollMode; +import org.hibernate.ScrollableResults; +import org.hibernate.query.Query; +import org.hl7.fhir.instance.model.api.*; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import com.google.common.collect.*; -import ca.uhn.fhir.context.BaseRuntimeChildDefinition; -import ca.uhn.fhir.context.BaseRuntimeDeclaredChildDefinition; -import ca.uhn.fhir.context.BaseRuntimeElementDefinition; -import ca.uhn.fhir.context.ConfigurationException; -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.context.FhirVersionEnum; -import ca.uhn.fhir.context.RuntimeChildChoiceDefinition; -import ca.uhn.fhir.context.RuntimeChildResourceDefinition; -import ca.uhn.fhir.context.RuntimeResourceDefinition; -import ca.uhn.fhir.context.RuntimeSearchParam; +import ca.uhn.fhir.context.*; import ca.uhn.fhir.jpa.dao.data.IForcedIdDao; import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao; -import ca.uhn.fhir.jpa.entity.BaseHasResource; -import ca.uhn.fhir.jpa.entity.BaseResourceIndexedSearchParam; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamDate; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamNumber; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamQuantity; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamString; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamToken; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamUri; -import ca.uhn.fhir.jpa.entity.ResourceLink; -import ca.uhn.fhir.jpa.entity.ResourceTable; -import ca.uhn.fhir.jpa.entity.ResourceTag; -import ca.uhn.fhir.jpa.entity.SearchParam; -import ca.uhn.fhir.jpa.entity.SearchParamPresent; -import ca.uhn.fhir.jpa.entity.TagDefinition; -import ca.uhn.fhir.jpa.entity.TagTypeEnum; +import ca.uhn.fhir.jpa.entity.*; import ca.uhn.fhir.jpa.term.IHapiTerminologySvc; import ca.uhn.fhir.jpa.term.VersionIndependentConcept; import ca.uhn.fhir.jpa.util.StopWatch; -import ca.uhn.fhir.model.api.IPrimitiveDatatype; -import ca.uhn.fhir.model.api.IQueryParameterType; -import ca.uhn.fhir.model.api.IResource; -import ca.uhn.fhir.model.api.Include; -import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum; -import ca.uhn.fhir.model.base.composite.BaseCodingDt; -import ca.uhn.fhir.model.base.composite.BaseIdentifierDt; -import ca.uhn.fhir.model.base.composite.BaseQuantityDt; +import ca.uhn.fhir.model.api.*; +import ca.uhn.fhir.model.base.composite.*; import ca.uhn.fhir.model.dstu.resource.BaseResource; import ca.uhn.fhir.model.primitive.IdDt; import ca.uhn.fhir.model.primitive.InstantDt; @@ -115,23 +63,9 @@ import ca.uhn.fhir.parser.DataFormatException; import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.method.RestSearchParameterTypeEnum; -import ca.uhn.fhir.rest.param.CompositeParam; -import ca.uhn.fhir.rest.param.DateParam; -import ca.uhn.fhir.rest.param.DateRangeParam; -import ca.uhn.fhir.rest.param.HasParam; -import ca.uhn.fhir.rest.param.NumberParam; -import ca.uhn.fhir.rest.param.ParamPrefixEnum; -import ca.uhn.fhir.rest.param.QuantityParam; -import ca.uhn.fhir.rest.param.ReferenceParam; -import ca.uhn.fhir.rest.param.StringParam; -import ca.uhn.fhir.rest.param.TokenParam; -import ca.uhn.fhir.rest.param.TokenParamModifier; -import ca.uhn.fhir.rest.param.UriParam; -import ca.uhn.fhir.rest.param.UriParamQualifierEnum; +import ca.uhn.fhir.rest.param.*; import ca.uhn.fhir.rest.server.Constants; -import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import ca.uhn.fhir.rest.server.exceptions.*; import ca.uhn.fhir.util.UrlUtil; /** @@ -139,8 +73,8 @@ import ca.uhn.fhir.util.UrlUtil; * searchs for resources */ public class SearchBuilder implements ISearchBuilder { + private static final List EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList()); - private static Long NO_MORE = Long.valueOf(-1); private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchBuilder.class); private List myAlsoIncludePids; @@ -161,7 +95,7 @@ public class SearchBuilder implements ISearchBuilder { private ISearchParamRegistry mySearchParamRegistry; private String mySearchUuid; private IHapiTerminologySvc myTerminologySvc; - + /** * Constructor */ @@ -2037,6 +1971,7 @@ public class SearchBuilder implements ISearchBuilder { static Predicate[] toArray(List thePredicates) { return thePredicates.toArray(new Predicate[thePredicates.size()]); } + public class IncludesIterator implements Iterator{ private Iterator myCurrentIterator; @@ -2094,7 +2029,6 @@ public class SearchBuilder implements ISearchBuilder { } } - private enum JoinEnum { DATE, NUMBER, QUANTITY, REFERENCE, STRING, TOKEN, URI @@ -2160,7 +2094,12 @@ public class SearchBuilder implements ISearchBuilder { Integer maximumResults = myCallingDao.getConfig().getFetchSizeDefaultMaximum(); final TypedQuery query = createQuery(mySort, maximumResults); - myResultsIterator = query.getResultList().iterator(); + + Query hibernateQuery = (Query) query; + ScrollableResults scroll = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY); + myResultsIterator = new ScrollableResultsIterator(scroll); + +// myResultsIterator = query.getResultList().iterator(); // If the query resulted in extra results being requested if (myAlsoIncludePids != null) { @@ -2245,4 +2184,40 @@ public class SearchBuilder implements ISearchBuilder { } } + public class ScrollableResultsIterator implements Iterator { + + private Long myNext; + private ScrollableResults myScroll; + + public ScrollableResultsIterator(ScrollableResults theScroll) { + myScroll = theScroll; + } + + private void ensureHaveNext() { + if (myNext == null) { + if (myScroll.next()) { + myNext = (Long) myScroll.get(0); + } else { + myNext = NO_MORE; + } + } + } + + @Override + public boolean hasNext() { + ensureHaveNext(); + return myNext != NO_MORE; + } + + @Override + public Long next() { + ensureHaveNext(); + Validate.isTrue(myNext != NO_MORE); + Long next = myNext; + myNext = null; + return next; + } + + } + } diff --git a/src/changes/changes.xml b/src/changes/changes.xml index ec61790271a..c33817e9798 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -62,6 +62,10 @@ actually notify listeners of the first 10 subscriptions. Thanks to Jeff Chung for the pull request! + + JPA search now uses hibernate ScrollableResults instead of plain JPA List. This + should improve performance over large search results. +