diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index 6008c0f9aa8..365b4a34a89 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -100,6 +100,11 @@ thymeleaf test + + net.ttddyy + datasource-proxy + test + org.javassist diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java index 051ee0bdeb7..5ed0a47b7f0 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java @@ -33,6 +33,7 @@ import javax.persistence.criteria.Root; import org.hl7.fhir.instance.model.api.IBaseResource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Propagation; @@ -40,8 +41,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; -import ca.uhn.fhir.jpa.dao.data.IForcedIdDao; -import ca.uhn.fhir.jpa.dao.data.ITermConceptDao; +import ca.uhn.fhir.jpa.dao.data.*; import ca.uhn.fhir.jpa.entity.ForcedId; import ca.uhn.fhir.jpa.entity.ResourceTable; import ca.uhn.fhir.jpa.util.ReindexFailureException; @@ -87,30 +87,33 @@ public abstract class BaseHapiFhirSystemDao extends BaseHapiFhirDao() { @SuppressWarnings("unchecked") @Override public Integer doInTransaction(TransactionStatus theStatus) { - TypedQuery q = myEntityManager.createQuery("SELECT t FROM " + ResourceTable.class.getSimpleName() + " t WHERE t.myIndexStatus IS null", ResourceTable.class); int maxResult = 500; if (theCount != null) { maxResult = Math.min(theCount, 2000); } + maxResult = Math.max(maxResult, 10); + TypedQuery q = myEntityManager.createQuery("SELECT t.myId FROM ResourceTable t WHERE t.myIndexStatus IS NULL", Long.class); + + ourLog.info("Beginning indexing query with maximum {}", maxResult); q.setMaxResults(maxResult); - List resources = q.getResultList(); - if (resources.isEmpty()) { - return 0; - } - - ourLog.info("Indexing {} resources", resources.size()); + Collection resources = q.getResultList(); int count = 0; long start = System.currentTimeMillis(); - for (ResourceTable resourceTable : resources) { + for (Long nextId : resources) { + ResourceTable resourceTable = myResourceTableDao.findOne(nextId); + try { /* * This part is because from HAPI 1.5 - 1.6 we changed the format of forced ID to be "type/id" instead of just "id" @@ -135,13 +138,17 @@ public abstract class BaseHapiFhirSystemDao extends BaseHapiFhirDao= maxResult) { + break; + } } long delay = System.currentTimeMillis() - start; - long avg = (delay / resources.size()); - ourLog.info("Indexed {} / {} resources in {}ms - Avg {}ms / resource", new Object[] { count, resources.size(), delay, avg }); + long avg = count > 0 ? (delay / count) : 0; + ourLog.info("Indexed {} resources in {}ms - Avg {}ms / resource", new Object[] { count, delay, avg }); - return resources.size(); + return count; } }); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 7de8452ee3b..110d0703276 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -27,90 +27,31 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import java.math.BigDecimal; import java.math.MathContext; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Set; import javax.persistence.EntityManager; import javax.persistence.TypedQuery; -import javax.persistence.criteria.AbstractQuery; -import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.*; import javax.persistence.criteria.CriteriaBuilder.In; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.Expression; -import javax.persistence.criteria.From; -import javax.persistence.criteria.Join; -import javax.persistence.criteria.JoinType; -import javax.persistence.criteria.Order; -import javax.persistence.criteria.Path; -import javax.persistence.criteria.Predicate; -import javax.persistence.criteria.Root; -import javax.persistence.criteria.Subquery; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.*; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.tuple.Pair; -import org.hl7.fhir.instance.model.api.IAnyResource; -import org.hl7.fhir.instance.model.api.IBaseResource; -import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.instance.model.api.*; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import com.google.common.collect.*; -import ca.uhn.fhir.context.BaseRuntimeChildDefinition; -import ca.uhn.fhir.context.BaseRuntimeDeclaredChildDefinition; -import ca.uhn.fhir.context.BaseRuntimeElementDefinition; -import ca.uhn.fhir.context.ConfigurationException; -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.context.FhirVersionEnum; -import ca.uhn.fhir.context.RuntimeChildChoiceDefinition; -import ca.uhn.fhir.context.RuntimeChildResourceDefinition; -import ca.uhn.fhir.context.RuntimeResourceDefinition; -import ca.uhn.fhir.context.RuntimeSearchParam; -import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao; -import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; -import ca.uhn.fhir.jpa.dao.IDao; -import ca.uhn.fhir.jpa.dao.IFhirResourceDao; -import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; -import ca.uhn.fhir.jpa.dao.ISearchBuilder; -import ca.uhn.fhir.jpa.dao.ISearchParamRegistry; +import ca.uhn.fhir.context.*; import ca.uhn.fhir.jpa.dao.data.IForcedIdDao; import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao; -import ca.uhn.fhir.jpa.entity.BaseHasResource; -import ca.uhn.fhir.jpa.entity.BaseResourceIndexedSearchParam; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamDate; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamNumber; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamQuantity; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamString; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamToken; -import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamUri; -import ca.uhn.fhir.jpa.entity.ResourceLink; -import ca.uhn.fhir.jpa.entity.ResourceTable; -import ca.uhn.fhir.jpa.entity.ResourceTag; -import ca.uhn.fhir.jpa.entity.SearchParam; -import ca.uhn.fhir.jpa.entity.SearchParamPresent; -import ca.uhn.fhir.jpa.entity.TagDefinition; -import ca.uhn.fhir.jpa.entity.TagTypeEnum; +import ca.uhn.fhir.jpa.entity.*; import ca.uhn.fhir.jpa.term.IHapiTerminologySvc; import ca.uhn.fhir.jpa.term.VersionIndependentConcept; import ca.uhn.fhir.jpa.util.StopWatch; -import ca.uhn.fhir.model.api.IPrimitiveDatatype; -import ca.uhn.fhir.model.api.IQueryParameterType; -import ca.uhn.fhir.model.api.IResource; -import ca.uhn.fhir.model.api.Include; -import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum; -import ca.uhn.fhir.model.base.composite.BaseCodingDt; -import ca.uhn.fhir.model.base.composite.BaseIdentifierDt; -import ca.uhn.fhir.model.base.composite.BaseQuantityDt; +import ca.uhn.fhir.model.api.*; +import ca.uhn.fhir.model.base.composite.*; import ca.uhn.fhir.model.dstu.resource.BaseResource; import ca.uhn.fhir.model.primitive.IdDt; import ca.uhn.fhir.model.primitive.InstantDt; @@ -119,23 +60,9 @@ import ca.uhn.fhir.parser.DataFormatException; import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.method.RestSearchParameterTypeEnum; -import ca.uhn.fhir.rest.param.CompositeParam; -import ca.uhn.fhir.rest.param.DateParam; -import ca.uhn.fhir.rest.param.DateRangeParam; -import ca.uhn.fhir.rest.param.HasParam; -import ca.uhn.fhir.rest.param.NumberParam; -import ca.uhn.fhir.rest.param.ParamPrefixEnum; -import ca.uhn.fhir.rest.param.QuantityParam; -import ca.uhn.fhir.rest.param.ReferenceParam; -import ca.uhn.fhir.rest.param.StringParam; -import ca.uhn.fhir.rest.param.TokenParam; -import ca.uhn.fhir.rest.param.TokenParamModifier; -import ca.uhn.fhir.rest.param.UriParam; -import ca.uhn.fhir.rest.param.UriParamQualifierEnum; +import ca.uhn.fhir.rest.param.*; import ca.uhn.fhir.rest.server.Constants; -import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import ca.uhn.fhir.rest.server.exceptions.*; import ca.uhn.fhir.util.UrlUtil; /** @@ -145,12 +72,14 @@ import ca.uhn.fhir.util.UrlUtil; public class SearchBuilder implements ISearchBuilder { private static Long NO_MORE = Long.valueOf(-1); private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchBuilder.class); + private List myAlsoIncludePids; private CriteriaBuilder myBuilder; private BaseHapiFhirDao myCallingDao; private FhirContext myContext; private EntityManager myEntityManager; private IForcedIdDao myForcedIdDao; private IFulltextSearchSvc myFulltextSearchSvc; + private Map> myIndexJoins = Maps.newHashMap(); private SearchParameterMap myParams; private ArrayList myPredicates; private IResourceIndexedSearchParamUriDao myResourceIndexedSearchParamUriDao; @@ -159,8 +88,8 @@ public class SearchBuilder implements ISearchBuilder { private Root myResourceTableRoot; private Class myResourceType; private ISearchParamRegistry mySearchParamRegistry; - private IHapiTerminologySvc myTerminologySvc; private String mySearchUuid; + private IHapiTerminologySvc myTerminologySvc; /** * Constructor @@ -199,7 +128,7 @@ public class SearchBuilder implements ISearchBuilder { private void addPredicateDate(String theResourceName, String theParamName, List theList) { - Join join = myResourceTableRoot.join("myParamsDate", JoinType.LEFT); + Join join = createOrReuseJoin(JoinEnum.DATE, theParamName); if (theList.get(0).getMissing() != null) { Boolean missing = theList.get(0).getMissing(); @@ -275,28 +204,6 @@ public class SearchBuilder implements ISearchBuilder { } } - // private void addPredicateId(Set thePids) { - // if (thePids == null || thePids.isEmpty()) { - // return; - // } - // - // CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); - // CriteriaQuery cq = builder.createQuery(Long.class); - // Root from = cq.from(ResourceTable.class); - // cq.select(from.get("myId").as(Long.class)); - // - // List predicates = new ArrayList(); - // predicates.add(builder.equal(from.get("myResourceType"), myResourceName)); - // predicates.add(from.get("myId").in(thePids)); - // createPredicateResourceId(builder, cq, predicates, from.get("myId").as(Long.class)); - // createPredicateLastUpdatedForResourceTable(builder, from, predicates); - // - // cq.where(toArray(predicates)); - // - // TypedQuery q = myEntityManager.createQuery(cq); - // doSetPids(q.getResultList()); - // } - private void addPredicateLanguage(List> theList) { for (List nextList : theList) { @@ -326,7 +233,7 @@ public class SearchBuilder implements ISearchBuilder { private void addPredicateNumber(String theResourceName, String theParamName, List theList) { - Join join = myResourceTableRoot.join("myParamsNumber", JoinType.LEFT); + Join join = createOrReuseJoin(JoinEnum.NUMBER, theParamName); if (theList.get(0).getMissing() != null) { addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join); @@ -378,7 +285,7 @@ public class SearchBuilder implements ISearchBuilder { } private void addPredicateQuantity(String theResourceName, String theParamName, List theList) { - Join join = myResourceTableRoot.join("myParamsQuantity", JoinType.LEFT); + Join join = createOrReuseJoin(JoinEnum.QUANTITY, theParamName); if (theList.get(0).getMissing() != null) { addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join); @@ -406,7 +313,7 @@ public class SearchBuilder implements ISearchBuilder { return; } - Join join = myResourceTableRoot.join("myResourceLinks", JoinType.LEFT); + Join join = createOrReuseJoin(JoinEnum.REFERENCE, theParamName); List codePredicates = new ArrayList(); @@ -466,7 +373,7 @@ public class SearchBuilder implements ISearchBuilder { RuntimeResourceDefinition resourceDef = myContext.getResourceDefinition(theResourceName); RuntimeSearchParam searchParamByName = myCallingDao.getSearchParamByName(resourceDef, theParamName); if (searchParamByName == null) { - throw new InternalErrorException("Could not find parameter " + theParamName ); + throw new InternalErrorException("Could not find parameter " + theParamName); } String paramPath = searchParamByName.getPath(); if (paramPath.endsWith(".as(Reference)")) { @@ -496,7 +403,7 @@ public class SearchBuilder implements ISearchBuilder { if (resourceTypes.isEmpty()) { for (BaseRuntimeElementDefinition next : myContext.getElementDefinitions()) { if (next instanceof RuntimeResourceDefinition) { - RuntimeResourceDefinition nextResDef = (RuntimeResourceDefinition)next; + RuntimeResourceDefinition nextResDef = (RuntimeResourceDefinition) next; resourceTypes.add(nextResDef.getImplementingClass()); } } @@ -582,8 +489,10 @@ public class SearchBuilder implements ISearchBuilder { */ Root stackRoot = myResourceTableRoot; ArrayList stackPredicates = myPredicates; + Map> stackIndexJoins = myIndexJoins; myResourceTableRoot = subQfrom; - myPredicates = new ArrayList(); + myPredicates = Lists.newArrayList(); + myIndexJoins = Maps.newHashMap(); // Create the subquery predicates myPredicates.add(myBuilder.equal(myResourceTableRoot.get("myResourceType"), subResourceName)); @@ -597,6 +506,7 @@ public class SearchBuilder implements ISearchBuilder { */ myResourceTableRoot = stackRoot; myPredicates = stackPredicates; + myIndexJoins = stackIndexJoins; Predicate pathPredicate = createResourceLinkPathPredicate(theResourceName, theParamName, join); Predicate pidPredicate = join.get("myTargetResourcePid").in(subQ); @@ -661,7 +571,7 @@ public class SearchBuilder implements ISearchBuilder { private void addPredicateString(String theResourceName, String theParamName, List theList) { - Join join = myResourceTableRoot.join("myParamsString", JoinType.LEFT); + Join join = createOrReuseJoin(JoinEnum.STRING, theParamName); if (theList.get(0).getMissing() != null) { addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join); @@ -809,7 +719,7 @@ public class SearchBuilder implements ISearchBuilder { private void addPredicateToken(String theResourceName, String theParamName, List theList) { - Join join = myResourceTableRoot.join("myParamsToken", JoinType.LEFT); + Join join = createOrReuseJoin(JoinEnum.TOKEN, theParamName); if (theList.get(0).getMissing() != null) { addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join); @@ -841,7 +751,7 @@ public class SearchBuilder implements ISearchBuilder { private void addPredicateUri(String theResourceName, String theParamName, List theList) { - Join join = myResourceTableRoot.join("myParamsUri", JoinType.LEFT); + Join join = createOrReuseJoin(JoinEnum.URI, theParamName); if (theList.get(0).getMissing() != null) { addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join); @@ -966,6 +876,42 @@ public class SearchBuilder implements ISearchBuilder { return retVal; } + @SuppressWarnings("unchecked") + private Join createOrReuseJoin(JoinEnum theType, String theSearchParameterName) { + Join join = null; + + switch (theType) { + case DATE: + join = myResourceTableRoot.join("myParamsDate", JoinType.LEFT); + break; + case NUMBER: + join = myResourceTableRoot.join("myParamsNumber", JoinType.LEFT); + break; + case QUANTITY: + join = myResourceTableRoot.join("myParamsQuantity", JoinType.LEFT); + break; + case REFERENCE: + join = myResourceTableRoot.join("myResourceLinks", JoinType.LEFT); + break; + case STRING: + join = myResourceTableRoot.join("myParamsString", JoinType.LEFT); + break; + case URI: + join = myResourceTableRoot.join("myParamsUri", JoinType.LEFT); + break; + case TOKEN: + join = myResourceTableRoot.join("myParamsToken", JoinType.LEFT); + break; + } + + JoinKey key = new JoinKey(theSearchParameterName, theType); + if (!myIndexJoins.containsKey(key)) { + myIndexJoins.put(key, join); + } + + return (Join) join; + } + private Predicate createPredicateDate(IQueryParameterType theParam, String theResourceName, String theParamName, CriteriaBuilder theBuilder, From theFrom) { Predicate p; if (theParam instanceof DateParam) { @@ -1245,10 +1191,10 @@ public class SearchBuilder implements ISearchBuilder { // Use "in" in case of large numbers of codes due to param modifiers final Path systemExpression = theFrom.get("mySystem"); final Path valueExpression = theFrom.get("myValue"); - for (Map.Entry> entry: map.entrySet()) { + for (Map.Entry> entry : map.entrySet()) { Predicate systemPredicate = theBuilder.equal(systemExpression, entry.getKey()); In codePredicate = theBuilder.in(valueExpression); - for (VersionIndependentConcept nextCode: entry.getValue()) { + for (VersionIndependentConcept nextCode : entry.getValue()) { codePredicate.value(nextCode.getCode()); } orPredicates.add(theBuilder.and(systemPredicate, codePredicate)); @@ -1298,8 +1244,6 @@ public class SearchBuilder implements ISearchBuilder { return new QueryIterator(); } - private List myAlsoIncludePids; - private TypedQuery createQuery(SortSpec sort) { CriteriaQuery outerQuery; /* @@ -1344,16 +1288,7 @@ public class SearchBuilder implements ISearchBuilder { } - myResourceTableQuery.distinct(true); myPredicates = new ArrayList(); - if (myParams.getEverythingMode() == null) { - myPredicates.add(myBuilder.equal(myResourceTableRoot.get("myResourceType"), myResourceName)); - } - myPredicates.add(myBuilder.isNull(myResourceTableRoot.get("myDeleted"))); - - DateRangeParam lu = myParams.getLastUpdated(); - List lastUpdatedPredicates = createLastUpdatedPredicates(lu, myBuilder, myResourceTableRoot); - myPredicates.addAll(lastUpdatedPredicates); if (myParams.getEverythingMode() != null) { Join join = myResourceTableRoot.join("myResourceLinks", JoinType.LEFT); @@ -1397,6 +1332,25 @@ public class SearchBuilder implements ISearchBuilder { myPredicates.add(myResourceTableRoot.get("myId").as(Long.class).in(pids)); } + /* + * Add a predicate to make sure we only include non-deleted resources, and only include + * resources of the right type. + * + * If we have any joins to index tables, we get this behaviour already guaranteed so we don't + * need an explicit predicate for it. + */ + if (myIndexJoins.isEmpty()) { + if (myParams.getEverythingMode() == null) { + myPredicates.add(myBuilder.equal(myResourceTableRoot.get("myResourceType"), myResourceName)); + } + myPredicates.add(myBuilder.isNull(myResourceTableRoot.get("myDeleted"))); + } + + // Last updated + DateRangeParam lu = myParams.getLastUpdated(); + List lastUpdatedPredicates = createLastUpdatedPredicates(lu, myBuilder, myResourceTableRoot); + myPredicates.addAll(lastUpdatedPredicates); + myResourceTableQuery.where(myBuilder.and(SearchBuilder.toArray(myPredicates))); /* @@ -1450,47 +1404,65 @@ public class SearchBuilder implements ISearchBuilder { String joinAttrName; String[] sortAttrName; + JoinEnum joinType; switch (param.getParamType()) { case STRING: joinAttrName = "myParamsString"; sortAttrName = new String[] { "myValueExact" }; + joinType = JoinEnum.STRING; break; case DATE: joinAttrName = "myParamsDate"; sortAttrName = new String[] { "myValueLow" }; + joinType = JoinEnum.DATE; break; case REFERENCE: joinAttrName = "myResourceLinks"; sortAttrName = new String[] { "myTargetResourcePid" }; + joinType = JoinEnum.REFERENCE; break; case TOKEN: joinAttrName = "myParamsToken"; sortAttrName = new String[] { "mySystem", "myValue" }; + joinType = JoinEnum.TOKEN; break; case NUMBER: joinAttrName = "myParamsNumber"; sortAttrName = new String[] { "myValue" }; + joinType = JoinEnum.NUMBER; break; case URI: joinAttrName = "myParamsUri"; sortAttrName = new String[] { "myUri" }; + joinType = JoinEnum.URI; break; case QUANTITY: joinAttrName = "myParamsQuantity"; sortAttrName = new String[] { "myValue" }; + joinType = JoinEnum.QUANTITY; break; default: throw new InvalidRequestException("This server does not support _sort specifications of type " + param.getParamType() + " - Can't serve _sort=" + theSort.getParamName()); } - From join = theFrom.join(joinAttrName, JoinType.LEFT); + /* + * If we've already got a join for the specific parameter we're + * sorting on, we'll also sort with it. Otherwise we need a new join. + */ + JoinKey key = new JoinKey(theSort.getParamName(), joinType); + Join join = myIndexJoins.get(key); + if (join == null) { + join = theFrom.join(joinAttrName, JoinType.LEFT); - if (param.getParamType() == RestSearchParameterTypeEnum.REFERENCE) { - thePredicates.add(join.get("mySourcePath").as(String.class).in(param.getPathsSplit())); + if (param.getParamType() == RestSearchParameterTypeEnum.REFERENCE) { + thePredicates.add(join.get("mySourcePath").as(String.class).in(param.getPathsSplit())); + } else { + Predicate joinParam1 = theBuilder.equal(join.get("myParamName"), theSort.getParamName()); + thePredicates.add(joinParam1); + } } else { - Predicate joinParam1 = theBuilder.equal(join.get("myParamName"), theSort.getParamName()); - thePredicates.add(joinParam1); + ourLog.info("Reusing join for {}", theSort.getParamName()); } for (String next : sortAttrName) { @@ -1536,41 +1508,6 @@ public class SearchBuilder implements ISearchBuilder { return retVal; } - @Override - public void loadResourcesByPid(Collection theIncludePids, List theResourceListToPopulate, Set theRevIncludedPids, boolean theForHistoryOperation, - EntityManager entityManager, FhirContext context, IDao theDao) { - if (theIncludePids.isEmpty()) { - ourLog.info("The include pids are empty"); - //return; - } - - // Dupes will cause a crash later anyhow, but this is expensive so only do it - // when running asserts - assert new HashSet(theIncludePids).size() == theIncludePids.size() : "PID list contains duplicates: " + theIncludePids; - - Map position = new HashMap(); - for (Long next : theIncludePids) { - position.put(next, theResourceListToPopulate.size()); - theResourceListToPopulate.add(null); - } - - /* - * As always, Oracle can't handle things that other databases don't mind.. In this - * case it doesn't like more than ~1000 IDs in a single load, so we break this up - * if it's lots of IDs. I suppose maybe we should be doing this as a join anyhow - * but this should work too. Sigh. - */ - int maxLoad = 800; - List pids = new ArrayList(theIncludePids); - for (int i = 0; i < pids.size(); i += maxLoad) { - int to = i + maxLoad; - to = Math.min(to, pids.size()); - List pidsSubList = pids.subList(i, to); - doLoadPids(theResourceListToPopulate, theRevIncludedPids, theForHistoryOperation, entityManager, context, theDao, position, pidsSubList); - } - - } - private void doLoadPids(List theResourceListToPopulate, Set theRevIncludedPids, boolean theForHistoryOperation, EntityManager entityManager, FhirContext context, IDao theDao, Map position, Collection pids) { CriteriaBuilder builder = entityManager.getCriteriaBuilder(); @@ -1606,6 +1543,41 @@ public class SearchBuilder implements ISearchBuilder { } } + @Override + public void loadResourcesByPid(Collection theIncludePids, List theResourceListToPopulate, Set theRevIncludedPids, boolean theForHistoryOperation, + EntityManager entityManager, FhirContext context, IDao theDao) { + if (theIncludePids.isEmpty()) { + ourLog.info("The include pids are empty"); + // return; + } + + // Dupes will cause a crash later anyhow, but this is expensive so only do it + // when running asserts + assert new HashSet(theIncludePids).size() == theIncludePids.size() : "PID list contains duplicates: " + theIncludePids; + + Map position = new HashMap(); + for (Long next : theIncludePids) { + position.put(next, theResourceListToPopulate.size()); + theResourceListToPopulate.add(null); + } + + /* + * As always, Oracle can't handle things that other databases don't mind.. In this + * case it doesn't like more than ~1000 IDs in a single load, so we break this up + * if it's lots of IDs. I suppose maybe we should be doing this as a join anyhow + * but this should work too. Sigh. + */ + int maxLoad = 800; + List pids = new ArrayList(theIncludePids); + for (int i = 0; i < pids.size(); i += maxLoad) { + int to = i + maxLoad; + to = Math.min(to, pids.size()); + List pidsSubList = pids.subList(i, to); + doLoadPids(theResourceListToPopulate, theRevIncludedPids, theForHistoryOperation, entityManager, context, theDao, position, pidsSubList); + } + + } + /** * THIS SHOULD RETURN HASHSET and not jsut Set because we add to it later (so it can't be Collections.emptySet()) * @@ -1844,7 +1816,7 @@ public class SearchBuilder implements ISearchBuilder { } @Override - public void setType(Class theResourceType, String theResourceName) { + public void setType(Class theResourceType, String theResourceName) { myResourceType = theResourceType; myResourceName = theResourceName; } @@ -1963,13 +1935,46 @@ public class SearchBuilder implements ISearchBuilder { return thePredicates.toArray(new Predicate[thePredicates.size()]); } + private enum JoinEnum { + DATE, NUMBER, QUANTITY, REFERENCE, STRING, TOKEN, URI + + } + + private static class JoinKey { + private final JoinEnum myJoinType; + private final String myParamName; + + public JoinKey(String theParamName, JoinEnum theJoinType) { + super(); + myParamName = theParamName; + myJoinType = theJoinType; + } + + @Override + public boolean equals(Object theObj) { + JoinKey obj = (JoinKey) theObj; + return new EqualsBuilder() + .append(myParamName, obj.myParamName) + .append(myJoinType, obj.myJoinType) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder() + .append(myParamName) + .append(myJoinType) + .toHashCode(); + } + } + private final class QueryIterator implements Iterator { + private boolean myFirst = true; private Long myNext; private final Set myPidSet = new HashSet(); + private Iterator myPreResultsIterator; private Iterator myResultsIterator; private SortSpec mySort; - private Iterator myPreResultsIterator; - private boolean myFirst = true; private StopWatch myStopwatch = null; private QueryIterator() { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTableDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTableDao.java index ee21079fe49..4f4cad1946c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTableDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTableDao.java @@ -1,5 +1,7 @@ package ca.uhn.fhir.jpa.dao.data; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; /* * #%L * HAPI FHIR JPA Server @@ -19,10 +21,7 @@ package ca.uhn.fhir.jpa.dao.data; * limitations under the License. * #L% */ - -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; +import org.springframework.data.jpa.repository.*; import org.springframework.data.repository.query.Param; import ca.uhn.fhir.jpa.entity.ResourceTable; @@ -33,4 +32,7 @@ public interface IResourceTableDao extends JpaRepository { @Query("UPDATE ResourceTable r SET r.myIndexStatus = null WHERE r.myResourceType = :restype") int markResourcesOfTypeAsRequiringReindexing(@Param("restype") String theResourceType); + @Query("SELECT t.myId FROM ResourceTable t WHERE t.myIndexStatus IS NULL") + Slice findUnindexed(Pageable thePageRequest); + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java index 69f1e5f0920..578edef274d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java @@ -1,6 +1,7 @@ package ca.uhn.fhir.jpa.config; import java.util.Properties; +import java.util.concurrent.TimeUnit; import javax.persistence.EntityManagerFactory; import javax.sql.DataSource; @@ -17,6 +18,8 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor; import ca.uhn.fhir.validation.ResultSeverityEnum; +import net.ttddyy.dsproxy.listener.logging.SLF4JLogLevel; +import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder; @Configuration @EnableTransactionManagement() @@ -34,7 +37,15 @@ public class TestDstu3Config extends BaseJavaConfigDstu3 { retVal.setUrl("jdbc:derby:memory:myUnitTestDB;create=true"); retVal.setUsername(""); retVal.setPassword(""); - return retVal; + + DataSource dataSource = ProxyDataSourceBuilder + .create(retVal) + .logQueryBySlf4j(SLF4JLogLevel.INFO, "SQL") + .logSlowQueryBySlf4j(10, TimeUnit.SECONDS) + .countQuery() + .build(); + + return dataSource; } @Bean() diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java index 69f46d46fcb..152f6479d89 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java @@ -1827,6 +1827,32 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test { } + @Test + public void testSearchStringParamDoesntMatchWrongType() throws Exception { + IIdType pid1; + IIdType pid2; + { + Patient patient = new Patient(); + patient.addName().setFamily("HELLO"); + pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); + } + { + Practitioner patient = new Practitioner(); + patient.addName().setFamily("HELLO"); + pid2 = myPractitionerDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); + } + + SearchParameterMap params; + List patients; + + params = new SearchParameterMap(); + params.add(Patient.SP_FAMILY, new StringParam("HELLO")); + patients = toUnqualifiedVersionlessIds(myPatientDao.search(params)); + assertThat(patients, containsInAnyOrder(pid1)); + assertThat(patients, not(containsInAnyOrder(pid2))); + } + + @Test public void testSearchStringParam() throws Exception { IIdType pid1; @@ -3251,11 +3277,26 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test { SearchParameterMap map; List ids; + // No search param map = new SearchParameterMap(); map.setSort(new SortSpec("family", SortOrderEnum.ASC).setChain(new SortSpec("given", SortOrderEnum.ASC))); ids = toUnqualifiedVersionlessIdValues(myPatientDao.search(map)); assertThat(ids, contains("Patient/AA", "Patient/AB", "Patient/BA", "Patient/BB")); + // Same SP as sort + map = new SearchParameterMap(); + map.add(Patient.SP_ACTIVE, new TokenParam(null, "true")); + map.setSort(new SortSpec("family", SortOrderEnum.ASC).setChain(new SortSpec("given", SortOrderEnum.ASC))); + ids = toUnqualifiedVersionlessIdValues(myPatientDao.search(map)); + assertThat(ids, contains("Patient/AA", "Patient/AB", "Patient/BA", "Patient/BB")); + + // Different SP from sort + map = new SearchParameterMap(); + map.add(Patient.SP_GENDER, new TokenParam(null, "male")); + map.setSort(new SortSpec("family", SortOrderEnum.ASC).setChain(new SortSpec("given", SortOrderEnum.ASC))); + ids = toUnqualifiedVersionlessIdValues(myPatientDao.search(map)); + assertThat(ids, contains("Patient/AA", "Patient/AB", "Patient/BA", "Patient/BB")); + map = new SearchParameterMap(); map.setSort(new SortSpec("gender").setChain(new SortSpec("family", SortOrderEnum.ASC).setChain(new SortSpec("given", SortOrderEnum.ASC)))); ids = toUnqualifiedVersionlessIdValues(myPatientDao.search(map)); diff --git a/pom.xml b/pom.xml index 16dc608f397..9d8cb8cd28d 100644 --- a/pom.xml +++ b/pom.xml @@ -538,6 +538,11 @@ Saxon-HE 9.5.1-5 + + net.ttddyy + datasource-proxy + 1.4.1 + org.apache.commons commons-dbcp2 diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 7dc713b157d..93ce40e6fa1 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -24,6 +24,10 @@ looked like before the update. This change was made to support the change above, but seems like a useful feature all around. + + Optimize queries in JPA server remove a few redundant select columns when performing + searches. This provides a slight speed increase in some cases. +