honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimize queries in JPA

This commit is contained in:
James Agnew 2017-06-14 08:35:41 -04:00
parent c2e5fa3f18
commit 69849dd3c5
8 changed files with 273 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hapi-fhir-jpaserver-base/pom.xml
View File

@ -100,6 +100,11 @@
<artifactId>thymeleaf</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.ttddyy</groupId>
<artifactId>datasource-proxy</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.javassist</groupId>

33
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java
View File

@ -33,6 +33,7 @@ import javax.persistence.criteria.Root;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.annotation.Propagation;
@ -40,8 +41,7 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionTemplate;
import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
import ca.uhn.fhir.jpa.dao.data.*;
import ca.uhn.fhir.jpa.entity.ForcedId;
import ca.uhn.fhir.jpa.entity.ResourceTable;
import ca.uhn.fhir.jpa.util.ReindexFailureException;
@ -87,30 +87,33 @@ public abstract class BaseHapiFhirSystemDao<T, MT> extends BaseHapiFhirDao<IBase
return retVal;
}
@Autowired
private IResourceTableDao myResourceTableDao;
private int doPerformReindexingPassForResources(final Integer theCount, TransactionTemplate txTemplate) {
return txTemplate.execute(new TransactionCallback<Integer>() {
@SuppressWarnings("unchecked")
@Override
public Integer doInTransaction(TransactionStatus theStatus) {
TypedQuery<ResourceTable> q = myEntityManager.createQuery("SELECT t FROM " + ResourceTable.class.getSimpleName() + " t WHERE t.myIndexStatus IS null", ResourceTable.class);
int maxResult = 500;
if (theCount != null) {
maxResult = Math.min(theCount, 2000);
}
maxResult = Math.max(maxResult, 10);
TypedQuery<Long> q = myEntityManager.createQuery("SELECT t.myId FROM ResourceTable t WHERE t.myIndexStatus IS NULL", Long.class);
ourLog.info("Beginning indexing query with maximum {}", maxResult);
q.setMaxResults(maxResult);
List<ResourceTable> resources = q.getResultList();
if (resources.isEmpty()) {
return 0;
}
ourLog.info("Indexing {} resources", resources.size());
Collection<Long> resources = q.getResultList();
int count = 0;
long start = System.currentTimeMillis();
for (ResourceTable resourceTable : resources) {
for (Long nextId : resources) {
ResourceTable resourceTable = myResourceTableDao.findOne(nextId);
try {
/*
* This part is because from HAPI 1.5 - 1.6 we changed the format of forced ID to be "type/id" instead of just "id"
@ -135,13 +138,17 @@ public abstract class BaseHapiFhirSystemDao<T, MT> extends BaseHapiFhirDao<IBase
throw new ReindexFailureException(resourceTable.getId());
}
count++;
if (count >= maxResult) {
break;
}
}
long delay = System.currentTimeMillis() - start;
long avg = (delay / resources.size());
ourLog.info("Indexed {} / {} resources in {}ms - Avg {}ms / resource", new Object[] { count, resources.size(), delay, avg });
long avg = count > 0 ? (delay / count) : 0;
ourLog.info("Indexed {} resources in {}ms - Avg {}ms / resource", new Object[] { count, delay, avg });
return resources.size();
return count;
}
});
}

355
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java
View File

@ -27,90 +27,31 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
import java.math.BigDecimal;
import java.math.MathContext;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.Map.Entry;
import java.util.Set;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.AbstractQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.*;
import javax.persistence.criteria.CriteriaBuilder.In;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.From;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import javax.persistence.criteria.Order;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.persistence.criteria.Subquery;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.*;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.tuple.Pair;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.instance.model.api.*;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.collect.*;
import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeDeclaredChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
import ca.uhn.fhir.context.ConfigurationException;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.FhirVersionEnum;
import ca.uhn.fhir.context.RuntimeChildChoiceDefinition;
import ca.uhn.fhir.context.RuntimeChildResourceDefinition;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao;
import ca.uhn.fhir.jpa.dao.IDao;
import ca.uhn.fhir.jpa.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
import ca.uhn.fhir.jpa.dao.ISearchBuilder;
import ca.uhn.fhir.jpa.dao.ISearchParamRegistry;
import ca.uhn.fhir.context.*;
import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao;
import ca.uhn.fhir.jpa.entity.BaseHasResource;
import ca.uhn.fhir.jpa.entity.BaseResourceIndexedSearchParam;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamDate;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamNumber;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamQuantity;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamString;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamToken;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamUri;
import ca.uhn.fhir.jpa.entity.ResourceLink;
import ca.uhn.fhir.jpa.entity.ResourceTable;
import ca.uhn.fhir.jpa.entity.ResourceTag;
import ca.uhn.fhir.jpa.entity.SearchParam;
import ca.uhn.fhir.jpa.entity.SearchParamPresent;
import ca.uhn.fhir.jpa.entity.TagDefinition;
import ca.uhn.fhir.jpa.entity.TagTypeEnum;
import ca.uhn.fhir.jpa.entity.*;
import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
import ca.uhn.fhir.jpa.term.VersionIndependentConcept;
import ca.uhn.fhir.jpa.util.StopWatch;
import ca.uhn.fhir.model.api.IPrimitiveDatatype;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.api.IResource;
import ca.uhn.fhir.model.api.Include;
import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
import ca.uhn.fhir.model.base.composite.BaseCodingDt;
import ca.uhn.fhir.model.base.composite.BaseIdentifierDt;
import ca.uhn.fhir.model.base.composite.BaseQuantityDt;
import ca.uhn.fhir.model.api.*;
import ca.uhn.fhir.model.base.composite.*;
import ca.uhn.fhir.model.dstu.resource.BaseResource;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.model.primitive.InstantDt;
@ -119,23 +60,9 @@ import ca.uhn.fhir.parser.DataFormatException;
import ca.uhn.fhir.rest.api.SortOrderEnum;
import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.method.RestSearchParameterTypeEnum;
import ca.uhn.fhir.rest.param.CompositeParam;
import ca.uhn.fhir.rest.param.DateParam;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.HasParam;
import ca.uhn.fhir.rest.param.NumberParam;
import ca.uhn.fhir.rest.param.ParamPrefixEnum;
import ca.uhn.fhir.rest.param.QuantityParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.param.TokenParamModifier;
import ca.uhn.fhir.rest.param.UriParam;
import ca.uhn.fhir.rest.param.UriParamQualifierEnum;
import ca.uhn.fhir.rest.param.*;
import ca.uhn.fhir.rest.server.Constants;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.exceptions.*;
import ca.uhn.fhir.util.UrlUtil;
/**
@ -145,12 +72,14 @@ import ca.uhn.fhir.util.UrlUtil;
public class SearchBuilder implements ISearchBuilder {
private static Long NO_MORE = Long.valueOf(-1);
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchBuilder.class);
private List<Long> myAlsoIncludePids;
private CriteriaBuilder myBuilder;
private BaseHapiFhirDao<?> myCallingDao;
private FhirContext myContext;
private EntityManager myEntityManager;
private IForcedIdDao myForcedIdDao;
private IFulltextSearchSvc myFulltextSearchSvc;
private Map<JoinKey, Join<?, ?>> myIndexJoins = Maps.newHashMap();
private SearchParameterMap myParams;
private ArrayList<Predicate> myPredicates;
private IResourceIndexedSearchParamUriDao myResourceIndexedSearchParamUriDao;
@ -159,8 +88,8 @@ public class SearchBuilder implements ISearchBuilder {
private Root<ResourceTable> myResourceTableRoot;
private Class<? extends IBaseResource> myResourceType;
private ISearchParamRegistry mySearchParamRegistry;
private IHapiTerminologySvc myTerminologySvc;
private String mySearchUuid;
private IHapiTerminologySvc myTerminologySvc;
/**
* Constructor
@ -199,7 +128,7 @@ public class SearchBuilder implements ISearchBuilder {
private void addPredicateDate(String theResourceName, String theParamName, List<? extends IQueryParameterType> theList) {
Join<ResourceTable, ResourceIndexedSearchParamDate> join = myResourceTableRoot.join("myParamsDate", JoinType.LEFT);
Join<ResourceTable, ResourceIndexedSearchParamDate> join = createOrReuseJoin(JoinEnum.DATE, theParamName);
if (theList.get(0).getMissing() != null) {
Boolean missing = theList.get(0).getMissing();
@ -275,28 +204,6 @@ public class SearchBuilder implements ISearchBuilder {
}
}
// private void addPredicateId(Set<Long> thePids) {
// if (thePids == null || thePids.isEmpty()) {
// return;
// }
//
// CriteriaBuilder builder = myEntityManager.getCriteriaBuilder();
// CriteriaQuery<Long> cq = builder.createQuery(Long.class);
// Root<ResourceTable> from = cq.from(ResourceTable.class);
// cq.select(from.get("myId").as(Long.class));
//
// List<Predicate> predicates = new ArrayList<Predicate>();
// predicates.add(builder.equal(from.get("myResourceType"), myResourceName));
// predicates.add(from.get("myId").in(thePids));
// createPredicateResourceId(builder, cq, predicates, from.get("myId").as(Long.class));
// createPredicateLastUpdatedForResourceTable(builder, from, predicates);
//
// cq.where(toArray(predicates));
//
// TypedQuery<Long> q = myEntityManager.createQuery(cq);
// doSetPids(q.getResultList());
// }
private void addPredicateLanguage(List<List<? extends IQueryParameterType>> theList) {
for (List<? extends IQueryParameterType> nextList : theList) {
@ -326,7 +233,7 @@ public class SearchBuilder implements ISearchBuilder {
private void addPredicateNumber(String theResourceName, String theParamName, List<? extends IQueryParameterType> theList) {
Join<ResourceTable, ResourceIndexedSearchParamNumber> join = myResourceTableRoot.join("myParamsNumber", JoinType.LEFT);
Join<ResourceTable, ResourceIndexedSearchParamNumber> join = createOrReuseJoin(JoinEnum.NUMBER, theParamName);
if (theList.get(0).getMissing() != null) {
addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join);
@ -378,7 +285,7 @@ public class SearchBuilder implements ISearchBuilder {
}
private void addPredicateQuantity(String theResourceName, String theParamName, List<? extends IQueryParameterType> theList) {
Join<ResourceTable, ResourceIndexedSearchParamQuantity> join = myResourceTableRoot.join("myParamsQuantity", JoinType.LEFT);
Join<ResourceTable, ResourceIndexedSearchParamQuantity> join = createOrReuseJoin(JoinEnum.QUANTITY, theParamName);
if (theList.get(0).getMissing() != null) {
addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join);
@ -406,7 +313,7 @@ public class SearchBuilder implements ISearchBuilder {
return;
}
Join<ResourceTable, ResourceLink> join = myResourceTableRoot.join("myResourceLinks", JoinType.LEFT);
Join<ResourceTable, ResourceLink> join = createOrReuseJoin(JoinEnum.REFERENCE, theParamName);
List<Predicate> codePredicates = new ArrayList<Predicate>();
@ -466,7 +373,7 @@ public class SearchBuilder implements ISearchBuilder {
RuntimeResourceDefinition resourceDef = myContext.getResourceDefinition(theResourceName);
RuntimeSearchParam searchParamByName = myCallingDao.getSearchParamByName(resourceDef, theParamName);
if (searchParamByName == null) {
throw new InternalErrorException("Could not find parameter " + theParamName );
throw new InternalErrorException("Could not find parameter " + theParamName);
}
String paramPath = searchParamByName.getPath();
if (paramPath.endsWith(".as(Reference)")) {
@ -496,7 +403,7 @@ public class SearchBuilder implements ISearchBuilder {
if (resourceTypes.isEmpty()) {
for (BaseRuntimeElementDefinition<?> next : myContext.getElementDefinitions()) {
if (next instanceof RuntimeResourceDefinition) {
RuntimeResourceDefinition nextResDef = (RuntimeResourceDefinition)next;
RuntimeResourceDefinition nextResDef = (RuntimeResourceDefinition) next;
resourceTypes.add(nextResDef.getImplementingClass());
}
}
@ -582,8 +489,10 @@ public class SearchBuilder implements ISearchBuilder {
*/
Root<ResourceTable> stackRoot = myResourceTableRoot;
ArrayList<Predicate> stackPredicates = myPredicates;
Map<JoinKey, Join<?, ?>> stackIndexJoins = myIndexJoins;
myResourceTableRoot = subQfrom;
myPredicates = new ArrayList<Predicate>();
myPredicates = Lists.newArrayList();
myIndexJoins = Maps.newHashMap();
// Create the subquery predicates
myPredicates.add(myBuilder.equal(myResourceTableRoot.get("myResourceType"), subResourceName));
@ -597,6 +506,7 @@ public class SearchBuilder implements ISearchBuilder {
*/
myResourceTableRoot = stackRoot;
myPredicates = stackPredicates;
myIndexJoins = stackIndexJoins;
Predicate pathPredicate = createResourceLinkPathPredicate(theResourceName, theParamName, join);
Predicate pidPredicate = join.get("myTargetResourcePid").in(subQ);
@ -661,7 +571,7 @@ public class SearchBuilder implements ISearchBuilder {
private void addPredicateString(String theResourceName, String theParamName, List<? extends IQueryParameterType> theList) {
Join<ResourceTable, ResourceIndexedSearchParamString> join = myResourceTableRoot.join("myParamsString", JoinType.LEFT);
Join<ResourceTable, ResourceIndexedSearchParamString> join = createOrReuseJoin(JoinEnum.STRING, theParamName);
if (theList.get(0).getMissing() != null) {
addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join);
@ -809,7 +719,7 @@ public class SearchBuilder implements ISearchBuilder {
private void addPredicateToken(String theResourceName, String theParamName, List<? extends IQueryParameterType> theList) {
Join<ResourceTable, ResourceIndexedSearchParamToken> join = myResourceTableRoot.join("myParamsToken", JoinType.LEFT);
Join<ResourceTable, ResourceIndexedSearchParamToken> join = createOrReuseJoin(JoinEnum.TOKEN, theParamName);
if (theList.get(0).getMissing() != null) {
addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join);
@ -841,7 +751,7 @@ public class SearchBuilder implements ISearchBuilder {
private void addPredicateUri(String theResourceName, String theParamName, List<? extends IQueryParameterType> theList) {
Join<ResourceTable, ResourceIndexedSearchParamUri> join = myResourceTableRoot.join("myParamsUri", JoinType.LEFT);
Join<ResourceTable, ResourceIndexedSearchParamUri> join = createOrReuseJoin(JoinEnum.URI, theParamName);
if (theList.get(0).getMissing() != null) {
addPredicateParamMissing(theResourceName, theParamName, theList.get(0).getMissing(), join);
@ -966,6 +876,42 @@ public class SearchBuilder implements ISearchBuilder {
return retVal;
}
@SuppressWarnings("unchecked")
private <T> Join<ResourceTable, T> createOrReuseJoin(JoinEnum theType, String theSearchParameterName) {
Join<ResourceTable, ResourceIndexedSearchParamDate> join = null;
switch (theType) {
case DATE:
join = myResourceTableRoot.join("myParamsDate", JoinType.LEFT);
break;
case NUMBER:
join = myResourceTableRoot.join("myParamsNumber", JoinType.LEFT);
break;
case QUANTITY:
join = myResourceTableRoot.join("myParamsQuantity", JoinType.LEFT);
break;
case REFERENCE:
join = myResourceTableRoot.join("myResourceLinks", JoinType.LEFT);
break;
case STRING:
join = myResourceTableRoot.join("myParamsString", JoinType.LEFT);
break;
case URI:
join = myResourceTableRoot.join("myParamsUri", JoinType.LEFT);
break;
case TOKEN:
join = myResourceTableRoot.join("myParamsToken", JoinType.LEFT);
break;
}
JoinKey key = new JoinKey(theSearchParameterName, theType);
if (!myIndexJoins.containsKey(key)) {
myIndexJoins.put(key, join);
}
return (Join<ResourceTable, T>) join;
}
private Predicate createPredicateDate(IQueryParameterType theParam, String theResourceName, String theParamName, CriteriaBuilder theBuilder, From<?, ResourceIndexedSearchParamDate> theFrom) {
Predicate p;
if (theParam instanceof DateParam) {
@ -1245,10 +1191,10 @@ public class SearchBuilder implements ISearchBuilder {
// Use "in" in case of large numbers of codes due to param modifiers
final Path<String> systemExpression = theFrom.get("mySystem");
final Path<String> valueExpression = theFrom.get("myValue");
for (Map.Entry<String, List<VersionIndependentConcept>> entry: map.entrySet()) {
for (Map.Entry<String, List<VersionIndependentConcept>> entry : map.entrySet()) {
Predicate systemPredicate = theBuilder.equal(systemExpression, entry.getKey());
In<String> codePredicate = theBuilder.in(valueExpression);
for (VersionIndependentConcept nextCode: entry.getValue()) {
for (VersionIndependentConcept nextCode : entry.getValue()) {
codePredicate.value(nextCode.getCode());
}
orPredicates.add(theBuilder.and(systemPredicate, codePredicate));
@ -1298,8 +1244,6 @@ public class SearchBuilder implements ISearchBuilder {
return new QueryIterator();
}
private List<Long> myAlsoIncludePids;
private TypedQuery<Long> createQuery(SortSpec sort) {
CriteriaQuery<Long> outerQuery;
/*
@ -1344,16 +1288,7 @@ public class SearchBuilder implements ISearchBuilder {
}
myResourceTableQuery.distinct(true);
myPredicates = new ArrayList<Predicate>();
if (myParams.getEverythingMode() == null) {
myPredicates.add(myBuilder.equal(myResourceTableRoot.get("myResourceType"), myResourceName));
}
myPredicates.add(myBuilder.isNull(myResourceTableRoot.get("myDeleted")));
DateRangeParam lu = myParams.getLastUpdated();
List<Predicate> lastUpdatedPredicates = createLastUpdatedPredicates(lu, myBuilder, myResourceTableRoot);
myPredicates.addAll(lastUpdatedPredicates);
if (myParams.getEverythingMode() != null) {
Join<ResourceTable, ResourceLink> join = myResourceTableRoot.join("myResourceLinks", JoinType.LEFT);
@ -1397,6 +1332,25 @@ public class SearchBuilder implements ISearchBuilder {
myPredicates.add(myResourceTableRoot.get("myId").as(Long.class).in(pids));
}
/*
* Add a predicate to make sure we only include non-deleted resources, and only include
* resources of the right type.
*
* If we have any joins to index tables, we get this behaviour already guaranteed so we don't
* need an explicit predicate for it.
*/
if (myIndexJoins.isEmpty()) {
if (myParams.getEverythingMode() == null) {
myPredicates.add(myBuilder.equal(myResourceTableRoot.get("myResourceType"), myResourceName));
}
myPredicates.add(myBuilder.isNull(myResourceTableRoot.get("myDeleted")));
}
// Last updated
DateRangeParam lu = myParams.getLastUpdated();
List<Predicate> lastUpdatedPredicates = createLastUpdatedPredicates(lu, myBuilder, myResourceTableRoot);
myPredicates.addAll(lastUpdatedPredicates);
myResourceTableQuery.where(myBuilder.and(SearchBuilder.toArray(myPredicates)));
/*
@ -1450,47 +1404,65 @@ public class SearchBuilder implements ISearchBuilder {
String joinAttrName;
String[] sortAttrName;
JoinEnum joinType;
switch (param.getParamType()) {
case STRING:
joinAttrName = "myParamsString";
sortAttrName = new String[] { "myValueExact" };
joinType = JoinEnum.STRING;
break;
case DATE:
joinAttrName = "myParamsDate";
sortAttrName = new String[] { "myValueLow" };
joinType = JoinEnum.DATE;
break;
case REFERENCE:
joinAttrName = "myResourceLinks";
sortAttrName = new String[] { "myTargetResourcePid" };
joinType = JoinEnum.REFERENCE;
break;
case TOKEN:
joinAttrName = "myParamsToken";
sortAttrName = new String[] { "mySystem", "myValue" };
joinType = JoinEnum.TOKEN;
break;
case NUMBER:
joinAttrName = "myParamsNumber";
sortAttrName = new String[] { "myValue" };
joinType = JoinEnum.NUMBER;
break;
case URI:
joinAttrName = "myParamsUri";
sortAttrName = new String[] { "myUri" };
joinType = JoinEnum.URI;
break;
case QUANTITY:
joinAttrName = "myParamsQuantity";
sortAttrName = new String[] { "myValue" };
joinType = JoinEnum.QUANTITY;
break;
default:
throw new InvalidRequestException("This server does not support _sort specifications of type " + param.getParamType() + " - Can't serve _sort=" + theSort.getParamName());
}
From<?, ?> join = theFrom.join(joinAttrName, JoinType.LEFT);
/*
* If we've already got a join for the specific parameter we're
* sorting on, we'll also sort with it. Otherwise we need a new join.
*/
JoinKey key = new JoinKey(theSort.getParamName(), joinType);
Join<?, ?> join = myIndexJoins.get(key);
if (join == null) {
join = theFrom.join(joinAttrName, JoinType.LEFT);
if (param.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
thePredicates.add(join.get("mySourcePath").as(String.class).in(param.getPathsSplit()));
if (param.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
thePredicates.add(join.get("mySourcePath").as(String.class).in(param.getPathsSplit()));
} else {
Predicate joinParam1 = theBuilder.equal(join.get("myParamName"), theSort.getParamName());
thePredicates.add(joinParam1);
}
} else {
Predicate joinParam1 = theBuilder.equal(join.get("myParamName"), theSort.getParamName());
thePredicates.add(joinParam1);
ourLog.info("Reusing join for {}", theSort.getParamName());
}
for (String next : sortAttrName) {
@ -1536,41 +1508,6 @@ public class SearchBuilder implements ISearchBuilder {
return retVal;
}
@Override
public void loadResourcesByPid(Collection<Long> theIncludePids, List<IBaseResource> theResourceListToPopulate, Set<Long> theRevIncludedPids, boolean theForHistoryOperation,
EntityManager entityManager, FhirContext context, IDao theDao) {
if (theIncludePids.isEmpty()) {
ourLog.info("The include pids are empty");
//return;
}
// Dupes will cause a crash later anyhow, but this is expensive so only do it
// when running asserts
assert new HashSet<Long>(theIncludePids).size() == theIncludePids.size() : "PID list contains duplicates: " + theIncludePids;
Map<Long, Integer> position = new HashMap<Long, Integer>();
for (Long next : theIncludePids) {
position.put(next, theResourceListToPopulate.size());
theResourceListToPopulate.add(null);
}
/*
* As always, Oracle can't handle things that other databases don't mind.. In this
* case it doesn't like more than ~1000 IDs in a single load, so we break this up
* if it's lots of IDs. I suppose maybe we should be doing this as a join anyhow
* but this should work too. Sigh.
*/
int maxLoad = 800;
List<Long> pids = new ArrayList<Long>(theIncludePids);
for (int i = 0; i < pids.size(); i += maxLoad) {
int to = i + maxLoad;
to = Math.min(to, pids.size());
List<Long> pidsSubList = pids.subList(i, to);
doLoadPids(theResourceListToPopulate, theRevIncludedPids, theForHistoryOperation, entityManager, context, theDao, position, pidsSubList);
}
}
private void doLoadPids(List<IBaseResource> theResourceListToPopulate, Set<Long> theRevIncludedPids, boolean theForHistoryOperation, EntityManager entityManager, FhirContext context, IDao theDao,
Map<Long, Integer> position, Collection<Long> pids) {
CriteriaBuilder builder = entityManager.getCriteriaBuilder();
@ -1606,6 +1543,41 @@ public class SearchBuilder implements ISearchBuilder {
}
}
@Override
public void loadResourcesByPid(Collection<Long> theIncludePids, List<IBaseResource> theResourceListToPopulate, Set<Long> theRevIncludedPids, boolean theForHistoryOperation,
EntityManager entityManager, FhirContext context, IDao theDao) {
if (theIncludePids.isEmpty()) {
ourLog.info("The include pids are empty");
// return;
}
// Dupes will cause a crash later anyhow, but this is expensive so only do it
// when running asserts
assert new HashSet<Long>(theIncludePids).size() == theIncludePids.size() : "PID list contains duplicates: " + theIncludePids;
Map<Long, Integer> position = new HashMap<Long, Integer>();
for (Long next : theIncludePids) {
position.put(next, theResourceListToPopulate.size());
theResourceListToPopulate.add(null);
}
/*
* As always, Oracle can't handle things that other databases don't mind.. In this
* case it doesn't like more than ~1000 IDs in a single load, so we break this up
* if it's lots of IDs. I suppose maybe we should be doing this as a join anyhow
* but this should work too. Sigh.
*/
int maxLoad = 800;
List<Long> pids = new ArrayList<Long>(theIncludePids);
for (int i = 0; i < pids.size(); i += maxLoad) {
int to = i + maxLoad;
to = Math.min(to, pids.size());
List<Long> pidsSubList = pids.subList(i, to);
doLoadPids(theResourceListToPopulate, theRevIncludedPids, theForHistoryOperation, entityManager, context, theDao, position, pidsSubList);
}
}
/**
* THIS SHOULD RETURN HASHSET and not jsut Set because we add to it later (so it can't be Collections.emptySet())
*
@ -1844,7 +1816,7 @@ public class SearchBuilder implements ISearchBuilder {
}
@Override
public void setType(Class<? extends IBaseResource> theResourceType, String theResourceName) {
public void setType(Class<? extends IBaseResource> theResourceType, String theResourceName) {
myResourceType = theResourceType;
myResourceName = theResourceName;
}
@ -1963,13 +1935,46 @@ public class SearchBuilder implements ISearchBuilder {
return thePredicates.toArray(new Predicate[thePredicates.size()]);
}
private enum JoinEnum {
DATE, NUMBER, QUANTITY, REFERENCE, STRING, TOKEN, URI
}
private static class JoinKey {
private final JoinEnum myJoinType;
private final String myParamName;
public JoinKey(String theParamName, JoinEnum theJoinType) {
super();
myParamName = theParamName;
myJoinType = theJoinType;
}
@Override
public boolean equals(Object theObj) {
JoinKey obj = (JoinKey) theObj;
return new EqualsBuilder()
.append(myParamName, obj.myParamName)
.append(myJoinType, obj.myJoinType)
.isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder()
.append(myParamName)
.append(myJoinType)
.toHashCode();
}
}
private final class QueryIterator implements Iterator<Long> {
private boolean myFirst = true;
private Long myNext;
private final Set<Long> myPidSet = new HashSet<Long>();
private Iterator<Long> myPreResultsIterator;
private Iterator<Long> myResultsIterator;
private SortSpec mySort;
private Iterator<Long> myPreResultsIterator;
private boolean myFirst = true;
private StopWatch myStopwatch = null;
private QueryIterator() {

10
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTableDao.java
View File

@ -1,5 +1,7 @@
package ca.uhn.fhir.jpa.dao.data;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
/*
* #%L
* HAPI FHIR JPA Server
@ -19,10 +21,7 @@ package ca.uhn.fhir.jpa.dao.data;
* limitations under the License.
* #L%
*/
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.jpa.repository.*;
import org.springframework.data.repository.query.Param;
import ca.uhn.fhir.jpa.entity.ResourceTable;
@ -33,4 +32,7 @@ public interface IResourceTableDao extends JpaRepository<ResourceTable, Long> {
@Query("UPDATE ResourceTable r SET r.myIndexStatus = null WHERE r.myResourceType = :restype")
int markResourcesOfTypeAsRequiringReindexing(@Param("restype") String theResourceType);
@Query("SELECT t.myId FROM ResourceTable t WHERE t.myIndexStatus IS NULL")
Slice<Long> findUnindexed(Pageable thePageRequest);
}

13
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java
View File

@ -1,6 +1,7 @@
package ca.uhn.fhir.jpa.config;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
@ -17,6 +18,8 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor;
import ca.uhn.fhir.validation.ResultSeverityEnum;
import net.ttddyy.dsproxy.listener.logging.SLF4JLogLevel;
import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder;
@Configuration
@EnableTransactionManagement()
@ -34,7 +37,15 @@ public class TestDstu3Config extends BaseJavaConfigDstu3 {
retVal.setUrl("jdbc:derby:memory:myUnitTestDB;create=true");
retVal.setUsername("");
retVal.setPassword("");
return retVal;
DataSource dataSource = ProxyDataSourceBuilder
.create(retVal)
.logQueryBySlf4j(SLF4JLogLevel.INFO, "SQL")
.logSlowQueryBySlf4j(10, TimeUnit.SECONDS)
.countQuery()
.build();
return dataSource;
}
@Bean()

41
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java
View File

@ -1827,6 +1827,32 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test {
}
@Test
public void testSearchStringParamDoesntMatchWrongType() throws Exception {
IIdType pid1;
IIdType pid2;
{
Patient patient = new Patient();
patient.addName().setFamily("HELLO");
pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
}
{
Practitioner patient = new Practitioner();
patient.addName().setFamily("HELLO");
pid2 = myPractitionerDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
}
SearchParameterMap params;
List<IIdType> patients;
params = new SearchParameterMap();
params.add(Patient.SP_FAMILY, new StringParam("HELLO"));
patients = toUnqualifiedVersionlessIds(myPatientDao.search(params));
assertThat(patients, containsInAnyOrder(pid1));
assertThat(patients, not(containsInAnyOrder(pid2)));
}
@Test
public void testSearchStringParam() throws Exception {
IIdType pid1;
@ -3251,11 +3277,26 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test {
SearchParameterMap map;
List<String> ids;
// No search param
map = new SearchParameterMap();
map.setSort(new SortSpec("family", SortOrderEnum.ASC).setChain(new SortSpec("given", SortOrderEnum.ASC)));
ids = toUnqualifiedVersionlessIdValues(myPatientDao.search(map));
assertThat(ids, contains("Patient/AA", "Patient/AB", "Patient/BA", "Patient/BB"));
// Same SP as sort
map = new SearchParameterMap();
map.add(Patient.SP_ACTIVE, new TokenParam(null, "true"));
map.setSort(new SortSpec("family", SortOrderEnum.ASC).setChain(new SortSpec("given", SortOrderEnum.ASC)));
ids = toUnqualifiedVersionlessIdValues(myPatientDao.search(map));
assertThat(ids, contains("Patient/AA", "Patient/AB", "Patient/BA", "Patient/BB"));
// Different SP from sort
map = new SearchParameterMap();
map.add(Patient.SP_GENDER, new TokenParam(null, "male"));
map.setSort(new SortSpec("family", SortOrderEnum.ASC).setChain(new SortSpec("given", SortOrderEnum.ASC)));
ids = toUnqualifiedVersionlessIdValues(myPatientDao.search(map));
assertThat(ids, contains("Patient/AA", "Patient/AB", "Patient/BA", "Patient/BB"));
map = new SearchParameterMap();
map.setSort(new SortSpec("gender").setChain(new SortSpec("family", SortOrderEnum.ASC).setChain(new SortSpec("given", SortOrderEnum.ASC))));
ids = toUnqualifiedVersionlessIdValues(myPatientDao.search(map));

5
pom.xml
View File

@ -538,6 +538,11 @@
<artifactId>Saxon-HE</artifactId>
<version>9.5.1-5</version>
</dependency>
<dependency>
<groupId>net.ttddyy</groupId>
<artifactId>datasource-proxy</artifactId>
<version>1.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>

4
src/changes/changes.xml
View File

@ -24,6 +24,10 @@
looked like before the update. This change was made to support the change above, but
seems like a useful feature all around.
</action>
<action type="add">
Optimize queries in JPA server remove a few redundant select columns when performing
searches. This provides a slight speed increase in some cases.
</action>
</release>
<release version="2.5" date="2017-06-08">
<action type="fix">