honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#963 Improved performance for tag loading

This commit is contained in:
frankjtao 2018-05-31 21:30:46 -04:00
parent 404d37cedc
commit 73f12744ae
5 changed files with 208 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java
View File

@ -1156,7 +1156,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
public SearchBuilder newSearchBuilder() {
SearchBuilder builder = new SearchBuilder(getContext(), myEntityManager, myFulltextSearchSvc, this, myResourceIndexedSearchParamUriDao,
myForcedIdDao,
myTerminologySvc, mySerarchParamRegistry, myResourceHistoryTableDao);
myTerminologySvc, mySerarchParamRegistry, myResourceHistoryTableDao, myResourceTagDao);
return builder;
}
@ -1337,7 +1337,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
}
@SuppressWarnings("unchecked")
private <R extends IBaseResource> R populateResourceMetadataHapi(Class<R> theResourceType, BaseHasResource theEntity, boolean theForHistoryOperation, IResource res) {
private <R extends IBaseResource> R populateResourceMetadataHapi(Class<R> theResourceType, BaseHasResource theEntity, Collection<? extends BaseTag> myTagList, boolean theForHistoryOperation, IResource res) {
R retVal = (R) res;
if (theEntity.getDeleted() != null) {
res = (IResource) myContext.getResourceDefinition(theResourceType).newInstance();
@ -1366,7 +1366,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
ResourceMetadataKeyEnum.UPDATED.put(res, theEntity.getUpdated());
IDao.RESOURCE_PID.put(res, theEntity.getId());
Collection<? extends BaseTag> tags = theEntity.getTags();
Collection<? extends BaseTag> tags = myTagList;
if (theEntity.isHasTags()) {
TagList tagList = new TagList();
List<IBaseCoding> securityLabels = new ArrayList<>();
@ -1403,7 +1403,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
}
@SuppressWarnings("unchecked")
private <R extends IBaseResource> R populateResourceMetadataRi(Class<R> theResourceType, BaseHasResource theEntity, boolean theForHistoryOperation, IAnyResource res) {
private <R extends IBaseResource> R populateResourceMetadataRi(Class<R> theResourceType, BaseHasResource theEntity, Collection<? extends BaseTag> myTagList, boolean theForHistoryOperation, IAnyResource res) {
R retVal = (R) res;
if (theEntity.getDeleted() != null) {
res = (IAnyResource) myContext.getResourceDefinition(theResourceType).newInstance();
@ -1436,7 +1436,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
res.getMeta().setLastUpdated(theEntity.getUpdatedDate());
IDao.RESOURCE_PID.put(res, theEntity.getId());
Collection<? extends BaseTag> tags = theEntity.getTags();
Collection<? extends BaseTag> tags = myTagList;
if (theEntity.isHasTags()) {
for (BaseTag next : tags) {
@ -1583,12 +1583,12 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
public IBaseResource toResource(BaseHasResource theEntity, boolean theForHistoryOperation) {
RuntimeResourceDefinition type = myContext.getResourceDefinition(theEntity.getResourceType());
Class<? extends IBaseResource> resourceType = type.getImplementingClass();
return toResource(resourceType, theEntity, null, theForHistoryOperation);
return toResource(resourceType, theEntity, null, null, theForHistoryOperation);
}
@SuppressWarnings("unchecked")
@Override
public <R extends IBaseResource> R toResource(Class<R> theResourceType, BaseHasResource theEntity, Collection<ResourceHistoryTable> historyList,
public <R extends IBaseResource> R toResource(Class<R> theResourceType, BaseHasResource theEntity, ResourceHistoryTable myHistory, Collection<ResourceTag> tagList,
boolean theForHistoryOperation) {
// May 28, 2018 - #936
@ -1597,10 +1597,10 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
if (theEntity instanceof ResourceHistoryTable) {
history = (ResourceHistoryTable) theEntity;
} else {
if (historyList == null) {
if (myHistory == null) {
history = myResourceHistoryTableDao.findForIdAndVersion(theEntity.getId(), theEntity.getVersion());
} else {
history = getHistory(historyList, theEntity.getId(), theEntity.getVersion());
history = myHistory;
}
}
@ -1627,12 +1627,21 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
break;
}
// get preload the tagList
Collection<? extends BaseTag> myTagList;
if (tagList == null)
myTagList = theEntity.getTags();
else
myTagList = tagList;
/*
* Use the appropriate custom type if one is specified in the context
*/
Class<R> resourceType = theResourceType;
if (myContext.hasDefaultTypeForProfile()) {
for (BaseTag nextTag : theEntity.getTags()) {
for (BaseTag nextTag : myTagList) {
if (nextTag.getTag().getTagType() == TagTypeEnum.PROFILE) {
String profile = nextTag.getTag().getCode();
if (isNotBlank(profile)) {
@ -1679,10 +1688,10 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
if (retVal instanceof IResource) {
IResource res = (IResource) retVal;
retVal = populateResourceMetadataHapi(resourceType, theEntity, theForHistoryOperation, res);
retVal = populateResourceMetadataHapi(resourceType, theEntity, myTagList, theForHistoryOperation, res);
} else {
IAnyResource res = (IAnyResource) retVal;
retVal = populateResourceMetadataRi(resourceType, theEntity, theForHistoryOperation, res);
retVal = populateResourceMetadataRi(resourceType, theEntity, myTagList, theForHistoryOperation, res);
}
@ -1718,16 +1727,6 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
return theResourceType + '/' + theId.toString();
}
}
private ResourceHistoryTable getHistory(Collection<ResourceHistoryTable> historyList, Long resourceId, Long versionId) {
if (resourceId == null || versionId == null)
return null;
for (ResourceHistoryTable history : historyList) {
if (resourceId.equals(history.getResourceId()) && versionId.equals( history.getVersion()))
return history;
}
return null;
}
@SuppressWarnings("unchecked")
protected ResourceTable updateEntity(RequestDetails theRequest, final IBaseResource theResource, ResourceTable

6
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java
View File

@ -207,7 +207,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
StopWatch w = new StopWatch();
T resourceToDelete = toResource(myResourceType, entity, null, false);
T resourceToDelete = toResource(myResourceType, entity, null, null, false);
// Notify IServerOperationInterceptors about pre-action call
if (theReques != null) {
@ -289,7 +289,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
ResourceTable entity = myEntityManager.find(ResourceTable.class, pid);
deletedResources.add(entity);
T resourceToDelete = toResource(myResourceType, entity, null, false);
T resourceToDelete = toResource(myResourceType, entity, null, null, false);
// Notify IServerOperationInterceptors about pre-action call
if (theRequest != null) {
@ -854,7 +854,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
BaseHasResource entity = readEntity(theId);
validateResourceType(entity);
T retVal = toResource(myResourceType, entity, null, false);
T retVal = toResource(myResourceType, entity, null, null, false);
IPrimitiveType<Date> deleted;
if (retVal instanceof IResource) {

14
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IDao.java
View File

@ -1,16 +1,18 @@
package ca.uhn.fhir.jpa.dao;
import java.util.Collection;
import java.util.Set;
import org.hl7.fhir.instance.model.api.IBaseResource;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.jpa.entity.BaseHasResource;
import ca.uhn.fhir.jpa.entity.ResourceTable;
import ca.uhn.fhir.jpa.entity.ResourceHistoryTable;
import ca.uhn.fhir.jpa.entity.ResourceTable;
import ca.uhn.fhir.jpa.entity.ResourceTag;
import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider;
import org.hl7.fhir.instance.model.api.IBaseResource;
import java.util.Collection;
import java.util.Set;
/*
* #%L
@ -57,6 +59,6 @@ public interface IDao {
IBaseResource toResource(BaseHasResource theEntity, boolean theForHistoryOperation);
<R extends IBaseResource> R toResource(Class<R> theResourceType, BaseHasResource theEntity, Collection<ResourceHistoryTable> historyList, boolean theForHistoryOperation);
<R extends IBaseResource> R toResource(Class<R> theResourceType, BaseHasResource theEntity, ResourceHistoryTable myHistory, Collection<ResourceTag> tagList, boolean theForHistoryOperation);
}

207
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java
View File

@ -1,5 +1,58 @@
package ca.uhn.fhir.jpa.dao;
import static org.apache.commons.lang3.StringUtils.defaultString;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import java.math.BigDecimal;
import java.math.MathContext;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.AbstractQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaBuilder.In;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.From;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import javax.persistence.criteria.Order;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.persistence.criteria.Subquery;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.tuple.Pair;
import org.hibernate.ScrollMode;
import org.hibernate.ScrollableResults;
import org.hibernate.query.Query;
import org.hl7.fhir.dstu3.model.BaseResource;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
/*
* #%L
* HAPI FHIR JPA Server
@ -19,18 +72,45 @@ package ca.uhn.fhir.jpa.dao;
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.context.*;
import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeDeclaredChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
import ca.uhn.fhir.context.ConfigurationException;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.RuntimeChildChoiceDefinition;
import ca.uhn.fhir.context.RuntimeChildResourceDefinition;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
import ca.uhn.fhir.jpa.dao.data.IResourceHistoryTableDao;
import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao;
import ca.uhn.fhir.jpa.entity.*;
import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
import ca.uhn.fhir.jpa.entity.BaseHasResource;
import ca.uhn.fhir.jpa.entity.BaseResourceIndexedSearchParam;
import ca.uhn.fhir.jpa.entity.ResourceHistoryTable;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamDate;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamNumber;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamQuantity;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamString;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamToken;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamUri;
import ca.uhn.fhir.jpa.entity.ResourceLink;
import ca.uhn.fhir.jpa.entity.ResourceTable;
import ca.uhn.fhir.jpa.entity.ResourceTag;
import ca.uhn.fhir.jpa.entity.SearchParam;
import ca.uhn.fhir.jpa.entity.SearchParamPresent;
import ca.uhn.fhir.jpa.entity.TagDefinition;
import ca.uhn.fhir.jpa.entity.TagTypeEnum;
import ca.uhn.fhir.jpa.search.JpaRuntimeSearchParam;
import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
import ca.uhn.fhir.jpa.term.VersionIndependentConcept;
import ca.uhn.fhir.jpa.util.BaseIterator;
import ca.uhn.fhir.jpa.util.ScrollableResultsIterator;
import ca.uhn.fhir.model.api.*;
import ca.uhn.fhir.model.api.IPrimitiveDatatype;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.api.IResource;
import ca.uhn.fhir.model.api.Include;
import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
import ca.uhn.fhir.model.base.composite.BaseCodingDt;
import ca.uhn.fhir.model.base.composite.BaseIdentifierDt;
import ca.uhn.fhir.model.base.composite.BaseQuantityDt;
@ -42,41 +122,25 @@ import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
import ca.uhn.fhir.rest.api.SortOrderEnum;
import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.param.*;
import ca.uhn.fhir.rest.param.CompositeParam;
import ca.uhn.fhir.rest.param.DateParam;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.HasParam;
import ca.uhn.fhir.rest.param.NumberParam;
import ca.uhn.fhir.rest.param.ParamPrefixEnum;
import ca.uhn.fhir.rest.param.QuantityParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.param.TokenParamModifier;
import ca.uhn.fhir.rest.param.UriParam;
import ca.uhn.fhir.rest.param.UriParamQualifierEnum;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.util.StopWatch;
import ca.uhn.fhir.util.UrlUtil;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.tuple.Pair;
import org.hibernate.ScrollMode;
import org.hibernate.ScrollableResults;
import org.hibernate.query.Query;
import org.hl7.fhir.dstu3.model.BaseResource;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.*;
import javax.persistence.criteria.CriteriaBuilder.In;
import java.math.BigDecimal;
import java.math.MathContext;
import java.util.*;
import java.util.Map.Entry;
import static org.apache.commons.lang3.StringUtils.*;
/**
* The SearchBuilder is responsible for actually forming the SQL query that handles
@ -109,13 +173,16 @@ public class SearchBuilder implements ISearchBuilder {
private int myFetchSize;
protected IResourceHistoryTableDao myResourceHistoryTableDao;
protected IResourceTagDao myResourceTagDao;
/**
* Constructor
*/
public SearchBuilder(FhirContext theFhirContext, EntityManager theEntityManager, IFulltextSearchSvc theFulltextSearchSvc,
BaseHapiFhirDao<?> theDao,
IResourceIndexedSearchParamUriDao theResourceIndexedSearchParamUriDao, IForcedIdDao theForcedIdDao, IHapiTerminologySvc theTerminologySvc, ISearchParamRegistry theSearchParamRegistry,
IResourceHistoryTableDao theResourceHistoryTableDao) {
public SearchBuilder(FhirContext theFhirContext, EntityManager theEntityManager,
IFulltextSearchSvc theFulltextSearchSvc, BaseHapiFhirDao<?> theDao,
IResourceIndexedSearchParamUriDao theResourceIndexedSearchParamUriDao, IForcedIdDao theForcedIdDao,
IHapiTerminologySvc theTerminologySvc, ISearchParamRegistry theSearchParamRegistry,
IResourceHistoryTableDao theResourceHistoryTableDao, IResourceTagDao theResourceTagDao) {
myContext = theFhirContext;
myEntityManager = theEntityManager;
myFulltextSearchSvc = theFulltextSearchSvc;
@ -125,6 +192,7 @@ public class SearchBuilder implements ISearchBuilder {
myTerminologySvc = theTerminologySvc;
mySearchParamRegistry = theSearchParamRegistry;
myResourceHistoryTableDao = theResourceHistoryTableDao;
myResourceTagDao = theResourceTagDao;
}
private void addPredicateComposite(String theResourceName, RuntimeSearchParam theParamDef, List<? extends IQueryParameterType> theNextAnd) {
@ -1608,11 +1676,14 @@ public class SearchBuilder implements ISearchBuilder {
List<ResourceTable> resultList = q.getResultList();
//-- Issue #963: Load resource histories based on pids once to improve the performance
Collection<ResourceHistoryTable> historyList = myResourceHistoryTableDao.findByResourceIds(pids);
Map<Long, ResourceHistoryTable> historyMap = getResourceHistoryMap(pids);
//-- preload all tags with tag definition if any
Map<Long, Collection<ResourceTag>> tagMap = getResourceTagMap(resultList);
for (ResourceTable next : resultList) {
Class<? extends IBaseResource> resourceType = context.getResourceDefinition(next.getResourceType()).getImplementingClass();
IBaseResource resource = theDao.toResource(resourceType, next, historyList, theForHistoryOperation);
IBaseResource resource = theDao.toResource(resourceType, next, historyMap.get(next.getId()), tagMap.get(next.getId()), theForHistoryOperation);
if (resource == null) {
ourLog.warn("Unable to find resource {}/{}/_history/{} in database", next.getResourceType(), next.getIdDt().getIdPart(), next.getVersion());
continue;
@ -1641,6 +1712,62 @@ public class SearchBuilder implements ISearchBuilder {
}
}
//-- load all history in to the map
private Map<Long, ResourceHistoryTable> getResourceHistoryMap(Collection<Long> pids) {
Map<Long, ResourceHistoryTable> historyMap = new HashMap<Long, ResourceHistoryTable>();
if (pids.size() == 0)
return historyMap;
Collection<ResourceHistoryTable> historyList = myResourceHistoryTableDao.findByResourceIds(pids);
for (ResourceHistoryTable history : historyList) {
historyMap.put(history.getResourceId(), history);
}
return historyMap;
}
private Map<Long, Collection<ResourceTag>> getResourceTagMap(List<ResourceTable> resourceList) {
List<Long> idList = new ArrayList<Long>(resourceList.size());
//-- find all resource has tags
for (ResourceTable resource: resourceList) {
if (resource.isHasTags())
idList.add(resource.getId());
}
Map<Long, Collection<ResourceTag>> tagMap = new HashMap<Long, Collection<ResourceTag>>();
//-- no tags
if (idList.size() == 0)
return tagMap;
//-- get all tags for the idList
Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(idList);
//-- build the map, key = resourceId, value = list of ResourceTag
Long resourceId;
Collection<ResourceTag> tagCol;
for (ResourceTag tag : tagList) {
resourceId = tag.getResourceId();
tagCol = tagMap.get(resourceId);
if (tagCol == null) {
tagCol = new ArrayList<ResourceTag>();
tagCol.add(tag);
tagMap.put(resourceId, tagCol);
} else {
tagCol.add(tag);
}
}
return tagMap;
}
@Override
public void loadResourcesByPid(Collection<Long> theIncludePids, List<IBaseResource> theResourceListToPopulate, Set<Long> theRevIncludedPids, boolean theForHistoryOperation,
EntityManager entityManager, FhirContext context, IDao theDao) {

10
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTagDao.java
View File

@ -1,5 +1,7 @@
package ca.uhn.fhir.jpa.dao.data;
import java.util.Collection;
/*
* #%L
* HAPI FHIR JPA Server
@ -21,9 +23,15 @@ package ca.uhn.fhir.jpa.dao.data;
*/
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import ca.uhn.fhir.jpa.entity.ResourceTag;
public interface IResourceTagDao extends JpaRepository<ResourceTag, Long> {
// nothing
@Query("" +
"SELECT t FROM ResourceTag t " +
"INNER JOIN TagDefinition td ON (td.myId = t.myTagId) " +
"WHERE t.myResourceId in (:pids)")
Collection<ResourceTag> findByResourceIds(@Param("pids") Collection<Long> pids);
}