Refactor $everything operations on JPA so that they perform better

This commit is contained in:
James Agnew 2015-10-05 16:49:50 -05:00
parent 80575b5380
commit 0958ab6f16
10 changed files with 183 additions and 69 deletions

View File

@ -49,6 +49,7 @@ import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.From;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import javax.persistence.criteria.Order;
import javax.persistence.criteria.Path;
@ -163,7 +164,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
@Autowired
private ISearchResultDao mySearchResultDao;
private Set<Long> addPredicateComposite(RuntimeSearchParam theParamDef, Set<Long> thePids, List<? extends IQueryParameterType> theNextAnd) {
// TODO: fail if missing is set for a composite query
@ -652,7 +653,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
throw new ConfigurationException("Property " + paramPath + " of type " + myResourceName + " is not a resource: " + def.getClass());
}
List<Class<? extends IBaseResource>> resourceTypes;
String resourceId;
if (!ref.getValue().matches("[a-zA-Z]+\\/.*")) {
RuntimeChildResourceDefinition resDef = (RuntimeChildResourceDefinition) def;
@ -677,20 +678,20 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
for (Class<? extends IBaseResource> nextType : resourceTypes) {
RuntimeResourceDefinition typeDef = getContext().getResourceDefinition(nextType);
IFhirResourceDao<?> dao = getDao(nextType);
if (dao == null) {
ourLog.debug("Don't have a DAO for type {}", nextType.getSimpleName());
continue;
}
int qualifierIndex = chain.indexOf(':');
String qualifier = null;
if (qualifierIndex != -1) {
qualifier = chain.substring(qualifierIndex);
chain = chain.substring(0, qualifierIndex);
}
boolean isMeta = RESOURCE_META_PARAMS.containsKey(chain);
RuntimeSearchParam param = null;
if (!isMeta) {
@ -756,7 +757,6 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
return new HashSet<Long>(q.getResultList());
}
private Set<Long> addPredicateString(String theParamName, Set<Long> thePids, List<? extends IQueryParameterType> theList) {
if (theList == null || theList.isEmpty()) {
return thePids;
@ -1665,7 +1665,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
search.setUuid(UUID.randomUUID().toString());
search.setCreated(new Date());
myEntityManager.persist(search);
List<SearchResult> results = new ArrayList<SearchResult>();
if (theId != null) {
Long pid = translateForcedIdToPid(theId);
@ -1682,27 +1682,70 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
results.add(res);
}
}
int totalCount = results.size();
mySearchResultDao.save(results);
mySearchResultDao.flush();
CriteriaBuilder builder = myEntityManager.getCriteriaBuilder();
// Load _revincludes
CriteriaQuery<Long> cq = builder.createQuery(Long.class);
// cq.
Subquery<Long> subQ = cq.subquery(Long.class);
Root<ResourceLink> subQfrom = subQ.from(ResourceLink.class);
subQ.select(subQfrom.get("mySourceResourceId").as(Long.class));
// subQ.where(builder.in(subQfrom.get("myTargetResourceId"), y));
return null;
Root<ResourceLink> from = cq.from(ResourceLink.class);
cq.select(from.get("mySourceResourcePid").as(Long.class));
Subquery<Long> pidsSubquery = cq.subquery(Long.class);
Root<SearchResult> pidsSubqueryFrom = pidsSubquery.from(SearchResult.class);
pidsSubquery.select(pidsSubqueryFrom.get("myResourcePid").as(Long.class));
pidsSubquery.where(pidsSubqueryFrom.get("mySearch").in(search));
cq.where(from.get("myTargetResourceId").in(pidsSubquery));
TypedQuery<Long> query = myEntityManager.createQuery(cq);
results = new ArrayList<SearchResult>();
for (Long next : query.getResultList()) {
SearchResult res = new SearchResult(search);
res.setResourcePid(next);
results.add(res);
}
// Save _revincludes
totalCount += results.size();
mySearchResultDao.save(results);
mySearchResultDao.flush();
final int finalTotalCount = totalCount;
return new IBundleProvider() {
@Override
public int size() {
return finalTotalCount;
}
@Override
public Integer preferredPageSize() {
return null;
}
@Override
public List<IBaseResource> getResources(int theFromIndex, int theToIndex) {
// TODO Auto-generated method stub
return null;
}
@Override
public InstantDt getPublished() {
// TODO Auto-generated method stub
return null;
}
};
}
/**
* THIS SHOULD RETURN HASHSET and not jsut Set because we add to it later (so it can't be Collections.emptySet())
* @param theLastUpdated
*/
private HashSet<Long> loadReverseIncludes(Collection<Long> theMatches, Set<Include> theRevIncludes, boolean theReverseMode, EverythingModeEnum theEverythingModeEnum) {
private HashSet<Long> loadReverseIncludes(Collection<Long> theMatches, Set<Include> theRevIncludes, boolean theReverseMode, EverythingModeEnum theEverythingModeEnum, DateRangeParam theLastUpdated) {
if (theMatches.size() == 0) {
return new HashSet<Long>();
}
@ -1718,11 +1761,11 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
int roundCounts = 0;
StopWatch w = new StopWatch();
boolean addedSomeThisRound;
do {
roundCounts++;
HashSet<Long> pidsToInclude = new HashSet<Long>();
Set<Long> nextRoundOmit = new HashSet<Long>();
@ -1795,6 +1838,9 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
}
}
if (theLastUpdated != null && (theLastUpdated.getLowerBoundAsInstant() != null || theLastUpdated.getUpperBoundAsInstant() != null)) {
pidsToInclude = new HashSet<Long>(filterResourceIdsByLastUpdated(pidsToInclude, theLastUpdated));
}
for (Long next : pidsToInclude) {
if (original.contains(next) == false && allAdded.contains(next) == false) {
theMatches.add(next);
@ -1807,10 +1853,8 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
nextRoundMatches = pidsToInclude;
} while (includes.size() > 0 && nextRoundMatches.size() > 0 && addedSomeThisRound);
ourLog.info("Loaded {} {} in {} rounds and {} ms", new Object[] {
allAdded.size(), theReverseMode ? "_revincludes" : "_includes", roundCounts, w.getMillisAndRestart()
});
ourLog.info("Loaded {} {} in {} rounds and {} ms", new Object[] { allAdded.size(), theReverseMode ? "_revincludes" : "_includes", roundCounts, w.getMillisAndRestart() });
return allAdded;
}
@ -2104,7 +2148,35 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
final InstantDt now = InstantDt.withCurrentTime();
Set<Long> loadPids;
if (theParams.isEmpty()) {
if (theParams.getEverythingMode() != null) {
CriteriaBuilder builder = myEntityManager.getCriteriaBuilder();
CriteriaQuery<Tuple> cq = builder.createTupleQuery();
Root<ResourceTable> from = cq.from(ResourceTable.class);
List<Predicate> predicates = new ArrayList<Predicate>();
if (theParams.get(BaseResource.RES_ID) != null) {
StringParam idParm = (StringParam) theParams.get(BaseResource.RES_ID).get(0).get(0);
predicates.add(builder.equal(from.get("myId"), idParm.getValue()));
}
predicates.add(builder.equal(from.get("myResourceType"), myResourceName));
predicates.add(builder.isNull(from.get("myDeleted")));
cq.where(builder.and(predicates.toArray(new Predicate[predicates.size()])));
Join<Object, Object> join = from.join("myIncomingResourceLinks", JoinType.LEFT);
cq.multiselect(from.get("myId").as(Long.class), join.get("mySourceResourcePid").as(Long.class));
TypedQuery<Tuple> query = myEntityManager.createQuery(cq);
loadPids = new HashSet<Long>();
for (Tuple next : query.getResultList()) {
loadPids.add(next.get(0, Long.class));
Long nextLong = next.get(1, Long.class);
if(nextLong != null) {
loadPids.add(nextLong);
}
}
} else if (theParams.isEmpty()) {
loadPids = new HashSet<Long>();
TypedQuery<Tuple> query = createSearchAllByTypeQuery();
for (Tuple next : query.getResultList()) {
@ -2114,42 +2186,28 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
return new SimpleBundleProvider();
}
} else {
loadPids = searchForIdsWithAndOr(theParams);
if (loadPids.isEmpty()) {
return new SimpleBundleProvider();
}
}
// Load _include and _revinclude before filter and sort in everything mode
if (theParams.getEverythingMode() != null) {
if (theParams.getRevIncludes() != null && theParams.getRevIncludes().isEmpty() == false) {
loadPids.addAll(loadReverseIncludes(loadPids, theParams.getRevIncludes(), true, theParams.getEverythingMode()));
loadPids.addAll(loadReverseIncludes(loadPids, theParams.getIncludes(), false, theParams.getEverythingMode()));
}
}
// // Load _include and _revinclude before filter and sort in everything mode
// if (theParams.getEverythingMode() != null) {
// if (theParams.getRevIncludes() != null && theParams.getRevIncludes().isEmpty() == false) {
// loadPids.addAll(loadReverseIncludes(loadPids, theParams.getRevIncludes(), true, theParams.getEverythingMode()));
// loadPids.addAll(loadReverseIncludes(loadPids, theParams.getIncludes(), false, theParams.getEverythingMode()));
// }
// }
// Handle _lastUpdated
DateRangeParam lu = theParams.getLastUpdated();
final DateRangeParam lu = theParams.getLastUpdated();
if (lu != null && (lu.getLowerBoundAsInstant() != null || lu.getUpperBoundAsInstant() != null)) {
CriteriaBuilder builder = myEntityManager.getCriteriaBuilder();
CriteriaQuery<Long> cq = builder.createQuery(Long.class);
Root<ResourceTable> from = cq.from(ResourceTable.class);
cq.select(from.get("myId").as(Long.class));
Predicate predicateIds = (from.get("myId").in(loadPids));
Predicate predicateLower = lu.getLowerBoundAsInstant() != null ? builder.greaterThanOrEqualTo(from.<Date> get("myUpdated"), lu.getLowerBoundAsInstant()) : null;
Predicate predicateUpper = lu.getUpperBoundAsInstant() != null ? builder.lessThanOrEqualTo(from.<Date> get("myUpdated"), lu.getUpperBoundAsInstant()) : null;
if (predicateLower != null && predicateUpper != null) {
cq.where(predicateIds, predicateLower, predicateUpper);
} else if (predicateLower != null) {
cq.where(predicateIds, predicateLower);
} else {
cq.where(predicateIds, predicateUpper);
}
TypedQuery<Long> query = myEntityManager.createQuery(cq);
List<Long> resultList = filterResourceIdsByLastUpdated(loadPids, lu);
loadPids.clear();
for (Long next : query.getResultList()) {
for (Long next : resultList) {
loadPids.add(next);
}
@ -2165,7 +2223,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
final Set<Long> revIncludedPids;
if (theParams.getEverythingMode() == null) {
if (theParams.getRevIncludes() != null && theParams.getRevIncludes().isEmpty() == false) {
revIncludedPids = loadReverseIncludes(pids, theParams.getRevIncludes(), true, null);
revIncludedPids = loadReverseIncludes(pids, theParams.getRevIncludes(), true, null, lu);
} else {
revIncludedPids = new HashSet<Long>();
}
@ -2192,10 +2250,8 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
List<Long> pidsSubList = pids.subList(theFromIndex, theToIndex);
// Load includes
if (theParams.getEverythingMode() == null) {
pidsSubList = new ArrayList<Long>(pidsSubList);
revIncludedPids.addAll(loadReverseIncludes(pidsSubList, theParams.getIncludes(), false, null));
}
pidsSubList = new ArrayList<Long>(pidsSubList);
revIncludedPids.addAll(loadReverseIncludes(pidsSubList, theParams.getIncludes(), false, null, lu));
// Execute the query and make sure we return distinct results
List<IBaseResource> resources = new ArrayList<IBaseResource>();
@ -2223,6 +2279,27 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
return retVal;
}
private List<Long> filterResourceIdsByLastUpdated(Set<Long> loadPids, final DateRangeParam lu) {
CriteriaBuilder builder = myEntityManager.getCriteriaBuilder();
CriteriaQuery<Long> cq = builder.createQuery(Long.class);
Root<ResourceTable> from = cq.from(ResourceTable.class);
cq.select(from.get("myId").as(Long.class));
Predicate predicateIds = (from.get("myId").in(loadPids));
Predicate predicateLower = lu.getLowerBoundAsInstant() != null ? builder.greaterThanOrEqualTo(from.<Date> get("myUpdated"), lu.getLowerBoundAsInstant()) : null;
Predicate predicateUpper = lu.getUpperBoundAsInstant() != null ? builder.lessThanOrEqualTo(from.<Date> get("myUpdated"), lu.getUpperBoundAsInstant()) : null;
if (predicateLower != null && predicateUpper != null) {
cq.where(predicateIds, predicateLower, predicateUpper);
} else if (predicateLower != null) {
cq.where(predicateIds, predicateLower);
} else {
cq.where(predicateIds, predicateUpper);
}
TypedQuery<Long> query = myEntityManager.createQuery(cq);
List<Long> resultList = query.getResultList();
return resultList;
}
private TypedQuery<Tuple> createSearchAllByTypeQuery() {
CriteriaBuilder builder = myEntityManager.getCriteriaBuilder();
CriteriaQuery<Tuple> cq = builder.createTupleQuery();

View File

@ -43,7 +43,7 @@ public class FhirResourceDaoEncounterDstu2 extends FhirResourceDaoDstu2<Encounte
paramMap.setCount(theCount.getValue());
}
paramMap.setRevIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive()));
// paramMap.setRevIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive()));
paramMap.setIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive()));
paramMap.setEverythingMode(theId != null ? EverythingModeEnum.ENCOUNTER_INSTANCE : EverythingModeEnum.ENCOUNTER_TYPE);
paramMap.setSort(theSort);

View File

@ -24,10 +24,11 @@ import java.util.Collections;
import javax.servlet.http.HttpServletRequest;
import org.hl7.fhir.instance.model.api.IIdType;
import ca.uhn.fhir.jpa.dao.SearchParameterMap.EverythingModeEnum;
import ca.uhn.fhir.model.api.IResource;
import ca.uhn.fhir.model.dstu2.resource.Patient;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.model.primitive.UnsignedIntDt;
import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.param.DateRangeParam;
@ -37,13 +38,13 @@ import ca.uhn.fhir.rest.server.IBundleProvider;
public class FhirResourceDaoPatientDstu2 extends FhirResourceDaoDstu2<Patient>implements IFhirResourceDaoPatient<Patient> {
@Override
public IBundleProvider patientInstanceEverything(HttpServletRequest theServletRequest, IdDt theId, UnsignedIntDt theCount, DateRangeParam theLastUpdated, SortSpec theSort) {
public IBundleProvider patientInstanceEverything(HttpServletRequest theServletRequest, IIdType theId, UnsignedIntDt theCount, DateRangeParam theLastUpdated, SortSpec theSort) {
SearchParameterMap paramMap = new SearchParameterMap();
if (theCount != null) {
paramMap.setCount(theCount.getValue());
}
paramMap.setRevIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive()));
// paramMap.setRevIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive()));
paramMap.setIncludes(Collections.singleton(IResource.INCLUDE_ALL.asRecursive()));
paramMap.setEverythingMode(theId != null ? EverythingModeEnum.PATIENT_INSTANCE : EverythingModeEnum.PATIENT_TYPE);
paramMap.setSort(theSort);

View File

@ -23,8 +23,8 @@ import javax.servlet.http.HttpServletRequest;
*/
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.model.primitive.UnsignedIntDt;
import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.param.DateRangeParam;
@ -32,7 +32,7 @@ import ca.uhn.fhir.rest.server.IBundleProvider;
public interface IFhirResourceDaoPatient<T extends IBaseResource> extends IFhirResourceDao<T> {
IBundleProvider patientInstanceEverything(HttpServletRequest theServletRequest, IdDt theId, UnsignedIntDt theCount, DateRangeParam theLastUpdate, SortSpec theSort);
IBundleProvider patientInstanceEverything(HttpServletRequest theServletRequest, IIdType theId, UnsignedIntDt theCount, DateRangeParam theLastUpdate, SortSpec theSort);
IBundleProvider patientTypeEverything(HttpServletRequest theServletRequest, UnsignedIntDt theCount, DateRangeParam theLastUpdated, SortSpec theSortSpec);

View File

@ -0,0 +1,9 @@
package ca.uhn.fhir.jpa.dao.data;
import org.springframework.data.jpa.repository.JpaRepository;
import ca.uhn.fhir.jpa.entity.ResourceTable;
public interface IResourceTableDao extends JpaRepository<ResourceTable, Long> {
}

View File

@ -28,6 +28,7 @@ import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Index;
import javax.persistence.SequenceGenerator;
import javax.persistence.Table;
import javax.persistence.Temporal;
@ -38,6 +39,8 @@ import javax.persistence.UniqueConstraint;
@Entity
@Table(name = "HFJ_SEARCH", uniqueConstraints= {
@UniqueConstraint(name="IDX_SEARCH_UUID", columnNames="SEARCH_UUID")
}, indexes= {
@Index(name="JDX_SEARCH_CREATED", columnList="CREATED")
})
//@formatter:on
public class Search implements Serializable {
@ -54,6 +57,9 @@ public class Search implements Serializable {
@Column(name = "PID")
private Long myId;
@Column(name="TOTAL_COUNT")
private int myTotalCount;
@Column(name="SEARCH_UUID", length=40, nullable=false)
private String myUuid;
@ -61,6 +67,10 @@ public class Search implements Serializable {
return myCreated;
}
public int getTotalCount() {
return myTotalCount;
}
public String getUuid() {
return myUuid;
}
@ -68,6 +78,10 @@ public class Search implements Serializable {
public void setCreated(Date theCreated) {
myCreated = theCreated;
}
public void setTotalCount(int theTotalCount) {
myTotalCount = theTotalCount;
}
public void setUuid(String theUuid) {
myUuid = theUuid;

View File

@ -60,7 +60,6 @@ import ca.uhn.fhir.model.dstu2.resource.Medication;
import ca.uhn.fhir.model.dstu2.resource.MedicationOrder;
import ca.uhn.fhir.model.dstu2.resource.Observation;
import ca.uhn.fhir.model.dstu2.resource.Organization;
import ca.uhn.fhir.model.dstu2.resource.Parameters;
import ca.uhn.fhir.model.dstu2.resource.Patient;
import ca.uhn.fhir.model.dstu2.resource.Practitioner;
import ca.uhn.fhir.model.dstu2.resource.Subscription;
@ -696,6 +695,11 @@ public class FhirResourceDaoDstu2SearchTest extends BaseJpaDstu2Test {
pat.getManagingOrganization().setReference(orgId);
IIdType patId = myPatientDao.create(pat).getId().toUnqualifiedVersionless();
Patient pat2 = new Patient();
pat2.addAddress().addLine(methodName);
pat2.getManagingOrganization().setReference(orgId);
IIdType patId2 = myPatientDao.create(pat2).getId().toUnqualifiedVersionless();
MedicationOrder mo = new MedicationOrder();
mo.getPatient().setReference(patId);
mo.setMedication(new ResourceReferenceDt(medId));
@ -703,7 +707,11 @@ public class FhirResourceDaoDstu2SearchTest extends BaseJpaDstu2Test {
HttpServletRequest request = mock(HttpServletRequest.class);
IBundleProvider resp = myPatientDao.patientTypeEverything(request, null, null, null);
assertEquals(4, resp.size());
assertThat(toUnqualifiedVersionlessIds(resp), containsInAnyOrder(orgId, medId, patId, moId, patId2));
request = mock(HttpServletRequest.class);
resp = myPatientDao.patientInstanceEverything(request, patId, null, null, null);
assertThat(toUnqualifiedVersionlessIds(resp), containsInAnyOrder(orgId, medId, patId, moId));
}

View File

@ -299,7 +299,7 @@ public class FhirSystemDaoDstu2Test extends BaseJpaDstu2Test {
}
@Test
public void testEverythingType() {
public void testTransactionSingleEmptyResource() {
Bundle request = new Bundle();
request.setType(BundleTypeEnum.SEARCH_RESULTS);

View File

@ -41,7 +41,6 @@ import org.apache.http.entity.StringEntity;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.junit.Test;
import org.thymeleaf.util.UrlUtils;
import ca.uhn.fhir.model.api.Bundle;
import ca.uhn.fhir.model.api.IResource;
@ -53,6 +52,7 @@ import ca.uhn.fhir.model.dstu2.composite.CodingDt;
import ca.uhn.fhir.model.dstu2.composite.MetaDt;
import ca.uhn.fhir.model.dstu2.composite.PeriodDt;
import ca.uhn.fhir.model.dstu2.composite.ResourceReferenceDt;
import ca.uhn.fhir.model.dstu2.resource.BaseResource;
import ca.uhn.fhir.model.dstu2.resource.Bundle.Entry;
import ca.uhn.fhir.model.dstu2.resource.Condition;
import ca.uhn.fhir.model.dstu2.resource.DiagnosticOrder;
@ -1047,6 +1047,7 @@ public class ResourceProviderDstu2Test extends BaseResourceProviderDstu2Test {
assertThat(ids, not(containsInRelativeOrder(c3Id)));
}
// retest
@Test
public void testEverythingPatientWithLastUpdatedAndSort() throws Exception {
String methodName = "testEverythingWithLastUpdatedAndSort";
@ -1438,8 +1439,8 @@ public class ResourceProviderDstu2Test extends BaseResourceProviderDstu2Test {
Bundle found = ourClient
.search()
.forResource(Patient.class)
.where(Patient.RES_ID.matches().values(id1.getIdPart(), id2.getIdPart()))
.and(Patient.RES_ID.matches().value(id1.getIdPart()))
.where(BaseResource.RES_ID.matches().values(id1.getIdPart(), id2.getIdPart()))
.and(BaseResource.RES_ID.matches().value(id1.getIdPart()))
.execute();
//@formatter:on

View File

@ -151,6 +151,10 @@
by user code. Thanks to Simone Heckmann
pointing out that this was needed!
</action>
<action type="add">
Refactor JPA $everything operations so that
they perform better
</action>
</release>
<release version="1.2" date="2015-09-18">
<action type="add">