More perf work

This commit is contained in:
James 2017-04-14 07:25:01 -04:00
parent 23b7de5dd8
commit 1039bbec97
15 changed files with 151 additions and 105 deletions

View File

@ -39,7 +39,7 @@ public class PagingPatientProvider implements IResourceProvider {
return new IBundleProvider() {
@Override
public int size() {
public Integer size() {
return matchingResourceIds.size();
}

View File

@ -196,7 +196,7 @@ public class HistoryMethodBinding extends BaseResourceReturningMethodBinding {
}
@Override
public int size() {
public Integer size() {
return resources.size();
}

View File

@ -51,7 +51,7 @@ public class BundleProviders {
}
@Override
public int size() {
public Integer size() {
return 0;
}

View File

@ -52,7 +52,7 @@ public class SimpleBundleProvider implements IBundleProvider {
}
@Override
public int size() {
public Integer size() {
return myList.size();
}

View File

@ -942,15 +942,6 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
ourLog.info("Processed remove tag {}/{} on {} in {}ms", new Object[] { theScheme, theTerm, theId.getValue(), w.getMillisAndRestart() });
}
@Override
public IBundleProvider search(Map<String, IQueryParameterType> theParams) {
SearchParameterMap map = new SearchParameterMap();
for (Entry<String, IQueryParameterType> nextEntry : theParams.entrySet()) {
map.add(nextEntry.getKey(), (nextEntry.getValue()));
}
return search(map);
}
@Override
public IBundleProvider search(final SearchParameterMap theParams) {
return search(theParams, null);
@ -959,20 +950,17 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
@Override
public IBundleProvider search(final SearchParameterMap theParams, RequestDetails theRequestDetails) {
// Notify interceptors
ActionRequestDetails requestDetails = new ActionRequestDetails(theRequestDetails, getContext(), getResourceName(), null);
notifyInterceptors(RestOperationTypeEnum.SEARCH_TYPE, requestDetails);
if (theRequestDetails != null) {
ActionRequestDetails requestDetails = new ActionRequestDetails(theRequestDetails, getContext(), getResourceName(), null);
notifyInterceptors(RestOperationTypeEnum.SEARCH_TYPE, requestDetails);
}
SearchBuilder builder = newSearchBuilder();
builder.setType(getResourceType(), getResourceName());
return builder.search(theParams);
}
@Override
public IBundleProvider search(String theParameterName, IQueryParameterType theValue) {
return search(Collections.singletonMap(theParameterName, theValue));
}
@Override
public Set<Long> searchForIds(Map<String, IQueryParameterType> theParams) {
SearchParameterMap map = new SearchParameterMap();

View File

@ -110,7 +110,10 @@ public class FhirResourceDaoValueSetDstu2 extends FhirResourceDaoDstu2<ValueSet>
if (defaultValueSet != null) {
source = getContext().newJsonParser().parseResource(ValueSet.class, myRiCtx.newJsonParser().encodeResourceToString(defaultValueSet));
} else {
IBundleProvider ids = search(ValueSet.SP_URL, new UriParam(theUri));
SearchParameterMap params = new SearchParameterMap();
params.setLoadSynchronousUpTo(1);
params.add(ValueSet.SP_URL, new UriParam(theUri));
IBundleProvider ids = search(params);
if (ids.size() == 0) {
throw new InvalidRequestException("Unknown ValueSet URI: " + theUri);
}

View File

@ -70,7 +70,7 @@ public class JpaValidationSupportDstu2 implements IJpaValidationSupportDstu2 {
params.setLoadSynchronousUpTo(10);
search = myValueSetDao.search(params);
} else if ("StructureDefinition".equals(resourceName)) {
search = myStructureDefinitionDao.search(ca.uhn.fhir.model.dstu2.resource.StructureDefinition.SP_URL, new UriParam(theUri));
search = myStructureDefinitionDao.search(new SearchParameterMap().setLoadSynchronous(true).add(ca.uhn.fhir.model.dstu2.resource.StructureDefinition.SP_URL, new UriParam(theUri)));
} else {
throw new IllegalArgumentException("Can't fetch resource type: " + resourceName);
}

View File

@ -38,6 +38,7 @@ import javax.persistence.criteria.*;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.tuple.Pair;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
@ -1438,7 +1439,6 @@ public class SearchBuilder {
mySearchEntity.setUuid(UUID.randomUUID().toString());
mySearchEntity.setCreated(new Date());
mySearchEntity.setTotalCount(-1);
mySearchEntity.setSearchParamMap(SerializationUtils.serialize(myParams));
mySearchEntity.setPreferredPageSize(myParams.getCount());
mySearchEntity.setSearchType(myParams.getEverythingMode() != null ? SearchTypeEnum.EVERYTHING : SearchTypeEnum.SEARCH);
mySearchEntity.setLastUpdated(myParams.getLastUpdated());
@ -1457,43 +1457,8 @@ public class SearchBuilder {
}
}
public IBundleProvider search(final SearchParameterMap theParams) {
public Iterator<Long> createQuery(SearchParameterMap theParams) {
myParams = theParams;
StopWatch w = new StopWatch();
mySearchEntity = new Search();
mySearchEntity.setUuid(UUID.randomUUID().toString());
mySearchEntity.setCreated(new Date());
mySearchEntity.setTotalCount(-1);
mySearchEntity.setSearchParamMap(SerializationUtils.serialize(myParams));
mySearchEntity.setPreferredPageSize(myParams.getCount());
mySearchEntity.setSearchType(myParams.getEverythingMode() != null ? SearchTypeEnum.EVERYTHING : SearchTypeEnum.SEARCH);
mySearchEntity.setLastUpdated(myParams.getLastUpdated());
mySearchEntity.setResourceType(myResourceName);
for (Include next : myParams.getIncludes()) {
mySearchEntity.getIncludes().add(new SearchInclude(mySearchEntity, next.getValue(), false, next.isRecurse()));
}
for (Include next : myParams.getRevIncludes()) {
mySearchEntity.getIncludes().add(new SearchInclude(mySearchEntity, next.getValue(), true, next.isRecurse()));
}
List<Long> firstPage = loadSearchPage(theParams, 0, 999);
mySearchEntity.setTotalCount(firstPage.size());
myEntityManager.persist(mySearchEntity);
for (SearchInclude next : mySearchEntity.getIncludes()) {
myEntityManager.persist(next);
}
IBundleProvider retVal = doReturnProvider();
ourLog.info("Search initial phase completed in {}ms", w);
return retVal;
}
public List<Long> loadSearchPage(SearchParameterMap theParams, int theFromIndex, int theToIndex) {
myBuilder = myEntityManager.getCriteriaBuilder();
CriteriaQuery<Long> outerQuery = null;
@ -1503,7 +1468,7 @@ public class SearchBuilder {
* If we have a sort, we wrap the criteria search (the search that actually
* finds the appropriate resources) in an outer search which is then sorted
*/
if (theParams.getSort() != null) {
if (myParams.getSort() != null) {
outerQuery = myBuilder.createQuery(Long.class);
Root<ResourceTable> outerQueryFrom = outerQuery.from(ResourceTable.class);
@ -1511,7 +1476,7 @@ public class SearchBuilder {
List<Order> orders = Lists.newArrayList();
List<Predicate> predicates = Lists.newArrayList();
createSort(myBuilder, outerQueryFrom, theParams.getSort(), orders, predicates);
createSort(myBuilder, outerQueryFrom, myParams.getSort(), orders, predicates);
if (orders.size() > 0) {
outerQuery.orderBy(orders);
}
@ -1542,20 +1507,20 @@ public class SearchBuilder {
myResourceTableQuery.distinct(true);
myPredicates = new ArrayList<Predicate>();
if (theParams.getEverythingMode() == null) {
if (myParams.getEverythingMode() == null) {
myPredicates.add(myBuilder.equal(myResourceTableRoot.get("myResourceType"), myResourceName));
}
myPredicates.add(myBuilder.isNull(myResourceTableRoot.get("myDeleted")));
DateRangeParam lu = theParams.getLastUpdated();
DateRangeParam lu = myParams.getLastUpdated();
List<Predicate> lastUpdatedPredicates = createLastUpdatedPredicates(lu, myBuilder, myResourceTableRoot);
myPredicates.addAll(lastUpdatedPredicates);
if (theParams.getEverythingMode() != null) {
if (myParams.getEverythingMode() != null) {
Join<ResourceTable, ResourceLink> join = myResourceTableRoot.join("myResourceLinks", JoinType.LEFT);
if (theParams.get(BaseResource.SP_RES_ID) != null) {
StringParam idParm = (StringParam) theParams.get(BaseResource.SP_RES_ID).get(0).get(0);
if (myParams.get(BaseResource.SP_RES_ID) != null) {
StringParam idParm = (StringParam) myParams.get(BaseResource.SP_RES_ID).get(0).get(0);
Long pid = BaseHapiFhirDao.translateForcedIdToPid(myResourceName, idParm.getValue(), myForcedIdDao);
myPredicates.add(myBuilder.equal(join.get("myTargetResourcePid").as(Long.class), pid));
} else {
@ -1566,21 +1531,21 @@ public class SearchBuilder {
} else {
// Normal search
searchForIdsWithAndOr(theParams);
searchForIdsWithAndOr(myParams);
}
/*
* Fulltext search
*/
if (theParams.containsKey(Constants.PARAM_CONTENT) || theParams.containsKey(Constants.PARAM_TEXT)) {
if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) {
if (myFulltextSearchSvc == null) {
if (theParams.containsKey(Constants.PARAM_TEXT)) {
if (myParams.containsKey(Constants.PARAM_TEXT)) {
throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT);
} else if (theParams.containsKey(Constants.PARAM_CONTENT)) {
} else if (myParams.containsKey(Constants.PARAM_CONTENT)) {
throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT);
}
}
List<Long> pids = myFulltextSearchSvc.everything(myResourceName, theParams);
List<Long> pids = myFulltextSearchSvc.everything(myResourceName, myParams);
if (pids.isEmpty()) {
// Will never match
pids = Collections.singletonList((Long) null);
@ -1595,19 +1560,89 @@ public class SearchBuilder {
* Now perform the search
*/
TypedQuery<Long> query = myEntityManager.createQuery(outerQuery);
query.setFirstResult(theFromIndex);
query.setMaxResults(theToIndex - theFromIndex);
List<Long> pids = new ArrayList<Long>();
Set<Long> pidSet = new HashSet<Long>();
final Iterator<Long> results = query.getResultList().iterator();
final Set<Long> pidSet = new HashSet<Long>();
for (Long next : query.getResultList()) {
if (next != null && pidSet.add(next)) {
pids.add(next);
return new Iterator<Long>() {
private Long myNext;
@Override
public boolean hasNext() {
if (myNext == null) {
fetchNext();
}
if (myNext == NO_MORE) {
return false;
}
return true;
}
private void fetchNext() {
if (myNext == null) {
while (results.hasNext()) {
Long next = results.next();
if (next != null && pidSet.add(next)) {
myNext = next;
break;
}
}
if (myNext == null) {
myNext = NO_MORE;
}
}
}
@Override
public Long next() {
fetchNext();
Validate.isTrue(myNext != NO_MORE, "No more elements");
return myNext;
}
};
}
private static Long NO_MORE = Long.valueOf(-1);
public IBundleProvider search(final SearchParameterMap theParams) {
myParams = theParams;
StopWatch w = new StopWatch();
if (theParams.isLoadSynchronous()) {
}
mySearchEntity = new Search();
mySearchEntity.setUuid(UUID.randomUUID().toString());
mySearchEntity.setCreated(new Date());
mySearchEntity.setTotalCount(-1);
mySearchEntity.setPreferredPageSize(myParams.getCount());
mySearchEntity.setSearchType(myParams.getEverythingMode() != null ? SearchTypeEnum.EVERYTHING : SearchTypeEnum.SEARCH);
mySearchEntity.setLastUpdated(myParams.getLastUpdated());
mySearchEntity.setResourceType(myResourceName);
for (Include next : myParams.getIncludes()) {
mySearchEntity.getIncludes().add(new SearchInclude(mySearchEntity, next.getValue(), false, next.isRecurse()));
}
for (Include next : myParams.getRevIncludes()) {
mySearchEntity.getIncludes().add(new SearchInclude(mySearchEntity, next.getValue(), true, next.isRecurse()));
}
return pids;
List<Long> firstPage = loadSearchPage(theParams, 0, 999);
mySearchEntity.setTotalCount(firstPage.size());
myEntityManager.persist(mySearchEntity);
for (SearchInclude next : mySearchEntity.getIncludes()) {
myEntityManager.persist(next);
}
IBundleProvider retVal = doReturnProvider();
ourLog.info("Search initial phase completed in {}ms", w);
return retVal;
}
public List<Long> loadSearchPage(SearchParameterMap theParams, int theFromIndex, int theToIndex) {
}
// public IBundleProvider loadPage(SearchParameterMap theParams, int theFromIndex, int theToIndex) {

View File

@ -38,6 +38,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.dao.SearchParameterMap;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.UriParam;
import ca.uhn.fhir.rest.server.IBundleProvider;
@ -92,22 +93,40 @@ public class JpaValidationSupportDstu3 implements IJpaValidationSupportDstu3 {
IBundleProvider search;
if ("ValueSet".equals(resourceName)) {
if (localReference) {
search = myValueSetDao.search(IAnyResource.SP_RES_ID, new StringParam(theUri));
SearchParameterMap params = new SearchParameterMap();
params.setLoadSynchronousUpTo(1);
params.add(IAnyResource.SP_RES_ID, new StringParam(theUri));
search = myValueSetDao.search(params);
if (search.size() == 0) {
search = myValueSetDao.search(ValueSet.SP_URL, new UriParam(theUri));
params = new SearchParameterMap();
params.setLoadSynchronousUpTo(1);
params.add(ValueSet.SP_URL, new UriParam(theUri));
search = myValueSetDao.search(params);
}
} else {
search = myValueSetDao.search(ValueSet.SP_URL, new UriParam(theUri));
SearchParameterMap params = new SearchParameterMap();
params.setLoadSynchronousUpTo(1);
params.add(ValueSet.SP_URL, new UriParam(theUri));
search = myValueSetDao.search(params);
}
} else if ("StructureDefinition".equals(resourceName)) {
if (theUri.startsWith("http://hl7.org/fhir/StructureDefinition/")) {
return null;
}
search = myStructureDefinitionDao.search(StructureDefinition.SP_URL, new UriParam(theUri));
SearchParameterMap params = new SearchParameterMap();
params.setLoadSynchronousUpTo(1);
params.add(StructureDefinition.SP_URL, new UriParam(theUri));
search = myStructureDefinitionDao.search(params);
} else if ("Questionnaire".equals(resourceName)) {
search = myQuestionnaireDao.search(IAnyResource.SP_RES_ID, new StringParam(id.getIdPart()));
SearchParameterMap params = new SearchParameterMap();
params.setLoadSynchronousUpTo(1);
params.add(IAnyResource.SP_RES_ID, new StringParam(id.getIdPart()));
search = myQuestionnaireDao.search(params);
} else if ("CodeSystem".equals(resourceName)) {
search = myCodeSystemDao.search(CodeSystem.SP_URL, new UriParam(theUri));
SearchParameterMap params = new SearchParameterMap();
params.setLoadSynchronousUpTo(1);
params.add(CodeSystem.SP_URL, new UriParam(theUri));
search = myCodeSystemDao.search(params);
} else {
throw new IllegalArgumentException("Can't fetch resource type: " + resourceName);
}

View File

@ -121,8 +121,9 @@ public final class PersistedJpaBundleProvider implements IBundleProvider {
Class<? extends IBaseResource> resourceType = myContext.getResourceDefinition(resourceName).getImplementingClass();
sb.setType(resourceType, resourceName);
SearchParameterMap parameterMap = SerializationUtils.deserialize(mySearchEntity.getSearchParamMap());
List<Long> pidsSubList = sb.loadSearchPage(parameterMap, theFromIndex, theToIndex);
// SearchParameterMap parameterMap = SerializationUtils.deserialize(mySearchEntity.getSearchParamMap());
// List<Long> pidsSubList = sb.loadSearchPage(parameterMap, theFromIndex, theToIndex);());
List<Long> pidsSubList = null;
Set<Long> revIncludedPids = new HashSet<Long>();
if (mySearchEntity.getSearchType() == SearchTypeEnum.SEARCH) {
@ -236,7 +237,7 @@ public final class PersistedJpaBundleProvider implements IBundleProvider {
}
@Override
public int size() {
public Integer size() {
ensureSearchEntityLoaded();
return Math.max(0, mySearchEntity.getTotalCount());
}

View File

@ -172,10 +172,10 @@ public class FhirSystemDaoDstu1Test extends BaseJpaTest {
// Try to search
IBundleProvider obsResults = ourObservationDao.search(Observation.SP_NAME, new IdentifierDt("urn:system", "testPersistWithSimpleLinkO01"));
IBundleProvider obsResults = ourObservationDao.search(new SearchParameterMap().setLoadSynchronous(true).add(Observation.SP_NAME, new IdentifierDt("urn:system", "testPersistWithSimpleLinkO01")));
assertEquals(1, obsResults.size().intValue());
IBundleProvider patResults = ourPatientDao.search(Patient.SP_IDENTIFIER, new IdentifierDt("urn:system", "testPersistWithSimpleLinkP01"));
IBundleProvider patResults = ourPatientDao.search(new SearchParameterMap().setLoadSynchronous(true).add(Patient.SP_IDENTIFIER, new IdentifierDt("urn:system", "testPersistWithSimpleLinkP01")));
assertEquals(1, obsResults.size().intValue());
IIdType foundPatientId = patResults.getResources(0, 1).get(0).getIdElement();
@ -446,8 +446,8 @@ public class FhirSystemDaoDstu1Test extends BaseJpaTest {
* Verify
*/
IBundleProvider results = ourPatientDao.search(Patient.SP_IDENTIFIER, new TokenParam("urn:system", "testTransactionWithDelete"));
assertEquals(3, results.size());
IBundleProvider results = ourPatientDao.search(new SearchParameterMap().setLoadSynchronous(true).add(Patient.SP_IDENTIFIER, new TokenParam("urn:system", "testTransactionWithDelete")));
assertEquals(3, results.size().intValue());
/*
* Now delete 2
@ -472,8 +472,8 @@ public class FhirSystemDaoDstu1Test extends BaseJpaTest {
* Verify
*/
IBundleProvider results2 = ourPatientDao.search(Patient.SP_IDENTIFIER, new TokenParam("urn:system", "testTransactionWithDelete"));
assertEquals(1, results2.size());
IBundleProvider results2 = ourPatientDao.search(new SearchParameterMap().setLoadSynchronous(true).add(Patient.SP_IDENTIFIER, new TokenParam("urn:system", "testTransactionWithDelete")));
assertEquals(1, results2.size().intValue());
List<IBaseResource> existing2 = results2.getResources(0, 1);
assertEquals(existing2.get(0).getIdElement(), existing.get(2).getIdElement());

View File

@ -875,7 +875,7 @@ public class FhirSystemDaoDstu2Test extends BaseJpaDstu2SystemTest {
}
IBundleProvider history = myPatientDao.history(id, null, null, mySrd);
assertEquals(2, history.size());
assertEquals(2, history.size().intValue());
assertNotNull(ResourceMetadataKeyEnum.DELETED_AT.get((IResource) history.getResources(0, 1).get(0)));
assertNotNull(ResourceMetadataKeyEnum.DELETED_AT.get((IResource) history.getResources(0, 1).get(0)).getValue());

View File

@ -126,7 +126,7 @@ public class FhirSystemDaoDstu3Test extends BaseJpaDstu3SystemTest {
ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(outputBundle));
IBundleProvider allPatients = myPatientDao.search(new SearchParameterMap());
assertEquals(1, allPatients.size());
assertEquals(1, allPatients.size().intValue());
}
@Test
@ -696,13 +696,13 @@ public class FhirSystemDaoDstu3Test extends BaseJpaDstu3SystemTest {
SearchParameterMap map = new SearchParameterMap();
map.add(Organization.SP_PARTOF, new ReferenceParam(id1.toUnqualifiedVersionless().getValue()));
IBundleProvider res = myOrganizationDao.search(map);
assertEquals(1, res.size());
assertEquals(1, res.size().intValue());
assertEquals(id2.toUnqualifiedVersionless().getValue(), res.getResources(0, 1).get(0).getIdElement().toUnqualifiedVersionless().getValue());
map = new SearchParameterMap();
map.add(Organization.SP_PARTOF, new ReferenceParam(id2.toUnqualifiedVersionless().getValue()));
res = myOrganizationDao.search(map);
assertEquals(1, res.size());
assertEquals(1, res.size().intValue());
assertEquals(id1.toUnqualifiedVersionless().getValue(), res.getResources(0, 1).get(0).getIdElement().toUnqualifiedVersionless().getValue());
/*
@ -735,13 +735,13 @@ public class FhirSystemDaoDstu3Test extends BaseJpaDstu3SystemTest {
map = new SearchParameterMap();
map.add(Organization.SP_PARTOF, new ReferenceParam(id1.toUnqualifiedVersionless().getValue()));
res = myOrganizationDao.search(map);
assertEquals(1, res.size());
assertEquals(1, res.size().intValue());
assertEquals(id1.toUnqualifiedVersionless().getValue(), res.getResources(0, 1).get(0).getIdElement().toUnqualifiedVersionless().getValue());
map = new SearchParameterMap();
map.add(Organization.SP_PARTOF, new ReferenceParam(id2.toUnqualifiedVersionless().getValue()));
res = myOrganizationDao.search(map);
assertEquals(1, res.size());
assertEquals(1, res.size().intValue());
assertEquals(id2.toUnqualifiedVersionless().getValue(), res.getResources(0, 1).get(0).getIdElement().toUnqualifiedVersionless().getValue());
}
@ -1139,7 +1139,7 @@ public class FhirSystemDaoDstu3Test extends BaseJpaDstu3SystemTest {
}
IBundleProvider history = myPatientDao.history(id, null, null, mySrd);
assertEquals(2, history.size());
assertEquals(2, history.size().intValue());
assertNotNull(ResourceMetadataKeyEnum.DELETED_AT.get((IAnyResource) history.getResources(0, 1).get(0)));
assertNotNull(ResourceMetadataKeyEnum.DELETED_AT.get((IAnyResource) history.getResources(0, 1).get(0)).getValue());

View File

@ -572,7 +572,7 @@ public class SearchDstu2Test {
}
@Override
public int size() {
public Integer size() {
return 0;
}

View File

@ -157,7 +157,7 @@ public class SearchHl7OrgDstu2Test {
}
@Override
public int size() {
public Integer size() {
return 0;
}