From 233eb54710f8e9e848f69bec9fcbc37bb9d84d7a Mon Sep 17 00:00:00 2001 From: James Agnew Date: Mon, 3 Apr 2017 22:05:53 -0400 Subject: [PATCH] Work on perf --- .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 210 ++++++++++-------- .../search/DatabaseBackedPagingProvider.java | 12 +- .../search/PersistedJpaBundleProvider.java | 17 +- .../FhirResourceDaoDstu3SearchNoFtTest.java | 16 +- 4 files changed, 133 insertions(+), 122 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 1cc587bacb8..e65df28d244 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -282,14 +282,9 @@ public class SearchBuilder { doSetPids(q.getResultList()); } - private void addPredicateLanguage(List> theList) { + private void addPredicateLanguage(List thePredicates, Root theResourceTableRoot, List> theList) { for (List nextList : theList) { - CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); - CriteriaQuery cq = builder.createQuery(Long.class); - Root from = cq.from(ResourceTable.class); - cq.select(from.get("myId").as(Long.class)); - Set values = new HashSet(); for (IQueryParameterType next : nextList) { if (next instanceof StringParam) { @@ -307,21 +302,8 @@ public class SearchBuilder { continue; } - List predicates = new ArrayList(); - predicates.add(builder.equal(from.get("myResourceType"), myResourceName)); - predicates.add(from.get("myLanguage").as(String.class).in(values)); - createPredicateResourceId(builder, cq, predicates, from.get("myId").as(Long.class)); - createPredicateLastUpdatedForResourceTable(builder, from, predicates); - - predicates.add(builder.isNull(from.get("myDeleted"))); - - cq.where(toArray(predicates)); - - TypedQuery q = myEntityManager.createQuery(cq); - doSetPids(q.getResultList()); - if (doHaveNoResults()) { - return; - } + Predicate predicate = theResourceTableRoot.get("myLanguage").as(String.class).in(values); + thePredicates.add(predicate); } return; @@ -719,24 +701,6 @@ public class SearchBuilder { throw new IllegalArgumentException("Param name: " + theParamName); // shouldn't happen } - /* - * CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); CriteriaQuery cq = - * builder.createQuery(Long.class); Root from = cq.from(ResourceTable.class); - * cq.select(from.get("myId").as(Long.class)); - * - * Subquery subQ = cq.subquery(Long.class); Root subQfrom = - * subQ.from(theParamTable); subQ.select(subQfrom.get("myResourcePid").as(Long.class)); - * Predicate subQname = builder.equal(subQfrom.get("myParamName"), theParamName); Predicate subQtype = - * builder.equal(subQfrom.get("myResourceType"), myResourceName); - * subQ.where(builder.and(subQtype, subQname)); - * - * List predicates = new ArrayList(); - * predicates.add(builder.not(builder.in(from.get("myId")).value(subQ))); - * predicates.add(builder.equal(from.get("myResourceType"), - * myResourceName)); predicates.add(builder.isNull(from.get("myDeleted"))); createPredicateResourceId(builder, cq, - * predicates, from.get("myId").as(Long.class)); - */ - List> notTags = Lists.newArrayList(); for (List nextAndParams : theList) { for (IQueryParameterType nextOrParams : nextAndParams) { @@ -1684,10 +1648,79 @@ public class SearchBuilder { myParams = theParams; StopWatch w = new StopWatch(); - doInitializeSearch(); - return doReturnProvider(); + mySearchEntity = new Search(); + mySearchEntity.setUuid(UUID.randomUUID().toString()); + mySearchEntity.setCreated(new Date()); + mySearchEntity.setTotalCount(-1); + mySearchEntity.setSearchParamMap(SerializationUtils.serialize(myParams)); + mySearchEntity.setPreferredPageSize(myParams.getCount()); + mySearchEntity.setSearchType(myParams.getEverythingMode() != null ? SearchTypeEnum.EVERYTHING : SearchTypeEnum.SEARCH); + mySearchEntity.setLastUpdated(myParams.getLastUpdated()); + + for (Include next : myParams.getIncludes()) { + mySearchEntity.getIncludes().add(new SearchInclude(mySearchEntity, next.getValue(), false, next.isRecurse())); + } + for (Include next : myParams.getRevIncludes()) { + mySearchEntity.getIncludes().add(new SearchInclude(mySearchEntity, next.getValue(), true, next.isRecurse())); + } + + List firstPage = loadSearchPage(theParams, 0, 999); + mySearchEntity.setTotalCount(firstPage.size()); + + myEntityManager.persist(mySearchEntity); + for (SearchInclude next : mySearchEntity.getIncludes()) { + myEntityManager.persist(next); + } + + IBundleProvider retVal = doReturnProvider(); + + ourLog.info("Search initial phase completed in {}ms", w); + return retVal; } + public List loadSearchPage(SearchParameterMap theParams, int theFromIndex, int theToIndex) { + + if (myFulltextSearchSvc == null) { + if (theParams.containsKey(Constants.PARAM_TEXT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + } else if (theParams.containsKey(Constants.PARAM_CONTENT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + } + } else { + // FIXME: add from and to + List searchResultPids = myFulltextSearchSvc.search(myResourceName, theParams); + if (searchResultPids != null) { + if (searchResultPids.isEmpty()) { + return Collections.emptyList(); + } + return searchResultPids; + } + } + + CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); + CriteriaQuery cq = builder.createTupleQuery(); + Root from = cq.from(ResourceTable.class); + List predicates = new ArrayList(); + predicates.add(builder.equal(from.get("myResourceType"), myResourceName)); + predicates.add(builder.isNull(from.get("myDeleted"))); + + searchForIdsWithAndOr(theParams, predicates, from); + + cq.where(builder.and(SearchBuilder.toArray(predicates))); + + cq.multiselect(from.get("myId").as(Long.class)); + TypedQuery query = myEntityManager.createQuery(cq); + query.setFirstResult(theFromIndex); + query.setMaxResults(theToIndex - theFromIndex); + + List pids = new ArrayList(); + for (Tuple next : query.getResultList()) { + pids.add(next.get(0, Long.class)); + } + + return pids; + } + public IBundleProvider loadPage(SearchParameterMap theParams, int theFromIndex, int theToIndex) { StopWatch sw = new StopWatch(); DateRangeParam lu = theParams.getLastUpdated(); @@ -1737,11 +1770,6 @@ public class SearchBuilder { } - } else if (theParams.isEmpty()) { - - TypedQuery query = createSearchAllByTypeQuery(lu); - doSetPids(query.getResultList()); - } else { if (myFulltextSearchSvc == null) { @@ -1760,8 +1788,9 @@ public class SearchBuilder { } } + if (!theParams.isEmpty()) { - searchForIdsWithAndOr(theParams, lu); +// searchForIdsWithAndOr(theParams, lu); } } @@ -1787,67 +1816,32 @@ public class SearchBuilder { } - private void searchForIdsWithAndOr(SearchParameterMap theParams, DateRangeParam theLastUpdated) { + private void searchForIdsWithAndOr(SearchParameterMap theParams, List thePredicates, Root theResourceTableRoot) { SearchParameterMap params = theParams; if (params == null) { params = new SearchParameterMap(); } myParams = theParams; - doInitializeSearch(); - - // RuntimeResourceDefinition resourceDef = myContext.getResourceDefinition(myResourceType); - for (Entry>> nextParamEntry : params.entrySet()) { String nextParamName = nextParamEntry.getKey(); if (nextParamName.equals(BaseResource.SP_RES_ID)) { - if (nextParamEntry.getValue().isEmpty()) { - continue; - } else { - for (List nextValue : nextParamEntry.getValue()) { - Set joinPids = new HashSet(); - if (nextValue == null || nextValue.size() == 0) { - continue; - } else { - for (IQueryParameterType next : nextValue) { - String value = next.getValueAsQueryToken(myContext); - IIdType valueId = new IdDt(value); - - try { - BaseHasResource entity = myCallingDao.readEntity(valueId); - if (entity.getDeleted() != null) { - continue; - } - joinPids.add(entity.getId()); - } catch (ResourceNotFoundException e) { - // This isn't an error, just means no result found - } - } - if (joinPids.isEmpty()) { - doSetPids(new HashSet()); - return; - } - } - - addPredicateId(joinPids); - if (doHaveNoResults()) { - return; - } - } - } + addPredicateResourceId(thePredicates, theResourceTableRoot, nextParamEntry.getValue()); } else if (nextParamName.equals(BaseResource.SP_RES_LANGUAGE)) { - addPredicateLanguage(nextParamEntry.getValue()); + addPredicateLanguage(thePredicates, theResourceTableRoot, nextParamEntry.getValue()); } else if (nextParamName.equals(Constants.PARAM_HAS)) { - addPredicateHas(nextParamEntry.getValue(), theLastUpdated); + // FIXME + addPredicateHas(nextParamEntry.getValue(), null); } else if (nextParamName.equals(Constants.PARAM_TAG) || nextParamName.equals(Constants.PARAM_PROFILE) || nextParamName.equals(Constants.PARAM_SECURITY)) { - addPredicateTag(nextParamEntry.getValue(), nextParamName, theLastUpdated); + // FIXME + addPredicateTag(nextParamEntry.getValue(), nextParamName, null); } else { @@ -1925,14 +1919,44 @@ public class SearchBuilder { } } - if (doHaveNoResults()) { - return; - } - } } + private void addPredicateResourceId(List thePredicates, Root theResourceTableRoot, List> theValues) { + for (List nextValue : theValues) { + Set orPids = new HashSet(); + for (IQueryParameterType next : nextValue) { + String value = next.getValueAsQueryToken(myContext); + IdDt valueAsId = new IdDt(value); + if (isNotBlank(value)) { + if (valueAsId.isIdPartValidLong()) { + orPids.add(valueAsId.getIdPartAsLong()); + } else { + try { +// BaseHasResource entity = myCallingDao.readEntity(valueId); +// if (entity.getDeleted() == null) { +// orPids.add(entity.getId()); +// } + } catch (ResourceNotFoundException e) { + /* + * This isn't an error, just means no result found + * that matches the ID the client provided + */ + } + } + } + + if (orPids.size() > 0) { + Predicate nextPredicate = theResourceTableRoot.get("myId").as(Long.class).in(orPids); + thePredicates.add(nextPredicate); + } + + } + + } + } + public void setType(Class theResourceType, String theResourceName) { myResourceType = theResourceType; myResourceName = theResourceName; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/DatabaseBackedPagingProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/DatabaseBackedPagingProvider.java index be65fc85d33..8d4a220cc6f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/DatabaseBackedPagingProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/DatabaseBackedPagingProvider.java @@ -39,15 +39,15 @@ import ca.uhn.fhir.rest.server.IPagingProvider; public class DatabaseBackedPagingProvider extends BasePagingProvider implements IPagingProvider { @Autowired - private PlatformTransactionManager thePlatformTransactionManager; + private PlatformTransactionManager myPlatformTransactionManager; @Autowired - private ISearchResultDao theSearchResultDao; + private ISearchResultDao mySearchResultDao; @Autowired - private EntityManager theEntityManager; + private EntityManager myEntityManager; @Autowired - private FhirContext theContext; + private FhirContext myContext; @Autowired - private IFhirSystemDao theDao; + private IFhirSystemDao myDao; /** * Constructor @@ -67,7 +67,7 @@ public class DatabaseBackedPagingProvider extends BasePagingProvider implements @Override public synchronized IBundleProvider retrieveResultList(String theId) { - PersistedJpaBundleProvider provider = new PersistedJpaBundleProvider(theId, theDao); + PersistedJpaBundleProvider provider = new PersistedJpaBundleProvider(theId, myDao); if (!provider.ensureSearchEntityLoaded()) { return null; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java index 1b9e1870eb1..0c4b0b1d807 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java @@ -118,22 +118,7 @@ public final class PersistedJpaBundleProvider implements IBundleProvider { SearchBuilder sb = myDao.newSearchBuilder(); SearchParameterMap parameterMap = SerializationUtils.deserialize(mySearchEntity.getSearchParamMap()); - sb.loadPage(parameterMap, theFromIndex, theToIndex); - - Pageable page = toPage(theFromIndex, theToIndex); - if (page == null) { - return Collections.emptyList(); - } - - Page search = mySearchResultDao.findWithSearchUuid(mySearchEntity, page); - - List pidsSubList = new ArrayList(); - for (SearchResult next : search) { - pidsSubList.add(next.getResourcePid()); - } - - // Load includes - pidsSubList = new ArrayList(pidsSubList); + List pidsSubList = sb.loadSearchPage(parameterMap, theFromIndex, theToIndex); Set revIncludedPids = new HashSet(); if (mySearchEntity.getSearchType() == SearchTypeEnum.SEARCH) { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java index 69a101fbc8b..b00b0f27090 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java @@ -522,27 +522,29 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test { @Test public void testSearchByIdParam() { - IIdType id1; + String id1; { Patient patient = new Patient(); patient.addIdentifier().setSystem("urn:system").setValue("001"); - id1 = myPatientDao.create(patient, mySrd).getId(); + id1 = myPatientDao.create(patient, mySrd).getId().getValue(); } - IIdType id2; + String id2; { Organization patient = new Organization(); patient.addIdentifier().setSystem("urn:system").setValue("001"); - id2 = myOrganizationDao.create(patient, mySrd).getId(); + id2 = myOrganizationDao.create(patient, mySrd).getId().getValue(); } Map params = new HashMap(); - params.put("_id", new StringParam(id1.getIdPart())); - assertEquals(1, toList(myPatientDao.search(params)).size()); + assertThat(toUnqualifiedVersionlessIdValues(myPatientDao.search(params)), contains(id1)); + + params.put("_id", new StringParam(id1)); + assertThat(toUnqualifiedVersionlessIdValues(myPatientDao.search(params)), contains(id1)); params.put("_id", new StringParam("9999999999999999")); assertEquals(0, toList(myPatientDao.search(params)).size()); - params.put("_id", new StringParam(id2.getIdPart())); + params.put("_id", new StringParam(id2)); assertEquals(0, toList(myPatientDao.search(params)).size()); }