From c5c154346e6957751f0037ad98ea301366025db2 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Sat, 8 Apr 2017 22:21:57 -0400 Subject: [PATCH] More work on perf --- .../fhir/jpa/dao/FulltextSearchSvcImpl.java | 7 + .../ca/uhn/fhir/jpa/dao/SearchBuilder.java | 452 +++++++++--------- .../config/TestDstu3WithoutLuceneConfig.java | 11 + 3 files changed, 255 insertions(+), 215 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java index aebb98b7765..1bda6f5509d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java @@ -66,6 +66,13 @@ public class FulltextSearchSvcImpl extends BaseHapiFhirDao implem @PersistenceContext(type = PersistenceContextType.TRANSACTION) private EntityManager myEntityManager; + /** + * Constructor + */ + public FulltextSearchSvcImpl() { + super(); + } + private void addTextSearch(QueryBuilder theQueryBuilder, BooleanJunction theBoolean, List> theTerms, String theFieldName, String theFieldNameEdgeNGram, String theFieldNameNGram) { if (theTerms == null) { return; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java index 9ef4a963f48..680b64e056c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java @@ -214,27 +214,27 @@ public class SearchBuilder { } } - private void addPredicateId(Set thePids) { - if (thePids == null || thePids.isEmpty()) { - return; - } - - CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); - CriteriaQuery cq = builder.createQuery(Long.class); - Root from = cq.from(ResourceTable.class); - cq.select(from.get("myId").as(Long.class)); - - List predicates = new ArrayList(); - predicates.add(builder.equal(from.get("myResourceType"), myResourceName)); - predicates.add(from.get("myId").in(thePids)); - createPredicateResourceId(builder, cq, predicates, from.get("myId").as(Long.class)); - createPredicateLastUpdatedForResourceTable(builder, from, predicates); - - cq.where(toArray(predicates)); - - TypedQuery q = myEntityManager.createQuery(cq); - doSetPids(q.getResultList()); - } + // private void addPredicateId(Set thePids) { + // if (thePids == null || thePids.isEmpty()) { + // return; + // } + // + // CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); + // CriteriaQuery cq = builder.createQuery(Long.class); + // Root from = cq.from(ResourceTable.class); + // cq.select(from.get("myId").as(Long.class)); + // + // List predicates = new ArrayList(); + // predicates.add(builder.equal(from.get("myResourceType"), myResourceName)); + // predicates.add(from.get("myId").in(thePids)); + // createPredicateResourceId(builder, cq, predicates, from.get("myId").as(Long.class)); + // createPredicateLastUpdatedForResourceTable(builder, from, predicates); + // + // cq.where(toArray(predicates)); + // + // TypedQuery q = myEntityManager.createQuery(cq); + // doSetPids(q.getResultList()); + // } private void addPredicateLanguage(List> theList) { for (List nextList : theList) { @@ -1319,43 +1319,43 @@ public class SearchBuilder { return new PersistedJpaBundleProvider(mySearchEntity.getUuid(), myCallingDao); } - private void doSetPids(Collection thePids) { - if (mySearchEntity.getTotalCount() != null) { - reinitializeSearch(); - } + // private void doSetPids(Collection thePids) { + // if (mySearchEntity.getTotalCount() != null) { + // reinitializeSearch(); + // } + // + // LinkedHashSet results = new LinkedHashSet(); + // int index = 0; + // for (Long next : thePids) { + // SearchResult nextResult = new SearchResult(mySearchEntity); + // nextResult.setResourcePid(next); + // nextResult.setOrder(index); + // results.add(nextResult); + // index++; + // } + // mySearchResultDao.save(results); + // + // mySearchEntity.setTotalCount(results.size()); + // mySearchEntity = myEntityManager.merge(mySearchEntity); + // + // myEntityManager.flush(); + // } - LinkedHashSet results = new LinkedHashSet(); - int index = 0; - for (Long next : thePids) { - SearchResult nextResult = new SearchResult(mySearchEntity); - nextResult.setResourcePid(next); - nextResult.setOrder(index); - results.add(nextResult); - index++; - } - mySearchResultDao.save(results); - - mySearchEntity.setTotalCount(results.size()); - mySearchEntity = myEntityManager.merge(mySearchEntity); - - myEntityManager.flush(); - } - - private void filterResourceIdsByLastUpdated(final DateRangeParam theLastUpdated) { - CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); - CriteriaQuery cq = builder.createQuery(Long.class); - Root from = cq.from(ResourceTable.class); - cq.select(from.get("myId").as(Long.class)); - - List lastUpdatedPredicates = createLastUpdatedPredicates(theLastUpdated, builder, from); - createPredicateResourceId(builder, cq, lastUpdatedPredicates, from.get("myId").as(Long.class)); - - cq.where(SearchBuilder.toArray(lastUpdatedPredicates)); - TypedQuery query = myEntityManager.createQuery(cq); - - List resultList = query.getResultList(); - doSetPids(resultList); - } + // private void filterResourceIdsByLastUpdated(final DateRangeParam theLastUpdated) { + // CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); + // CriteriaQuery cq = builder.createQuery(Long.class); + // Root from = cq.from(ResourceTable.class); + // cq.select(from.get("myId").as(Long.class)); + // + // List lastUpdatedPredicates = createLastUpdatedPredicates(theLastUpdated, builder, from); + // createPredicateResourceId(builder, cq, lastUpdatedPredicates, from.get("myId").as(Long.class)); + // + // cq.where(SearchBuilder.toArray(lastUpdatedPredicates)); + // TypedQuery query = myEntityManager.createQuery(cq); + // + // List resultList = query.getResultList(); + // doSetPids(resultList); + // } private void loadResourcesByPid(Collection theIncludePids, List theResourceListToPopulate, Set theRevIncludedPids, boolean theForHistoryOperation) { EntityManager entityManager = myEntityManager; @@ -1365,49 +1365,49 @@ public class SearchBuilder { loadResourcesByPid(theIncludePids, theResourceListToPopulate, theRevIncludedPids, theForHistoryOperation, entityManager, context, dao); } - private void processSort(final SearchParameterMap theParams) { - - // Set loadPids = theLoadPids; - if (theParams.getSort() != null && isNotBlank(theParams.getSort().getParamName())) { - List orders = new ArrayList(); - List predicates = new ArrayList(); - CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); - CriteriaQuery cq = builder.createTupleQuery(); - Root from = cq.from(ResourceTable.class); - - createPredicateResourceId(builder, cq, predicates, from.get("myId").as(Long.class)); - - createSort(builder, from, theParams.getSort(), orders, predicates); - - if (orders.size() > 0) { - - LinkedHashSet loadPids = new LinkedHashSet(); - cq.multiselect(from.get("myId").as(Long.class)); - cq.where(toArray(predicates)); - cq.orderBy(orders); - - TypedQuery query = myEntityManager.createQuery(cq); - - for (Tuple next : query.getResultList()) { - loadPids.add(next.get(0, Long.class)); - } - - ourLog.debug("Sort PID order is now: {}", loadPids); - - ArrayList pids = new ArrayList(loadPids); - - // Any ressources which weren't matched by the sort get added to the bottom - // for (Long next : originalPids) { - // if (loadPids.contains(next) == false) { - // pids.add(next); - // } - // } - - doSetPids(pids); - } - } - - } + // private void processSort(final SearchParameterMap theParams) { + // + // // Set loadPids = theLoadPids; + // if (theParams.getSort() != null && isNotBlank(theParams.getSort().getParamName())) { + // List orders = new ArrayList(); + // List predicates = new ArrayList(); + // CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); + // CriteriaQuery cq = builder.createTupleQuery(); + // Root from = cq.from(ResourceTable.class); + // + // createPredicateResourceId(builder, cq, predicates, from.get("myId").as(Long.class)); + // + // createSort(builder, from, theParams.getSort(), orders, predicates); + // + // if (orders.size() > 0) { + // + // LinkedHashSet loadPids = new LinkedHashSet(); + // cq.multiselect(from.get("myId").as(Long.class)); + // cq.where(toArray(predicates)); + // cq.orderBy(orders); + // + // TypedQuery query = myEntityManager.createQuery(cq); + // + // for (Tuple next : query.getResultList()) { + // loadPids.add(next.get(0, Long.class)); + // } + // + // ourLog.debug("Sort PID order is now: {}", loadPids); + // + // ArrayList pids = new ArrayList(loadPids); + // + // // Any ressources which weren't matched by the sort get added to the bottom + // // for (Long next : originalPids) { + // // if (loadPids.contains(next) == false) { + // // pids.add(next); + // // } + // // } + // + // doSetPids(pids); + // } + // } + // + // } private void reinitializeSearch() { mySearchEntity = new Search(); @@ -1469,23 +1469,23 @@ public class SearchBuilder { } public List loadSearchPage(SearchParameterMap theParams, int theFromIndex, int theToIndex) { - - if (myFulltextSearchSvc == null) { - if (theParams.containsKey(Constants.PARAM_TEXT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); - } else if (theParams.containsKey(Constants.PARAM_CONTENT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); - } - } else { - // FIXME: add from and to - List searchResultPids = myFulltextSearchSvc.search(myResourceName, theParams); - if (searchResultPids != null) { - if (searchResultPids.isEmpty()) { - return Collections.emptyList(); - } - return searchResultPids; - } - } + + // if (myFulltextSearchSvc == null) { + // if (theParams.containsKey(Constants.PARAM_TEXT)) { + // throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + // } else if (theParams.containsKey(Constants.PARAM_CONTENT)) { + // throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + // } + // } else { + // // FIXME: add from and to + // List searchResultPids = myFulltextSearchSvc.search(myResourceName, theParams); + // if (searchResultPids != null) { + // if (searchResultPids.isEmpty()) { + // return Collections.emptyList(); + // } + // return searchResultPids; + // } + // } myBuilder = myEntityManager.getCriteriaBuilder(); myResourceTableQuery = myBuilder.createTupleQuery(); @@ -1502,19 +1502,41 @@ public class SearchBuilder { myPredicates.addAll(lastUpdatedPredicates); if (theParams.getEverythingMode() != null) { + Join join = myResourceTableRoot.join("myResourceLinks", JoinType.LEFT); + if (theParams.get(BaseResource.SP_RES_ID) != null) { StringParam idParm = (StringParam) theParams.get(BaseResource.SP_RES_ID).get(0).get(0); Long pid = BaseHapiFhirDao.translateForcedIdToPid(myResourceName, idParm.getValue(), myForcedIdDao); - - Join join = myResourceTableRoot.join("myResourceLinks", JoinType.LEFT); myPredicates.add(myBuilder.equal(join.get("myTargetResourcePid").as(Long.class), pid)); - + } else { + myPredicates.add(myBuilder.equal(join.get("myTargetResourceType").as(String.class), myResourceName)); } + } else { // Normal search searchForIdsWithAndOr(theParams); } - + + /* + * Fulltext search + */ + if (theParams.containsKey(Constants.PARAM_CONTENT) || theParams.containsKey(Constants.PARAM_TEXT)) { + if (myFulltextSearchSvc == null) { + if (theParams.containsKey(Constants.PARAM_TEXT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + } else if (theParams.containsKey(Constants.PARAM_CONTENT)) { + throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + } + } + List pids = myFulltextSearchSvc.everything(myResourceName, theParams); + if (pids.isEmpty()) { + // Will never match + pids = Collections.singletonList((Long) null); + } + + myPredicates.add(myResourceTableRoot.get("myId").as(Long.class).in(pids)); + } + myResourceTableQuery.where(myBuilder.and(SearchBuilder.toArray(myPredicates))); myResourceTableQuery.multiselect(myResourceTableRoot.get("myId").as(Long.class)); @@ -1530,99 +1552,99 @@ public class SearchBuilder { return pids; } - public IBundleProvider loadPage(SearchParameterMap theParams, int theFromIndex, int theToIndex) { - StopWatch sw = new StopWatch(); - DateRangeParam lu = theParams.getLastUpdated(); - - // Collection loadPids; - if (theParams.getEverythingMode() != null) { - - Long pid = null; - if (theParams.get(BaseResource.SP_RES_ID) != null) { - StringParam idParm = (StringParam) theParams.get(BaseResource.SP_RES_ID).get(0).get(0); - pid = BaseHapiFhirDao.translateForcedIdToPid(myResourceName, idParm.getValue(), myForcedIdDao); - } - - if (theParams.containsKey(Constants.PARAM_CONTENT) || theParams.containsKey(Constants.PARAM_TEXT)) { - List pids = myFulltextSearchSvc.everything(myResourceName, theParams); - if (pids.isEmpty()) { - return doReturnProvider(); - } - - doSetPids(pids); - - } else { - CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); - CriteriaQuery cq = builder.createTupleQuery(); - Root from = cq.from(ResourceTable.class); - List predicates = new ArrayList(); - if (pid != null) { - predicates.add(builder.equal(from.get("myId"), pid)); - } - predicates.add(builder.equal(from.get("myResourceType"), myResourceName)); - predicates.add(builder.isNull(from.get("myDeleted"))); - cq.where(builder.and(SearchBuilder.toArray(predicates))); - - Join join = from.join("myIncomingResourceLinks", JoinType.LEFT); - cq.multiselect(from.get("myId").as(Long.class), join.get("mySourceResourcePid").as(Long.class)); - - TypedQuery query = myEntityManager.createQuery(cq); - Set pids = new HashSet(); - for (Tuple next : query.getResultList()) { - pids.add(next.get(0, Long.class)); - Long nextLong = next.get(1, Long.class); - if (nextLong != null) { - pids.add(nextLong); - } - } - doSetPids(pids); - - } - - } else { - - if (myFulltextSearchSvc == null) { - if (theParams.containsKey(Constants.PARAM_TEXT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); - } else if (theParams.containsKey(Constants.PARAM_CONTENT)) { - throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); - } - } else { - List searchResultPids = myFulltextSearchSvc.search(myResourceName, theParams); - if (searchResultPids != null) { - if (searchResultPids.isEmpty()) { - return doReturnProvider(); - } - doSetPids(searchResultPids); - } - } - - if (!theParams.isEmpty()) { - // searchForIdsWithAndOr(theParams, lu); - } - - } - - if (doHaveNoResults()) { - return doReturnProvider(); - } - - // Handle _lastUpdated - if (lu != null) { - filterResourceIdsByLastUpdated(lu); - - if (doHaveNoResults()) { - return doReturnProvider(); - } - } - - // Handle sorting if any was provided - processSort(theParams); - - ourLog.info(" {} on {} in {}ms", new Object[] { myResourceName, theParams, sw.getMillisAndRestart() }); - return doReturnProvider(); - - } + // public IBundleProvider loadPage(SearchParameterMap theParams, int theFromIndex, int theToIndex) { + // StopWatch sw = new StopWatch(); + // DateRangeParam lu = theParams.getLastUpdated(); + // + // // Collection loadPids; + // if (theParams.getEverythingMode() != null) { + // + // Long pid = null; + // if (theParams.get(BaseResource.SP_RES_ID) != null) { + // StringParam idParm = (StringParam) theParams.get(BaseResource.SP_RES_ID).get(0).get(0); + // pid = BaseHapiFhirDao.translateForcedIdToPid(myResourceName, idParm.getValue(), myForcedIdDao); + // } + // + // if (theParams.containsKey(Constants.PARAM_CONTENT) || theParams.containsKey(Constants.PARAM_TEXT)) { + // List pids = myFulltextSearchSvc.everything(myResourceName, theParams); + // if (pids.isEmpty()) { + // return doReturnProvider(); + // } + // + // doSetPids(pids); + // + // } else { + // CriteriaBuilder builder = myEntityManager.getCriteriaBuilder(); + // CriteriaQuery cq = builder.createTupleQuery(); + // Root from = cq.from(ResourceTable.class); + // List predicates = new ArrayList(); + // if (pid != null) { + // predicates.add(builder.equal(from.get("myId"), pid)); + // } + // predicates.add(builder.equal(from.get("myResourceType"), myResourceName)); + // predicates.add(builder.isNull(from.get("myDeleted"))); + // cq.where(builder.and(SearchBuilder.toArray(predicates))); + // + // Join join = from.join("myIncomingResourceLinks", JoinType.LEFT); + // cq.multiselect(from.get("myId").as(Long.class), join.get("mySourceResourcePid").as(Long.class)); + // + // TypedQuery query = myEntityManager.createQuery(cq); + // Set pids = new HashSet(); + // for (Tuple next : query.getResultList()) { + // pids.add(next.get(0, Long.class)); + // Long nextLong = next.get(1, Long.class); + // if (nextLong != null) { + // pids.add(nextLong); + // } + // } + // doSetPids(pids); + // + // } + // + // } else { + // + // if (myFulltextSearchSvc == null) { + // if (theParams.containsKey(Constants.PARAM_TEXT)) { + // throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT); + // } else if (theParams.containsKey(Constants.PARAM_CONTENT)) { + // throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT); + // } + // } else { + // List searchResultPids = myFulltextSearchSvc.search(myResourceName, theParams); + // if (searchResultPids != null) { + // if (searchResultPids.isEmpty()) { + // return doReturnProvider(); + // } + // doSetPids(searchResultPids); + // } + // } + // + // if (!theParams.isEmpty()) { + // // searchForIdsWithAndOr(theParams, lu); + // } + // + // } + // + // if (doHaveNoResults()) { + // return doReturnProvider(); + // } + // + // // Handle _lastUpdated + // if (lu != null) { + // filterResourceIdsByLastUpdated(lu); + // + // if (doHaveNoResults()) { + // return doReturnProvider(); + // } + // } + // + // // Handle sorting if any was provided + // processSort(theParams); + // + // ourLog.info(" {} on {} in {}ms", new Object[] { myResourceName, theParams, sw.getMillisAndRestart() }); + // return doReturnProvider(); + // + // } private void searchForIdsWithAndOr(SearchParameterMap theParams) { SearchParameterMap params = theParams; @@ -1708,7 +1730,7 @@ public class SearchBuilder { break; } } else { - throw new InternalErrorException("Unknown search parameter " + theParamName + " for reource type " + theResourceName); + // throw new InternalErrorException("Unknown search parameter " + theParamName + " for reource type " + theResourceName); } } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3WithoutLuceneConfig.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3WithoutLuceneConfig.java index 20bc3d6883c..d0680df9c8d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3WithoutLuceneConfig.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3WithoutLuceneConfig.java @@ -3,15 +3,26 @@ package ca.uhn.fhir.jpa.config; import java.util.Properties; import org.hibernate.jpa.HibernatePersistenceProvider; +import org.springframework.beans.factory.annotation.Autowire; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.transaction.annotation.EnableTransactionManagement; +import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl; +import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; + @Configuration @EnableTransactionManagement() public class TestDstu3WithoutLuceneConfig extends TestDstu3Config { + /** + * Disable fulltext searching + */ + public IFulltextSearchSvc searchDaoDstu3() { + return null; + } + @Override @Bean() public LocalContainerEntityManagerFactoryBean entityManagerFactory() {