honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Paging now working

This commit is contained in:
James 2017-06-19 07:35:46 -04:00
parent 526a1fa7d0
commit 571045b63d
3 changed files with 224 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java
View File

@ -87,15 +87,16 @@ public class DaoConfig {
private boolean myEnforceReferentialIntegrityOnWrite = true;
private int myEverythingIncludesFetchPageSize = 50;
/**
* update setter javadoc if default changes
*/
private long myExpireSearchResultsAfterMillis = DateUtils.MILLIS_PER_HOUR;
/**
* update setter javadoc if default changes
*/
private Integer myFetchSizeDefaultMaximum = null;
private int myHardTagListLimit = 1000;
private int myIncludeLimit = 2000;
/**
@ -147,6 +148,22 @@ public class DaoConfig {
return myDeferIndexingForCodesystemsOfSize;
}
/**
* Unlike with normal search queries, $everything queries have their _includes loaded by the main search thread and these included results
* are added to the normal search results instead of being added on as extras in a page. This means that they will not appear multiple times
* as the search results are paged over.
* <p>
* In order to recursively load _includes, we process the original results in batches of this size. Adjust with caution, increasing this
* value may improve performance but may also cause memory issues.
* </p>
* <p>
* The default value is 50
* </p>
*/
public int getEverythingIncludesFetchPageSize() {
return myEverythingIncludesFetchPageSize;
}
/**
* Sets the number of milliseconds that search results for a given client search
* should be preserved before being purged from the database.
@ -537,6 +554,23 @@ public class DaoConfig {
myEnforceReferentialIntegrityOnWrite = theEnforceReferentialIntegrityOnWrite;
}
/**
* Unlike with normal search queries, $everything queries have their _includes loaded by the main search thread and these included results
* are added to the normal search results instead of being added on as extras in a page. This means that they will not appear multiple times
* as the search results are paged over.
* <p>
* In order to recursively load _includes, we process the original results in batches of this size. Adjust with caution, increasing this
* value may improve performance but may also cause memory issues.
* </p>
* <p>
* The default value is 50
* </p>
*/
public void setEverythingIncludesFetchPageSize(int theEverythingIncludesFetchPageSize) {
Validate.inclusiveBetween(1, Integer.MAX_VALUE, theEverythingIncludesFetchPageSize);
myEverythingIncludesFetchPageSize = theEverythingIncludesFetchPageSize;
}
/**
* If this is set to <code>false</code> (default is <code>true</code>) the stale search deletion
* task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION.

145
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java
View File

@ -27,32 +27,86 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
import java.math.BigDecimal;
import java.math.MathContext;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.*;
import javax.persistence.criteria.AbstractQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaBuilder.In;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.From;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import javax.persistence.criteria.Order;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.persistence.criteria.Subquery;
import org.apache.commons.lang3.*;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.tuple.Pair;
import org.hl7.fhir.instance.model.api.*;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import com.google.common.collect.*;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import ca.uhn.fhir.context.*;
import ca.uhn.fhir.jpa.dao.SearchBuilder.IncludesIterator;
import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeDeclaredChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
import ca.uhn.fhir.context.ConfigurationException;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.FhirVersionEnum;
import ca.uhn.fhir.context.RuntimeChildChoiceDefinition;
import ca.uhn.fhir.context.RuntimeChildResourceDefinition;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao;
import ca.uhn.fhir.jpa.entity.*;
import ca.uhn.fhir.jpa.entity.BaseHasResource;
import ca.uhn.fhir.jpa.entity.BaseResourceIndexedSearchParam;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamDate;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamNumber;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamQuantity;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamString;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamToken;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamUri;
import ca.uhn.fhir.jpa.entity.ResourceLink;
import ca.uhn.fhir.jpa.entity.ResourceTable;
import ca.uhn.fhir.jpa.entity.ResourceTag;
import ca.uhn.fhir.jpa.entity.SearchParam;
import ca.uhn.fhir.jpa.entity.SearchParamPresent;
import ca.uhn.fhir.jpa.entity.TagDefinition;
import ca.uhn.fhir.jpa.entity.TagTypeEnum;
import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
import ca.uhn.fhir.jpa.term.VersionIndependentConcept;
import ca.uhn.fhir.jpa.util.StopWatch;
import ca.uhn.fhir.model.api.*;
import ca.uhn.fhir.model.base.composite.*;
import ca.uhn.fhir.model.api.IPrimitiveDatatype;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.api.IResource;
import ca.uhn.fhir.model.api.Include;
import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
import ca.uhn.fhir.model.base.composite.BaseCodingDt;
import ca.uhn.fhir.model.base.composite.BaseIdentifierDt;
import ca.uhn.fhir.model.base.composite.BaseQuantityDt;
import ca.uhn.fhir.model.dstu.resource.BaseResource;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.model.primitive.InstantDt;
@ -61,9 +115,23 @@ import ca.uhn.fhir.parser.DataFormatException;
import ca.uhn.fhir.rest.api.SortOrderEnum;
import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.method.RestSearchParameterTypeEnum;
import ca.uhn.fhir.rest.param.*;
import ca.uhn.fhir.rest.param.CompositeParam;
import ca.uhn.fhir.rest.param.DateParam;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.HasParam;
import ca.uhn.fhir.rest.param.NumberParam;
import ca.uhn.fhir.rest.param.ParamPrefixEnum;
import ca.uhn.fhir.rest.param.QuantityParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.param.TokenParamModifier;
import ca.uhn.fhir.rest.param.UriParam;
import ca.uhn.fhir.rest.param.UriParamQualifierEnum;
import ca.uhn.fhir.rest.server.Constants;
import ca.uhn.fhir.rest.server.exceptions.*;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.util.UrlUtil;
/**
@ -71,8 +139,9 @@ import ca.uhn.fhir.util.UrlUtil;
* searchs for resources
*/
public class SearchBuilder implements ISearchBuilder {
private static Long NO_MORE = Long.valueOf(-1);
private static final List<Long> EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<Long>());
private static Long NO_MORE = Long.valueOf(-1);
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchBuilder.class);
private List<Long> myAlsoIncludePids;
private CriteriaBuilder myBuilder;
@ -92,6 +161,7 @@ public class SearchBuilder implements ISearchBuilder {
private ISearchParamRegistry mySearchParamRegistry;
private String mySearchUuid;
private IHapiTerminologySvc myTerminologySvc;
/**
* Constructor
*/
@ -1967,10 +2037,47 @@ public class SearchBuilder implements ISearchBuilder {
static Predicate[] toArray(List<Predicate> thePredicates) {
return thePredicates.toArray(new Predicate[thePredicates.size()]);
}
public class IncludesIterator implements Iterator<Long>{
private Iterator<Long> myCurrentIterator;
private int myCurrentOffset;
private ArrayList<Long> myCurrentPids;
private Long myNext;
private int myPageSize = myCallingDao.getConfig().getEverythingIncludesFetchPageSize();
public IncludesIterator(Set<Long> thePidSet) {
myCurrentPids = new ArrayList<Long>(thePidSet);
myCurrentIterator = EMPTY_LONG_LIST.iterator();
myCurrentOffset = 0;
}
private void fetchNext() {
while (myNext == null) {
if (myCurrentIterator.hasNext()) {
myNext = myCurrentIterator.next();
break;
}
if (!myCurrentIterator.hasNext()) {
int start = myCurrentOffset;
int end = myCurrentOffset + myPageSize;
if (end > myCurrentPids.size()) {
end = myCurrentPids.size();
}
if (end - start <= 0) {
myNext = NO_MORE;
break;
}
myCurrentOffset = end;
Collection<Long> pidsToScan = myCurrentPids.subList(start, end);
Set<Include> includes = Collections.singleton(new Include("*", true));
Set<Long> newPids = loadReverseIncludes(myCallingDao, myContext, myEntityManager, pidsToScan, includes, false, myParams.getLastUpdated());
myCurrentIterator = newPids.iterator();
}
}
}
@Override
public boolean hasNext() {
@ -1986,14 +2093,6 @@ public class SearchBuilder implements ISearchBuilder {
return retVal;
}
private void fetchNext() {
if (myNext == null) {
}
}
}
private enum JoinEnum {
@ -2093,7 +2192,7 @@ public class SearchBuilder implements ISearchBuilder {
if (myNext == null) {
if (myStillNeedToFetchIncludes) {
myIncludesIterator = new IncludesIterator();
myIncludesIterator = new IncludesIterator(myPidSet);
myStillNeedToFetchIncludes = false;
}
if (myIncludesIterator != null) {

75
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/PatientEverythingDstu3Test.java
View File

@ -15,13 +15,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.startsWith;
import static org.hamcrest.Matchers.stringContainsInOrder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assert.*;
import java.io.*;
import java.net.*;
@ -83,6 +77,8 @@ public class PatientEverythingDstu3Test extends BaseResourceProviderDstu3Test {
private String encId1;
private String encId2;
private ArrayList<String> myObsIds;
private String myWrongPatId;
private String myWrongEnc1;
@Before
public void beforeDisableResultReuse() {
@ -95,6 +91,7 @@ public class PatientEverythingDstu3Test extends BaseResourceProviderDstu3Test {
super.after();
myDaoConfig.setReuseCachedSearchResultsForMillis(new DaoConfig().getReuseCachedSearchResultsForMillis());
myDaoConfig.setEverythingIncludesFetchPageSize(new DaoConfig().getEverythingIncludesFetchPageSize());
}
@Override
@ -107,6 +104,7 @@ public class PatientEverythingDstu3Test extends BaseResourceProviderDstu3Test {
Organization org = new Organization();
org.setName("an org");
orgId = ourClient.create().resource(org).execute().getId().toUnqualifiedVersionless().getValue();
ourLog.info("OrgId: {}", orgId);
Patient patient = new Patient();
patient.getManagingOrganization().setReference(orgId);
@ -114,18 +112,26 @@ public class PatientEverythingDstu3Test extends BaseResourceProviderDstu3Test {
Patient patient2 = new Patient();
patient2.getManagingOrganization().setReference(orgId);
ourClient.create().resource(patient2).execute().getId().toUnqualifiedVersionless().getValue();
myWrongPatId = ourClient.create().resource(patient2).execute().getId().toUnqualifiedVersionless().getValue();
Encounter enc1 = new Encounter();
enc1.setStatus(EncounterStatus.CANCELLED);
enc1.getSubject().setReference(patId);
enc1.getServiceProvider().setReference(orgId);
encId1 = ourClient.create().resource(enc1).execute().getId().toUnqualifiedVersionless().getValue();
Encounter enc2 = new Encounter();
enc2.setStatus(EncounterStatus.ARRIVED);
enc2.getSubject().setReference(patId);
enc2.getServiceProvider().setReference(orgId);
encId2 = ourClient.create().resource(enc2).execute().getId().toUnqualifiedVersionless().getValue();
Encounter wrongEnc1 = new Encounter();
wrongEnc1.setStatus(EncounterStatus.ARRIVED);
wrongEnc1.getSubject().setReference(myWrongPatId);
wrongEnc1.getServiceProvider().setReference(orgId);
myWrongEnc1 = ourClient.create().resource(wrongEnc1).execute().getId().toUnqualifiedVersionless().getValue();
myObsIds = new ArrayList<String>();
for (int i = 0; i < 20; i++) {
Observation obs = new Observation();
@ -137,6 +143,59 @@ public class PatientEverythingDstu3Test extends BaseResourceProviderDstu3Test {
}
/**
* See #674
*/
@Test
public void testEverythingReturnsCorrectResources() throws Exception {
Bundle bundle = fetchBundle(ourServerBase + "/" + patId + "/$everything?_format=json&_count=100", EncodingEnum.JSON);
assertNull(bundle.getLink("next"));
Set<String> actual = new TreeSet<String>();
for (BundleEntryComponent nextEntry : bundle.getEntry()) {
actual.add(nextEntry.getResource().getIdElement().toUnqualifiedVersionless().getValue());
}
ourLog.info("Found IDs: {}", actual);
assertThat(actual, hasItem(patId));
assertThat(actual, hasItem(encId1));
assertThat(actual, hasItem(encId2));
assertThat(actual, hasItem(orgId));
assertThat(actual, hasItems(myObsIds.toArray(new String[0])));
assertThat(actual, not(hasItem(myWrongPatId)));
assertThat(actual, not(hasItem(myWrongEnc1)));
}
/**
* See #674
*/
@Test
public void testEverythingReturnsCorrectResourcesSmallPage() throws Exception {
myDaoConfig.setEverythingIncludesFetchPageSize(1);
Bundle bundle = fetchBundle(ourServerBase + "/" + patId + "/$everything?_format=json&_count=100", EncodingEnum.JSON);
assertNull(bundle.getLink("next"));
Set<String> actual = new TreeSet<String>();
for (BundleEntryComponent nextEntry : bundle.getEntry()) {
actual.add(nextEntry.getResource().getIdElement().toUnqualifiedVersionless().getValue());
}
ourLog.info("Found IDs: {}", actual);
assertThat(actual, hasItem(patId));
assertThat(actual, hasItem(encId1));
assertThat(actual, hasItem(encId2));
assertThat(actual, hasItem(orgId));
assertThat(actual, hasItems(myObsIds.toArray(new String[0])));
assertThat(actual, not(hasItem(myWrongPatId)));
assertThat(actual, not(hasItem(myWrongEnc1)));
}
/**
* See #674
*/