Fixes to enable $lastn to return more than 32K records.
This commit is contained in:
ianmarshall
parent
1e882d640d
commit
c290fa3493
|
|
@ -0,0 +1,15 @@
|
|||
diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java
|
||||
index e575041cd9..93e364bc93 100644
|
||||
--- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java
|
||||
+++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java
|
||||
@@ -49,8 +49,8 @@ import java.util.List;
|
||||
public class DeleteConflictService {
|
||||
private static final Logger ourLog = LoggerFactory.getLogger(DeleteConflictService.class);
|
||||
public static final int FIRST_QUERY_RESULT_COUNT = 1;
|
||||
- public static final int RETRY_QUERY_RESULT_COUNT = 60;
|
||||
- public static final int MAX_RETRY_ATTEMPTS = 10;
|
||||
+ public static final int RETRY_QUERY_RESULT_COUNT = 100;
|
||||
+ public static final int MAX_RETRY_ATTEMPTS = 100;
|
||||
|
||||
@Autowired
|
||||
DeleteConflictFinderService myDeleteConflictFinderService;
|
||||
|
|
@ -37,6 +37,7 @@ public abstract class BaseHapiFhirResourceDaoObservation<T extends IBaseResource
|
|||
theSearchParameterMap.setLastN(true);
|
||||
if (theSearchParameterMap.getSort() == null) {
|
||||
SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC);
|
||||
// TODO: Should probably remove these constants, maybe move this logic to the version-specific classes.
|
||||
SortSpec observationCode = new SortSpec(IndexConstants.CODE_SEARCH_PARAM).setOrder(SortOrderEnum.ASC).setChain(effectiveDtm);
|
||||
theSearchParameterMap.setSort(new SortSpec(IndexConstants.SUBJECT_SEARCH_PARAM).setOrder(SortOrderEnum.ASC).setChain(observationCode));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ import ca.uhn.fhir.jpa.searchparam.JpaRuntimeSearchParam;
|
|||
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
|
||||
import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
|
||||
import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper;
|
||||
import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
|
||||
import ca.uhn.fhir.jpa.util.BaseIterator;
|
||||
import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
|
||||
import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster;
|
||||
|
|
@ -113,7 +114,6 @@ import java.util.Iterator;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.apache.commons.lang3.StringUtils.defaultString;
|
||||
import static org.apache.commons.lang3.StringUtils.isBlank;
|
||||
|
|
@ -131,6 +131,8 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
*/
|
||||
// NB: keep public
|
||||
public static final int MAXIMUM_PAGE_SIZE = 800;
|
||||
public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 4;
|
||||
public static boolean myIsTest = false;
|
||||
|
||||
private static final List<ResourcePersistentId> EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<>());
|
||||
private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class);
|
||||
|
|
@ -182,6 +184,18 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
myResourceType = theResourceType;
|
||||
}
|
||||
|
||||
public static int getMaximumPageSize() {
|
||||
if (myIsTest) {
|
||||
return MAXIMUM_PAGE_SIZE_FOR_TESTING;
|
||||
} else {
|
||||
return MAXIMUM_PAGE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
public static void setIsTest(boolean theIsTest) {
|
||||
myIsTest = theIsTest;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setMaxResultsToFetch(Integer theMaxResultsToFetch) {
|
||||
myMaxResultsToFetch = theMaxResultsToFetch;
|
||||
|
|
@ -210,6 +224,10 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
// Handle each parameter
|
||||
for (Map.Entry<String, List<List<IQueryParameterType>>> nextParamEntry : myParams.entrySet()) {
|
||||
String nextParamName = nextParamEntry.getKey();
|
||||
if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) {
|
||||
// Skip parameters for Subject, Patient, Code and Category for LastN
|
||||
continue;
|
||||
}
|
||||
List<List<IQueryParameterType>> andOrParams = nextParamEntry.getValue();
|
||||
searchForIdsWithAndOr(myResourceName, nextParamName, andOrParams, theRequest);
|
||||
}
|
||||
|
|
@ -231,8 +249,8 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
|
||||
init(theParams, theSearchUuid, theRequestPartitionId);
|
||||
|
||||
TypedQuery<Long> query = createQuery(null, null, true, theRequest);
|
||||
return new CountQueryIterator(query);
|
||||
List<TypedQuery<Long>> queries = createQuery(null, null, true, theRequest);
|
||||
return new CountQueryIterator(queries.get(0));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -265,7 +283,72 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
}
|
||||
|
||||
|
||||
private TypedQuery<Long> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) {
|
||||
private List<TypedQuery<Long>> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) {
|
||||
List<ResourcePersistentId> pids = new ArrayList<>();
|
||||
|
||||
/*
|
||||
* Fulltext or lastn search
|
||||
*/
|
||||
if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) {
|
||||
if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) {
|
||||
if (myFulltextSearchSvc == null) {
|
||||
if (myParams.containsKey(Constants.PARAM_TEXT)) {
|
||||
throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT);
|
||||
} else if (myParams.containsKey(Constants.PARAM_CONTENT)) {
|
||||
throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT);
|
||||
}
|
||||
}
|
||||
|
||||
if (myParams.getEverythingMode() != null) {
|
||||
pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest);
|
||||
} else {
|
||||
pids = myFulltextSearchSvc.search(myResourceName, myParams);
|
||||
}
|
||||
} else if (myParams.isLastN()) {
|
||||
if (myIElasticsearchSvc == null) {
|
||||
if (myParams.isLastN()) {
|
||||
throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request");
|
||||
}
|
||||
}
|
||||
Integer myMaxObservationsPerCode = null;
|
||||
if(myParams.getLastNMax() != null) {
|
||||
myMaxObservationsPerCode = myParams.getLastNMax();
|
||||
} else {
|
||||
throw new InvalidRequestException("Max parameter is required for $lastn operation");
|
||||
}
|
||||
List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode);
|
||||
for (String lastnResourceId : lastnResourceIds) {
|
||||
pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId));
|
||||
}
|
||||
}
|
||||
if (pids.isEmpty()) {
|
||||
// Will never match
|
||||
pids = Collections.singletonList(new ResourcePersistentId(-1L));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ArrayList<TypedQuery<Long>> myQueries = new ArrayList<>();
|
||||
|
||||
if (!pids.isEmpty()) {
|
||||
new QueryChunker<Long>().chunk(ResourcePersistentId.toLongList(pids), t->{
|
||||
doCreateChunkedQueries(t, sort, theMaximumResults, theCount, theRequest, myQueries);
|
||||
});
|
||||
} else {
|
||||
myQueries.add(createQuery(sort,theMaximumResults, theCount, theRequest, null));
|
||||
}
|
||||
|
||||
return myQueries;
|
||||
}
|
||||
|
||||
private void doCreateChunkedQueries(List<Long> thePids, SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, ArrayList<TypedQuery<Long>> theQueries) {
|
||||
if(thePids.size() < MAXIMUM_PAGE_SIZE) {
|
||||
thePids = normalizeIdListForLastNInClause(thePids);
|
||||
}
|
||||
theQueries.add(createQuery(sort, theMaximumResults, theCount, theRequest, thePids));
|
||||
}
|
||||
|
||||
private TypedQuery<Long> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List<Long> thePidList) {
|
||||
CriteriaQuery<Long> outerQuery;
|
||||
/*
|
||||
* Sort
|
||||
|
|
@ -329,7 +412,7 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
/*
|
||||
* Fulltext or lastn search
|
||||
*/
|
||||
if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) {
|
||||
/* if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) {
|
||||
List<ResourcePersistentId> pids = new ArrayList<>();
|
||||
if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) {
|
||||
if (myFulltextSearchSvc == null) {
|
||||
|
|
@ -352,19 +435,16 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
}
|
||||
}
|
||||
Integer myMaxObservationsPerCode = null;
|
||||
// String[] maxCountParams = theRequest.getParameters().get("max");
|
||||
// if (maxCountParams != null && maxCountParams.length > 0) {
|
||||
// myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]);
|
||||
if(myParams.getLastNMax() != null) {
|
||||
myMaxObservationsPerCode = myParams.getLastNMax();
|
||||
} else {
|
||||
throw new InvalidRequestException("Max parameter is required for $lastn operation");
|
||||
}
|
||||
List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode);
|
||||
// for (String lastnResourceId : lastnResourceIds) {
|
||||
// pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId));
|
||||
// }
|
||||
pids = normalizeIdListForLastNInClause(lastnResourceIds);
|
||||
for (String lastnResourceId : lastnResourceIds) {
|
||||
pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId));
|
||||
}
|
||||
// pids = normalizeIdListForLastNInClause(lastnResourceIds);
|
||||
}
|
||||
if (pids.isEmpty()) {
|
||||
// Will never match
|
||||
|
|
@ -374,6 +454,11 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids)));
|
||||
|
||||
}
|
||||
*/
|
||||
// Add PID list predicate for full text search and/or lastn operation
|
||||
if (thePidList != null && thePidList.size() > 0) {
|
||||
myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(thePidList));
|
||||
}
|
||||
|
||||
/*
|
||||
* Add a predicate to make sure we only include non-deleted resources, and only include
|
||||
|
|
@ -415,10 +500,10 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
return query;
|
||||
}
|
||||
|
||||
private List<ResourcePersistentId> normalizeIdListForLastNInClause(List<String> lastnResourceIds) {
|
||||
List<ResourcePersistentId> retVal = new ArrayList<>();
|
||||
for (String lastnResourceId : lastnResourceIds) {
|
||||
retVal.add(new ResourcePersistentId(Long.parseLong(lastnResourceId)));
|
||||
private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) {
|
||||
List<Long> retVal = new ArrayList<>();
|
||||
for (Long lastnResourceId : lastnResourceIds) {
|
||||
retVal.add(lastnResourceId);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -430,32 +515,27 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
arguments never exceeds the maximum specified below.
|
||||
*/
|
||||
int listSize = retVal.size();
|
||||
|
||||
if(listSize > 1 && listSize < 10) {
|
||||
padIdListWithPlaceholders(retVal, 10);
|
||||
} else if (listSize > 10 && listSize < 100) {
|
||||
} else if (listSize > 10 && listSize < 50) {
|
||||
padIdListWithPlaceholders(retVal, 50);
|
||||
} else if (listSize > 50 && listSize < 100) {
|
||||
padIdListWithPlaceholders(retVal, 100);
|
||||
} else if (listSize > 100 && listSize < 200) {
|
||||
padIdListWithPlaceholders(retVal, 200);
|
||||
} else if (listSize > 200 && listSize < 500) {
|
||||
padIdListWithPlaceholders(retVal, 500);
|
||||
} else if (listSize > 500 && listSize < 1000) {
|
||||
padIdListWithPlaceholders(retVal, 1000);
|
||||
} else if (listSize > 1000 && listSize < 500) {
|
||||
padIdListWithPlaceholders(retVal, 5000);
|
||||
} else if (listSize > 5000 && listSize < 10000) {
|
||||
padIdListWithPlaceholders(retVal, 10000);
|
||||
} else if (listSize > 10000 && listSize < 20000) {
|
||||
padIdListWithPlaceholders(retVal, 20000);
|
||||
} else if (listSize > 20000 && listSize < 30000) {
|
||||
padIdListWithPlaceholders(retVal, 30000);
|
||||
} else if (listSize > 500 && listSize < 800) {
|
||||
padIdListWithPlaceholders(retVal, 800);
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
private void padIdListWithPlaceholders(List<ResourcePersistentId> theIdList, int preferredListSize) {
|
||||
private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
|
||||
while(theIdList.size() < preferredListSize) {
|
||||
theIdList.add(new ResourcePersistentId(-1L));
|
||||
theIdList.add(-1L);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -733,7 +813,7 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
if (matchAll) {
|
||||
String sql;
|
||||
sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r." + searchFieldName + " IN (:target_pids) ";
|
||||
List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, MAXIMUM_PAGE_SIZE);
|
||||
List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
|
||||
for (Collection<ResourcePersistentId> nextPartition : partitions) {
|
||||
TypedQuery<Long> q = theEntityManager.createQuery(sql, Long.class);
|
||||
q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition));
|
||||
|
|
@ -786,7 +866,7 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids)";
|
||||
}
|
||||
|
||||
List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, MAXIMUM_PAGE_SIZE);
|
||||
List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
|
||||
for (Collection<ResourcePersistentId> nextPartition : partitions) {
|
||||
TypedQuery<Long> q = theEntityManager.createQuery(sql, Long.class);
|
||||
q.setParameter("src_path", nextPath);
|
||||
|
|
@ -1076,6 +1156,8 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
private int mySkipCount = 0;
|
||||
private int myNonSkipCount = 0;
|
||||
|
||||
private List<TypedQuery<Long>> myQueryList;
|
||||
|
||||
private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
|
||||
mySearchRuntimeDetails = theSearchRuntimeDetails;
|
||||
mySort = myParams.getSort();
|
||||
|
|
@ -1126,7 +1208,12 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
}
|
||||
|
||||
if (myNext == null) {
|
||||
while (myResultsIterator.hasNext()) {
|
||||
while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) {
|
||||
// Update iterator with next chunk if necessary.
|
||||
if (!myResultsIterator.hasNext() && !myQueryList.isEmpty()) {
|
||||
retrieveNextIteratorQuery();
|
||||
}
|
||||
|
||||
Long nextLong = myResultsIterator.next();
|
||||
if (myHavePerfTraceFoundIdHook) {
|
||||
HookParams params = new HookParams()
|
||||
|
|
@ -1225,19 +1312,31 @@ public class SearchBuilder implements ISearchBuilder {
|
|||
}
|
||||
|
||||
private void initializeIteratorQuery(Integer theMaxResultsToFetch) {
|
||||
final TypedQuery<Long> query = createQuery(mySort, theMaxResultsToFetch, false, myRequest);
|
||||
if (myQueryList == null || myQueryList.isEmpty()) {
|
||||
myQueryList = createQuery(mySort, theMaxResultsToFetch, false, myRequest);
|
||||
}
|
||||
|
||||
mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
|
||||
|
||||
Query<Long> hibernateQuery = (Query<Long>) query;
|
||||
hibernateQuery.setFetchSize(myFetchSize);
|
||||
ScrollableResults scroll = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY);
|
||||
myResultsIterator = new ScrollableResultsIterator<>(scroll);
|
||||
retrieveNextIteratorQuery();
|
||||
|
||||
mySkipCount = 0;
|
||||
myNonSkipCount = 0;
|
||||
}
|
||||
|
||||
private void retrieveNextIteratorQuery() {
|
||||
if (myQueryList != null && myQueryList.size() > 0) {
|
||||
final TypedQuery<Long> query = myQueryList.remove(0);
|
||||
Query<Long> hibernateQuery = (Query<Long>) (query);
|
||||
hibernateQuery.setFetchSize(myFetchSize);
|
||||
ScrollableResults scroll = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY);
|
||||
myResultsIterator = new ScrollableResultsIterator<>(scroll);
|
||||
} else {
|
||||
myResultsIterator = null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (myNext == null) {
|
||||
|
|
|
|||
|
|
@ -13,79 +13,80 @@ import java.util.*;
|
|||
@Indexed(index = "observation_index")
|
||||
public class ObservationIndexedSearchParamLastNEntity {
|
||||
|
||||
@Id
|
||||
@SequenceGenerator(name = "SEQ_LASTN", sequenceName = "SEQ_LASTN")
|
||||
@GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_LASTN")
|
||||
@Column(name = "LASTN_ID")
|
||||
private Long myId;
|
||||
@Id
|
||||
@SequenceGenerator(name = "SEQ_LASTN", sequenceName = "SEQ_LASTN")
|
||||
@GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_LASTN")
|
||||
@Column(name = "LASTN_ID")
|
||||
private Long myId;
|
||||
|
||||
@Field(name = "subject", analyze = Analyze.NO)
|
||||
@Column(name = "LASTN_SUBJECT_ID", nullable = true)
|
||||
private String mySubject;
|
||||
@Column(name = "LASTN_SUBJECT_ID", nullable = true)
|
||||
private String mySubject;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_OBSERVATION_CODE_FK"))
|
||||
@IndexedEmbedded(depth = 2, prefix = "codeconcept")
|
||||
private ObservationIndexedCodeCodeableConceptEntity myObservationCode;
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_OBSERVATION_CODE_FK"))
|
||||
@IndexedEmbedded(depth = 2, prefix = "codeconcept")
|
||||
private ObservationIndexedCodeCodeableConceptEntity myObservationCode;
|
||||
|
||||
@Field(name = "codeconceptid", analyze = Analyze.NO)
|
||||
@Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false)
|
||||
private String myCodeNormalizedId;
|
||||
@Field(name = "codeconceptid", analyze = Analyze.NO)
|
||||
@Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false)
|
||||
private String myCodeNormalizedId;
|
||||
|
||||
@IndexedEmbedded(depth = 2, prefix = "categoryconcept")
|
||||
@Transient
|
||||
private Set<ObservationIndexedCategoryCodeableConceptEntity> myCategoryCodeableConcepts;
|
||||
@IndexedEmbedded(depth = 2, prefix = "categoryconcept")
|
||||
@Transient
|
||||
private Set<ObservationIndexedCategoryCodeableConceptEntity> myCategoryCodeableConcepts;
|
||||
|
||||
@Field(name = "effectivedtm", analyze = Analyze.NO)
|
||||
@Temporal(TemporalType.TIMESTAMP)
|
||||
@Column(name = "LASTN_EFFECTIVE_DATETIME", nullable = true)
|
||||
private Date myEffectiveDtm;
|
||||
@Field(name = "effectivedtm", analyze = Analyze.NO)
|
||||
@Temporal(TemporalType.TIMESTAMP)
|
||||
@Column(name = "LASTN_EFFECTIVE_DATETIME", nullable = true)
|
||||
private Date myEffectiveDtm;
|
||||
|
||||
@DocumentId(name = "identifier")
|
||||
@Column(name = "RESOURCE_IDENTIFIER", nullable = false)
|
||||
private String myIdentifier;
|
||||
@DocumentId(name = "identifier")
|
||||
@Column(name = "RESOURCE_IDENTIFIER", nullable = false)
|
||||
private String myIdentifier;
|
||||
|
||||
public ObservationIndexedSearchParamLastNEntity() {}
|
||||
public ObservationIndexedSearchParamLastNEntity() {
|
||||
}
|
||||
|
||||
public String getSubject() {
|
||||
return mySubject;
|
||||
}
|
||||
public String getSubject() {
|
||||
return mySubject;
|
||||
}
|
||||
|
||||
public void setSubject(String theSubject) {
|
||||
mySubject = theSubject;
|
||||
}
|
||||
public void setSubject(String theSubject) {
|
||||
mySubject = theSubject;
|
||||
}
|
||||
|
||||
public String getIdentifier() {
|
||||
return myIdentifier;
|
||||
}
|
||||
public String getIdentifier() {
|
||||
return myIdentifier;
|
||||
}
|
||||
|
||||
public void setIdentifier(String theIdentifier) {
|
||||
myIdentifier = theIdentifier;
|
||||
}
|
||||
public void setIdentifier(String theIdentifier) {
|
||||
myIdentifier = theIdentifier;
|
||||
}
|
||||
|
||||
public void setEffectiveDtm(Date theEffectiveDtm) {
|
||||
myEffectiveDtm = theEffectiveDtm;
|
||||
}
|
||||
public void setEffectiveDtm(Date theEffectiveDtm) {
|
||||
myEffectiveDtm = theEffectiveDtm;
|
||||
}
|
||||
|
||||
public Date getEffectiveDtm() {
|
||||
return myEffectiveDtm;
|
||||
}
|
||||
public Date getEffectiveDtm() {
|
||||
return myEffectiveDtm;
|
||||
}
|
||||
|
||||
public void setCodeNormalizedId(String theCodeNormalizedId) {
|
||||
myCodeNormalizedId = theCodeNormalizedId;
|
||||
}
|
||||
public void setCodeNormalizedId(String theCodeNormalizedId) {
|
||||
myCodeNormalizedId = theCodeNormalizedId;
|
||||
}
|
||||
|
||||
public String getCodeNormalizedId() {
|
||||
return myCodeNormalizedId;
|
||||
}
|
||||
public String getCodeNormalizedId() {
|
||||
return myCodeNormalizedId;
|
||||
}
|
||||
|
||||
|
||||
public void setObservationCode(ObservationIndexedCodeCodeableConceptEntity theObservationCode) {
|
||||
myObservationCode = theObservationCode;
|
||||
}
|
||||
public void setObservationCode(ObservationIndexedCodeCodeableConceptEntity theObservationCode) {
|
||||
myObservationCode = theObservationCode;
|
||||
}
|
||||
|
||||
public void setCategoryCodeableConcepts(Set<ObservationIndexedCategoryCodeableConceptEntity> theCategoryCodeableConcepts) {
|
||||
myCategoryCodeableConcepts = theCategoryCodeableConcepts;
|
||||
}
|
||||
public void setCategoryCodeableConcepts(Set<ObservationIndexedCategoryCodeableConceptEntity> theCategoryCodeableConcepts) {
|
||||
myCategoryCodeableConcepts = theCategoryCodeableConcepts;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -200,6 +200,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
|
|||
}
|
||||
|
||||
@Override
|
||||
// TODO: Should eliminate dependency on SearchParameterMap in API.
|
||||
public List<String> executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) {
|
||||
String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME};
|
||||
try {
|
||||
|
|
@ -252,6 +253,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
|
|||
}
|
||||
|
||||
@VisibleForTesting
|
||||
// TODO: Should eliminate dependency on SearchParameterMap in API.
|
||||
List<ObservationJson> executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) {
|
||||
try {
|
||||
List<SearchResponse> responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, null);
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.search.lastn;
|
|||
|
||||
public class IndexConstants {
|
||||
|
||||
// TODO: These should all be moved into ElasticSearchSvcImpl.
|
||||
public static final String OBSERVATION_INDEX = "observation_index";
|
||||
public static final String CODE_INDEX = "code_index";
|
||||
public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity";
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ import java.util.function.Consumer;
|
|||
public class QueryChunker<T> {
|
||||
|
||||
public void chunk(List<T> theInput, Consumer<List<T>> theBatchConsumer) {
|
||||
for (int i = 0; i < theInput.size(); i += SearchBuilder.MAXIMUM_PAGE_SIZE) {
|
||||
int to = i + SearchBuilder.MAXIMUM_PAGE_SIZE;
|
||||
for (int i = 0; i < theInput.size(); i += SearchBuilder.getMaximumPageSize()) {
|
||||
int to = i + SearchBuilder.getMaximumPageSize();
|
||||
to = Math.min(to, theInput.size());
|
||||
List<T> batch = theInput.subList(i, to);
|
||||
theBatchConsumer.accept(batch);
|
||||
|
|
|
|||
|
|
@ -5,13 +5,16 @@ import ca.uhn.fhir.jpa.api.dao.*;
|
|||
import ca.uhn.fhir.jpa.api.config.DaoConfig;
|
||||
import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient;
|
||||
import ca.uhn.fhir.jpa.dao.BaseJpaTest;
|
||||
import ca.uhn.fhir.jpa.dao.SearchBuilder;
|
||||
import ca.uhn.fhir.jpa.rp.r4.ObservationResourceProvider;
|
||||
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
|
||||
import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener;
|
||||
import ca.uhn.fhir.rest.param.*;
|
||||
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
|
||||
import ca.uhn.fhir.util.TestUtil;
|
||||
import org.hl7.fhir.instance.model.api.IIdType;
|
||||
import org.hl7.fhir.r4.model.*;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
|
@ -23,7 +26,9 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
|
|||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.hamcrest.Matchers.matchesPattern;
|
||||
import static org.junit.Assert.*;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
|
|
@ -58,6 +63,9 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
|
|||
return myPlatformTransactionManager;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
protected CircularQueueCaptureQueriesListener myCaptureQueriesListener;
|
||||
|
||||
ObservationResourceProvider observationRp = new ObservationResourceProvider();
|
||||
|
||||
private final String observationCd0 = "code0";
|
||||
|
|
@ -108,6 +116,11 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
|
|||
|
||||
}
|
||||
|
||||
@After
|
||||
public void resetMaximumPageSize() {
|
||||
SearchBuilder.setIsTest(false);
|
||||
}
|
||||
|
||||
private void createObservationsForPatient(IIdType thePatientId) {
|
||||
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15);
|
||||
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10);
|
||||
|
|
@ -192,9 +205,6 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
|
|||
params.setLastN(true);
|
||||
|
||||
Map<String, String[]> requestParameters = new HashMap<>();
|
||||
// String[] maxParam = new String[1];
|
||||
// maxParam[0] = "100";
|
||||
// requestParameters.put("max", maxParam);
|
||||
params.setLastNMax(100);
|
||||
|
||||
when(mySrd.getParameters()).thenReturn(requestParameters);
|
||||
|
|
@ -520,6 +530,82 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
|
|||
return new TokenAndListParam().addAnd(myTokenOrListParam);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLastNWithChunkedQuery() {
|
||||
SearchBuilder.setIsTest(true);
|
||||
Integer numberOfObservations = SearchBuilder.getMaximumPageSize()+1;
|
||||
Calendar observationDate = new GregorianCalendar();
|
||||
|
||||
List<IIdType> myObservationIds = new ArrayList<>();
|
||||
List<IIdType> myPatientIds = new ArrayList<>();
|
||||
List<ReferenceParam> myPatientReferences = new ArrayList<>();
|
||||
for (int idx=0; idx<numberOfObservations; idx++ ) {
|
||||
Patient pt = new Patient();
|
||||
pt.addName().setFamily("Lastn_" + idx).addGiven("Chunked");
|
||||
IIdType patientId = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
|
||||
myPatientIds.add(patientId);
|
||||
ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId.getValue());
|
||||
myPatientReferences.add(subjectParam);
|
||||
Observation obs = new Observation();
|
||||
obs.getSubject().setReferenceElement(patientId);
|
||||
obs.getCode().addCoding().setCode(observationCd0).setSystem(codeSystem);
|
||||
obs.setValue(new StringType(observationCd0 + "_0"));
|
||||
observationDate.add(Calendar.HOUR, -1);
|
||||
Date effectiveDtm = observationDate.getTime();
|
||||
obs.setEffective(new DateTimeType(effectiveDtm));
|
||||
obs.getCategoryFirstRep().addCoding().setCode(categoryCd0).setSystem(categorySystem);
|
||||
myObservationIds.add(myObservationDao.create(obs, mockSrd()).getId());
|
||||
}
|
||||
|
||||
SearchParameterMap params = new SearchParameterMap();
|
||||
ReferenceParam[] referenceParams = new ReferenceParam[numberOfObservations];
|
||||
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(myPatientReferences.toArray(referenceParams)));
|
||||
|
||||
TokenParam codeParam = new TokenParam(codeSystem, observationCd0);
|
||||
params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
|
||||
|
||||
TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0);
|
||||
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
|
||||
|
||||
List<String> actual;
|
||||
params.setLastN(true);
|
||||
|
||||
Map<String, String[]> requestParameters = new HashMap<>();
|
||||
params.setLastNMax(1);
|
||||
|
||||
params.setCount(numberOfObservations);
|
||||
|
||||
when(mySrd.getParameters()).thenReturn(requestParameters);
|
||||
|
||||
myCaptureQueriesListener.clear();
|
||||
actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null));
|
||||
|
||||
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
|
||||
List<String> queries = myCaptureQueriesListener
|
||||
.getSelectQueriesForCurrentThread()
|
||||
.stream()
|
||||
.map(t -> t.getSql(true, false))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// First chunked query
|
||||
String resultingQueryNotFormatted = queries.get(0);
|
||||
assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '[0-9]+' , '[0-9]+' , '[0-9]+'\\).*"));
|
||||
|
||||
// Second chunked query chunk
|
||||
resultingQueryNotFormatted = queries.get(1);
|
||||
assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '-1' , '-1' , '-1'\\).*"));
|
||||
|
||||
assertEquals(numberOfObservations, (Integer)actual.size());
|
||||
for(IIdType observationId : myObservationIds) {
|
||||
myObservationDao.delete(observationId);
|
||||
}
|
||||
|
||||
for (IIdType patientId : myPatientIds) {
|
||||
myPatientDao.delete(patientId);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterClassClearContext() {
|
||||
TestUtil.clearAllStaticFieldsForUnitTest();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
package ca.uhn.fhir.jpa.searchparam.util;
|
||||
|
||||
import ca.uhn.fhir.context.FhirContext;
|
||||
import ca.uhn.fhir.context.FhirVersionEnum;
|
||||
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
|
||||
|
||||
public class LastNParameterHelper {
|
||||
|
||||
public static boolean isLastNParameter(String theParamName, FhirContext theContext) {
|
||||
if (theParamName == null) {
|
||||
return false;
|
||||
}
|
||||
if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) {
|
||||
if (theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_PATIENT)
|
||||
|| theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else if (theContext.getVersion().getVersion() == FhirVersionEnum.R4) {
|
||||
if (theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_PATIENT)
|
||||
|| theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CODE)) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else if (theContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) {
|
||||
if (theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_PATIENT)
|
||||
|| theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CODE)) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString());
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue