honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixes to enable $lastn to return more than 32K records.

This commit is contained in:
ianmarshall 2020-05-08 09:19:14 -04:00
parent 1e882d640d
commit c290fa3493
9 changed files with 340 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
DeleteConflictService_fix.patch Normal file
View File

@ -0,0 +1,15 @@
diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java
index e575041cd9..93e364bc93 100644
--- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java
+++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteConflictService.java
@@ -49,8 +49,8 @@ import java.util.List;
public class DeleteConflictService {
private static final Logger ourLog = LoggerFactory.getLogger(DeleteConflictService.class);
public static final int FIRST_QUERY_RESULT_COUNT = 1;
- public static final int RETRY_QUERY_RESULT_COUNT = 60;
- public static final int MAX_RETRY_ATTEMPTS = 10;
+ public static final int RETRY_QUERY_RESULT_COUNT = 100;
+ public static final int MAX_RETRY_ATTEMPTS = 100;
@Autowired
DeleteConflictFinderService myDeleteConflictFinderService;

1
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java
View File

@ -37,6 +37,7 @@ public abstract class BaseHapiFhirResourceDaoObservation<T extends IBaseResource
theSearchParameterMap.setLastN(true); theSearchParameterMap.setLastN(true);
if (theSearchParameterMap.getSort() == null) { if (theSearchParameterMap.getSort() == null) {
SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC); SortSpec effectiveDtm = new SortSpec("date").setOrder(SortOrderEnum.DESC);
// TODO: Should probably remove these constants, maybe move this logic to the version-specific classes.
SortSpec observationCode = new SortSpec(IndexConstants.CODE_SEARCH_PARAM).setOrder(SortOrderEnum.ASC).setChain(effectiveDtm); SortSpec observationCode = new SortSpec(IndexConstants.CODE_SEARCH_PARAM).setOrder(SortOrderEnum.ASC).setChain(effectiveDtm);
theSearchParameterMap.setSort(new SortSpec(IndexConstants.SUBJECT_SEARCH_PARAM).setOrder(SortOrderEnum.ASC).setChain(observationCode)); theSearchParameterMap.setSort(new SortSpec(IndexConstants.SUBJECT_SEARCH_PARAM).setOrder(SortOrderEnum.ASC).setChain(observationCode));
} }

173
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java
View File

@ -54,6 +54,7 @@ import ca.uhn.fhir.jpa.searchparam.JpaRuntimeSearchParam;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry; import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper; import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper;
import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
import ca.uhn.fhir.jpa.util.BaseIterator; import ca.uhn.fhir.jpa.util.BaseIterator;
import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener; import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster; import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster;
@ -113,7 +114,6 @@ import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.commons.lang3.StringUtils.defaultString; import static org.apache.commons.lang3.StringUtils.defaultString;
import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isBlank;
@ -131,6 +131,8 @@ public class SearchBuilder implements ISearchBuilder {
*/ */
// NB: keep public // NB: keep public
public static final int MAXIMUM_PAGE_SIZE = 800; public static final int MAXIMUM_PAGE_SIZE = 800;
public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 4;
public static boolean myIsTest = false;
private static final List<ResourcePersistentId> EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<>()); private static final List<ResourcePersistentId> EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<>());
private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class); private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class);
@ -182,6 +184,18 @@ public class SearchBuilder implements ISearchBuilder {
myResourceType = theResourceType; myResourceType = theResourceType;
} }
public static int getMaximumPageSize() {
if (myIsTest) {
return MAXIMUM_PAGE_SIZE_FOR_TESTING;
} else {
return MAXIMUM_PAGE_SIZE;
}
}
public static void setIsTest(boolean theIsTest) {
myIsTest = theIsTest;
}
@Override @Override
public void setMaxResultsToFetch(Integer theMaxResultsToFetch) { public void setMaxResultsToFetch(Integer theMaxResultsToFetch) {
myMaxResultsToFetch = theMaxResultsToFetch; myMaxResultsToFetch = theMaxResultsToFetch;
@ -210,6 +224,10 @@ public class SearchBuilder implements ISearchBuilder {
// Handle each parameter // Handle each parameter
for (Map.Entry<String, List<List<IQueryParameterType>>> nextParamEntry : myParams.entrySet()) { for (Map.Entry<String, List<List<IQueryParameterType>>> nextParamEntry : myParams.entrySet()) {
String nextParamName = nextParamEntry.getKey(); String nextParamName = nextParamEntry.getKey();
if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) {
// Skip parameters for Subject, Patient, Code and Category for LastN
continue;
}
List<List<IQueryParameterType>> andOrParams = nextParamEntry.getValue(); List<List<IQueryParameterType>> andOrParams = nextParamEntry.getValue();
searchForIdsWithAndOr(myResourceName, nextParamName, andOrParams, theRequest); searchForIdsWithAndOr(myResourceName, nextParamName, andOrParams, theRequest);
} }
@ -231,8 +249,8 @@ public class SearchBuilder implements ISearchBuilder {
init(theParams, theSearchUuid, theRequestPartitionId); init(theParams, theSearchUuid, theRequestPartitionId);
TypedQuery<Long> query = createQuery(null, null, true, theRequest); List<TypedQuery<Long>> queries = createQuery(null, null, true, theRequest);
return new CountQueryIterator(query); return new CountQueryIterator(queries.get(0));
} }
/** /**
@ -265,7 +283,72 @@ public class SearchBuilder implements ISearchBuilder {
} }
private TypedQuery<Long> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) { private List<TypedQuery<Long>> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest) {
List<ResourcePersistentId> pids = new ArrayList<>();
/*
* Fulltext or lastn search
*/
if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) {
if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) {
if (myFulltextSearchSvc == null) {
if (myParams.containsKey(Constants.PARAM_TEXT)) {
throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT);
} else if (myParams.containsKey(Constants.PARAM_CONTENT)) {
throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT);
}
}
if (myParams.getEverythingMode() != null) {
pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest);
} else {
pids = myFulltextSearchSvc.search(myResourceName, myParams);
}
} else if (myParams.isLastN()) {
if (myIElasticsearchSvc == null) {
if (myParams.isLastN()) {
throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request");
}
}
Integer myMaxObservationsPerCode = null;
if(myParams.getLastNMax() != null) {
myMaxObservationsPerCode = myParams.getLastNMax();
} else {
throw new InvalidRequestException("Max parameter is required for $lastn operation");
}
List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode);
for (String lastnResourceId : lastnResourceIds) {
pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId));
}
}
if (pids.isEmpty()) {
// Will never match
pids = Collections.singletonList(new ResourcePersistentId(-1L));
}
}
ArrayList<TypedQuery<Long>> myQueries = new ArrayList<>();
if (!pids.isEmpty()) {
new QueryChunker<Long>().chunk(ResourcePersistentId.toLongList(pids), t->{
doCreateChunkedQueries(t, sort, theMaximumResults, theCount, theRequest, myQueries);
});
} else {
myQueries.add(createQuery(sort,theMaximumResults, theCount, theRequest, null));
}
return myQueries;
}
private void doCreateChunkedQueries(List<Long> thePids, SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, ArrayList<TypedQuery<Long>> theQueries) {
if(thePids.size() < MAXIMUM_PAGE_SIZE) {
thePids = normalizeIdListForLastNInClause(thePids);
}
theQueries.add(createQuery(sort, theMaximumResults, theCount, theRequest, thePids));
}
private TypedQuery<Long> createQuery(SortSpec sort, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List<Long> thePidList) {
CriteriaQuery<Long> outerQuery; CriteriaQuery<Long> outerQuery;
/* /*
* Sort * Sort
@ -329,7 +412,7 @@ public class SearchBuilder implements ISearchBuilder {
/* /*
* Fulltext or lastn search * Fulltext or lastn search
*/ */
if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) { /* if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) {
List<ResourcePersistentId> pids = new ArrayList<>(); List<ResourcePersistentId> pids = new ArrayList<>();
if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) { if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) {
if (myFulltextSearchSvc == null) { if (myFulltextSearchSvc == null) {
@ -352,19 +435,16 @@ public class SearchBuilder implements ISearchBuilder {
} }
} }
Integer myMaxObservationsPerCode = null; Integer myMaxObservationsPerCode = null;
// String[] maxCountParams = theRequest.getParameters().get("max");
// if (maxCountParams != null && maxCountParams.length > 0) {
// myMaxObservationsPerCode = Integer.valueOf(maxCountParams[0]);
if(myParams.getLastNMax() != null) { if(myParams.getLastNMax() != null) {
myMaxObservationsPerCode = myParams.getLastNMax(); myMaxObservationsPerCode = myParams.getLastNMax();
} else { } else {
throw new InvalidRequestException("Max parameter is required for $lastn operation"); throw new InvalidRequestException("Max parameter is required for $lastn operation");
} }
List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode); List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myMaxObservationsPerCode);
// for (String lastnResourceId : lastnResourceIds) { for (String lastnResourceId : lastnResourceIds) {
// pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)); pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId));
// } }
pids = normalizeIdListForLastNInClause(lastnResourceIds); // pids = normalizeIdListForLastNInClause(lastnResourceIds);
} }
if (pids.isEmpty()) { if (pids.isEmpty()) {
// Will never match // Will never match
@ -374,6 +454,11 @@ public class SearchBuilder implements ISearchBuilder {
myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids))); myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(pids)));
} }
*/
// Add PID list predicate for full text search and/or lastn operation
if (thePidList != null && thePidList.size() > 0) {
myQueryRoot.addPredicate(myQueryRoot.get("myId").as(Long.class).in(thePidList));
}
/* /*
* Add a predicate to make sure we only include non-deleted resources, and only include * Add a predicate to make sure we only include non-deleted resources, and only include
@ -415,10 +500,10 @@ public class SearchBuilder implements ISearchBuilder {
return query; return query;
} }
private List<ResourcePersistentId> normalizeIdListForLastNInClause(List<String> lastnResourceIds) { private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) {
List<ResourcePersistentId> retVal = new ArrayList<>(); List<Long> retVal = new ArrayList<>();
for (String lastnResourceId : lastnResourceIds) { for (Long lastnResourceId : lastnResourceIds) {
retVal.add(new ResourcePersistentId(Long.parseLong(lastnResourceId))); retVal.add(lastnResourceId);
} }
/* /*
@ -430,32 +515,27 @@ public class SearchBuilder implements ISearchBuilder {
arguments never exceeds the maximum specified below. arguments never exceeds the maximum specified below.
*/ */
int listSize = retVal.size(); int listSize = retVal.size();
if(listSize > 1 && listSize < 10) { if(listSize > 1 && listSize < 10) {
padIdListWithPlaceholders(retVal, 10); padIdListWithPlaceholders(retVal, 10);
} else if (listSize > 10 && listSize < 100) { } else if (listSize > 10 && listSize < 50) {
padIdListWithPlaceholders(retVal, 50);
} else if (listSize > 50 && listSize < 100) {
padIdListWithPlaceholders(retVal, 100); padIdListWithPlaceholders(retVal, 100);
} else if (listSize > 100 && listSize < 200) { } else if (listSize > 100 && listSize < 200) {
padIdListWithPlaceholders(retVal, 200); padIdListWithPlaceholders(retVal, 200);
} else if (listSize > 200 && listSize < 500) { } else if (listSize > 200 && listSize < 500) {
padIdListWithPlaceholders(retVal, 500); padIdListWithPlaceholders(retVal, 500);
} else if (listSize > 500 && listSize < 1000) { } else if (listSize > 500 && listSize < 800) {
padIdListWithPlaceholders(retVal, 1000); padIdListWithPlaceholders(retVal, 800);
} else if (listSize > 1000 && listSize < 500) {
padIdListWithPlaceholders(retVal, 5000);
} else if (listSize > 5000 && listSize < 10000) {
padIdListWithPlaceholders(retVal, 10000);
} else if (listSize > 10000 && listSize < 20000) {
padIdListWithPlaceholders(retVal, 20000);
} else if (listSize > 20000 && listSize < 30000) {
padIdListWithPlaceholders(retVal, 30000);
} }
return retVal; return retVal;
} }
private void padIdListWithPlaceholders(List<ResourcePersistentId> theIdList, int preferredListSize) { private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
while(theIdList.size() < preferredListSize) { while(theIdList.size() < preferredListSize) {
theIdList.add(new ResourcePersistentId(-1L)); theIdList.add(-1L);
} }
} }
@ -733,7 +813,7 @@ public class SearchBuilder implements ISearchBuilder {
if (matchAll) { if (matchAll) {
String sql; String sql;
sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r." + searchFieldName + " IN (:target_pids) "; sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r." + searchFieldName + " IN (:target_pids) ";
List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, MAXIMUM_PAGE_SIZE); List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
for (Collection<ResourcePersistentId> nextPartition : partitions) { for (Collection<ResourcePersistentId> nextPartition : partitions) {
TypedQuery<Long> q = theEntityManager.createQuery(sql, Long.class); TypedQuery<Long> q = theEntityManager.createQuery(sql, Long.class);
q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition)); q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition));
@ -786,7 +866,7 @@ public class SearchBuilder implements ISearchBuilder {
sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids)"; sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids)";
} }
List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, MAXIMUM_PAGE_SIZE); List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
for (Collection<ResourcePersistentId> nextPartition : partitions) { for (Collection<ResourcePersistentId> nextPartition : partitions) {
TypedQuery<Long> q = theEntityManager.createQuery(sql, Long.class); TypedQuery<Long> q = theEntityManager.createQuery(sql, Long.class);
q.setParameter("src_path", nextPath); q.setParameter("src_path", nextPath);
@ -1076,6 +1156,8 @@ public class SearchBuilder implements ISearchBuilder {
private int mySkipCount = 0; private int mySkipCount = 0;
private int myNonSkipCount = 0; private int myNonSkipCount = 0;
private List<TypedQuery<Long>> myQueryList;
private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) { private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
mySearchRuntimeDetails = theSearchRuntimeDetails; mySearchRuntimeDetails = theSearchRuntimeDetails;
mySort = myParams.getSort(); mySort = myParams.getSort();
@ -1126,7 +1208,12 @@ public class SearchBuilder implements ISearchBuilder {
} }
if (myNext == null) { if (myNext == null) {
while (myResultsIterator.hasNext()) { while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) {
// Update iterator with next chunk if necessary.
if (!myResultsIterator.hasNext() && !myQueryList.isEmpty()) {
retrieveNextIteratorQuery();
}
Long nextLong = myResultsIterator.next(); Long nextLong = myResultsIterator.next();
if (myHavePerfTraceFoundIdHook) { if (myHavePerfTraceFoundIdHook) {
HookParams params = new HookParams() HookParams params = new HookParams()
@ -1225,19 +1312,31 @@ public class SearchBuilder implements ISearchBuilder {
} }
private void initializeIteratorQuery(Integer theMaxResultsToFetch) { private void initializeIteratorQuery(Integer theMaxResultsToFetch) {
final TypedQuery<Long> query = createQuery(mySort, theMaxResultsToFetch, false, myRequest); if (myQueryList == null || myQueryList.isEmpty()) {
myQueryList = createQuery(mySort, theMaxResultsToFetch, false, myRequest);
}
mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
Query<Long> hibernateQuery = (Query<Long>) query; retrieveNextIteratorQuery();
hibernateQuery.setFetchSize(myFetchSize);
ScrollableResults scroll = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY);
myResultsIterator = new ScrollableResultsIterator<>(scroll);
mySkipCount = 0; mySkipCount = 0;
myNonSkipCount = 0; myNonSkipCount = 0;
} }
private void retrieveNextIteratorQuery() {
if (myQueryList != null && myQueryList.size() > 0) {
final TypedQuery<Long> query = myQueryList.remove(0);
Query<Long> hibernateQuery = (Query<Long>) (query);
hibernateQuery.setFetchSize(myFetchSize);
ScrollableResults scroll = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY);
myResultsIterator = new ScrollableResultsIterator<>(scroll);
} else {
myResultsIterator = null;
}
}
@Override @Override
public boolean hasNext() { public boolean hasNext() {
if (myNext == null) { if (myNext == null) {

111
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/lastn/entity/ObservationIndexedSearchParamLastNEntity.java
View File

@ -13,79 +13,80 @@ import java.util.*;
@Indexed(index = "observation_index") @Indexed(index = "observation_index")
public class ObservationIndexedSearchParamLastNEntity { public class ObservationIndexedSearchParamLastNEntity {
@Id @Id
@SequenceGenerator(name = "SEQ_LASTN", sequenceName = "SEQ_LASTN") @SequenceGenerator(name = "SEQ_LASTN", sequenceName = "SEQ_LASTN")
@GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_LASTN") @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_LASTN")
@Column(name = "LASTN_ID") @Column(name = "LASTN_ID")
private Long myId; private Long myId;
@Field(name = "subject", analyze = Analyze.NO) @Field(name = "subject", analyze = Analyze.NO)
@Column(name = "LASTN_SUBJECT_ID", nullable = true) @Column(name = "LASTN_SUBJECT_ID", nullable = true)
private String mySubject; private String mySubject;
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_OBSERVATION_CODE_FK")) @JoinColumn(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_OBSERVATION_CODE_FK"))
@IndexedEmbedded(depth = 2, prefix = "codeconcept") @IndexedEmbedded(depth = 2, prefix = "codeconcept")
private ObservationIndexedCodeCodeableConceptEntity myObservationCode; private ObservationIndexedCodeCodeableConceptEntity myObservationCode;
@Field(name = "codeconceptid", analyze = Analyze.NO) @Field(name = "codeconceptid", analyze = Analyze.NO)
@Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false) @Column(name = "CODEABLE_CONCEPT_ID", nullable = false, updatable = false, insertable = false)
private String myCodeNormalizedId; private String myCodeNormalizedId;
@IndexedEmbedded(depth = 2, prefix = "categoryconcept") @IndexedEmbedded(depth = 2, prefix = "categoryconcept")
@Transient @Transient
private Set<ObservationIndexedCategoryCodeableConceptEntity> myCategoryCodeableConcepts; private Set<ObservationIndexedCategoryCodeableConceptEntity> myCategoryCodeableConcepts;
@Field(name = "effectivedtm", analyze = Analyze.NO) @Field(name = "effectivedtm", analyze = Analyze.NO)
@Temporal(TemporalType.TIMESTAMP) @Temporal(TemporalType.TIMESTAMP)
@Column(name = "LASTN_EFFECTIVE_DATETIME", nullable = true) @Column(name = "LASTN_EFFECTIVE_DATETIME", nullable = true)
private Date myEffectiveDtm; private Date myEffectiveDtm;
@DocumentId(name = "identifier") @DocumentId(name = "identifier")
@Column(name = "RESOURCE_IDENTIFIER", nullable = false) @Column(name = "RESOURCE_IDENTIFIER", nullable = false)
private String myIdentifier; private String myIdentifier;
public ObservationIndexedSearchParamLastNEntity() {} public ObservationIndexedSearchParamLastNEntity() {
}
public String getSubject() { public String getSubject() {
return mySubject; return mySubject;
} }
public void setSubject(String theSubject) { public void setSubject(String theSubject) {
mySubject = theSubject; mySubject = theSubject;
} }
public String getIdentifier() { public String getIdentifier() {
return myIdentifier; return myIdentifier;
} }
public void setIdentifier(String theIdentifier) { public void setIdentifier(String theIdentifier) {
myIdentifier = theIdentifier; myIdentifier = theIdentifier;
} }
public void setEffectiveDtm(Date theEffectiveDtm) { public void setEffectiveDtm(Date theEffectiveDtm) {
myEffectiveDtm = theEffectiveDtm; myEffectiveDtm = theEffectiveDtm;
} }
public Date getEffectiveDtm() { public Date getEffectiveDtm() {
return myEffectiveDtm; return myEffectiveDtm;
} }
public void setCodeNormalizedId(String theCodeNormalizedId) { public void setCodeNormalizedId(String theCodeNormalizedId) {
myCodeNormalizedId = theCodeNormalizedId; myCodeNormalizedId = theCodeNormalizedId;
} }
public String getCodeNormalizedId() { public String getCodeNormalizedId() {
return myCodeNormalizedId; return myCodeNormalizedId;
} }
public void setObservationCode(ObservationIndexedCodeCodeableConceptEntity theObservationCode) { public void setObservationCode(ObservationIndexedCodeCodeableConceptEntity theObservationCode) {
myObservationCode = theObservationCode; myObservationCode = theObservationCode;
} }
public void setCategoryCodeableConcepts(Set<ObservationIndexedCategoryCodeableConceptEntity> theCategoryCodeableConcepts) { public void setCategoryCodeableConcepts(Set<ObservationIndexedCategoryCodeableConceptEntity> theCategoryCodeableConcepts) {
myCategoryCodeableConcepts = theCategoryCodeableConcepts; myCategoryCodeableConcepts = theCategoryCodeableConcepts;
} }
} }

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java
View File

@ -200,6 +200,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
} }
@Override @Override
// TODO: Should eliminate dependency on SearchParameterMap in API.
public List<String> executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { public List<String> executeLastN(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) {
String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME};
try { try {
@ -252,6 +253,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
} }
@VisibleForTesting @VisibleForTesting
// TODO: Should eliminate dependency on SearchParameterMap in API.
List<ObservationJson> executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) { List<ObservationJson> executeLastNWithAllFields(SearchParameterMap theSearchParameterMap, Integer theMaxObservationsPerCode) {
try { try {
List<SearchResponse> responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, null); List<SearchResponse> responses = buildAndExecuteSearch(theSearchParameterMap, theMaxObservationsPerCode, null);

1
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/IndexConstants.java
View File

@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.search.lastn;
public class IndexConstants { public class IndexConstants {
// TODO: These should all be moved into ElasticSearchSvcImpl.
public static final String OBSERVATION_INDEX = "observation_index"; public static final String OBSERVATION_INDEX = "observation_index";
public static final String CODE_INDEX = "code_index"; public static final String CODE_INDEX = "code_index";
public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity"; public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.dao.lastn.entity.ObservationIndexedSearchParamLastNEntity";

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/QueryChunker.java
View File

@ -34,8 +34,8 @@ import java.util.function.Consumer;
public class QueryChunker<T> { public class QueryChunker<T> {
public void chunk(List<T> theInput, Consumer<List<T>> theBatchConsumer) { public void chunk(List<T> theInput, Consumer<List<T>> theBatchConsumer) {
for (int i = 0; i < theInput.size(); i += SearchBuilder.MAXIMUM_PAGE_SIZE) { for (int i = 0; i < theInput.size(); i += SearchBuilder.getMaximumPageSize()) {
int to = i + SearchBuilder.MAXIMUM_PAGE_SIZE; int to = i + SearchBuilder.getMaximumPageSize();
to = Math.min(to, theInput.size()); to = Math.min(to, theInput.size());
List<T> batch = theInput.subList(i, to); List<T> batch = theInput.subList(i, to);
theBatchConsumer.accept(batch); theBatchConsumer.accept(batch);

92
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNIT.java
View File

@ -5,13 +5,16 @@ import ca.uhn.fhir.jpa.api.dao.*;
import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient; import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticsearchClient;
import ca.uhn.fhir.jpa.dao.BaseJpaTest; import ca.uhn.fhir.jpa.dao.BaseJpaTest;
import ca.uhn.fhir.jpa.dao.SearchBuilder;
import ca.uhn.fhir.jpa.rp.r4.ObservationResourceProvider; import ca.uhn.fhir.jpa.rp.r4.ObservationResourceProvider;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener;
import ca.uhn.fhir.rest.param.*; import ca.uhn.fhir.rest.param.*;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.util.TestUtil; import ca.uhn.fhir.util.TestUtil;
import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.*; import org.hl7.fhir.r4.model.*;
import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@ -23,7 +26,9 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.PlatformTransactionManager;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.matchesPattern;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ -58,6 +63,9 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
return myPlatformTransactionManager; return myPlatformTransactionManager;
} }
@Autowired
protected CircularQueueCaptureQueriesListener myCaptureQueriesListener;
ObservationResourceProvider observationRp = new ObservationResourceProvider(); ObservationResourceProvider observationRp = new ObservationResourceProvider();
private final String observationCd0 = "code0"; private final String observationCd0 = "code0";
@ -108,6 +116,11 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
} }
@After
public void resetMaximumPageSize() {
SearchBuilder.setIsTest(false);
}
private void createObservationsForPatient(IIdType thePatientId) { private void createObservationsForPatient(IIdType thePatientId) {
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15); createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd0, 15);
createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10); createFiveObservationsForPatientCodeCategory(thePatientId,observationCd0, categoryCd1, 10);
@ -192,9 +205,6 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
params.setLastN(true); params.setLastN(true);
Map<String, String[]> requestParameters = new HashMap<>(); Map<String, String[]> requestParameters = new HashMap<>();
// String[] maxParam = new String[1];
// maxParam[0] = "100";
// requestParameters.put("max", maxParam);
params.setLastNMax(100); params.setLastNMax(100);
when(mySrd.getParameters()).thenReturn(requestParameters); when(mySrd.getParameters()).thenReturn(requestParameters);
@ -520,6 +530,82 @@ public class FhirResourceDaoR4SearchLastNIT extends BaseJpaTest {
return new TokenAndListParam().addAnd(myTokenOrListParam); return new TokenAndListParam().addAnd(myTokenOrListParam);
} }
@Test
public void testLastNWithChunkedQuery() {
SearchBuilder.setIsTest(true);
Integer numberOfObservations = SearchBuilder.getMaximumPageSize()+1;
Calendar observationDate = new GregorianCalendar();
List<IIdType> myObservationIds = new ArrayList<>();
List<IIdType> myPatientIds = new ArrayList<>();
List<ReferenceParam> myPatientReferences = new ArrayList<>();
for (int idx=0; idx<numberOfObservations; idx++ ) {
Patient pt = new Patient();
pt.addName().setFamily("Lastn_" + idx).addGiven("Chunked");
IIdType patientId = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
myPatientIds.add(patientId);
ReferenceParam subjectParam = new ReferenceParam("Patient", "", patientId.getValue());
myPatientReferences.add(subjectParam);
Observation obs = new Observation();
obs.getSubject().setReferenceElement(patientId);
obs.getCode().addCoding().setCode(observationCd0).setSystem(codeSystem);
obs.setValue(new StringType(observationCd0 + "_0"));
observationDate.add(Calendar.HOUR, -1);
Date effectiveDtm = observationDate.getTime();
obs.setEffective(new DateTimeType(effectiveDtm));
obs.getCategoryFirstRep().addCoding().setCode(categoryCd0).setSystem(categorySystem);
myObservationIds.add(myObservationDao.create(obs, mockSrd()).getId());
}
SearchParameterMap params = new SearchParameterMap();
ReferenceParam[] referenceParams = new ReferenceParam[numberOfObservations];
params.add(Observation.SP_SUBJECT, buildReferenceAndListParam(myPatientReferences.toArray(referenceParams)));
TokenParam codeParam = new TokenParam(codeSystem, observationCd0);
params.add(Observation.SP_CODE, buildTokenAndListParam(codeParam));
TokenParam categoryParam = new TokenParam(categorySystem, categoryCd0);
params.add(Observation.SP_CATEGORY, buildTokenAndListParam(categoryParam));
List<String> actual;
params.setLastN(true);
Map<String, String[]> requestParameters = new HashMap<>();
params.setLastNMax(1);
params.setCount(numberOfObservations);
when(mySrd.getParameters()).thenReturn(requestParameters);
myCaptureQueriesListener.clear();
actual = toUnqualifiedVersionlessIdValues(myObservationDao.observationsLastN(params, mockSrd(),null));
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
List<String> queries = myCaptureQueriesListener
.getSelectQueriesForCurrentThread()
.stream()
.map(t -> t.getSql(true, false))
.collect(Collectors.toList());
// First chunked query
String resultingQueryNotFormatted = queries.get(0);
assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '[0-9]+' , '[0-9]+' , '[0-9]+'\\).*"));
// Second chunked query chunk
resultingQueryNotFormatted = queries.get(1);
assertThat(resultingQueryNotFormatted, matchesPattern(".*RES_ID in \\('[0-9]+' , '-1' , '-1' , '-1'\\).*"));
assertEquals(numberOfObservations, (Integer)actual.size());
for(IIdType observationId : myObservationIds) {
myObservationDao.delete(observationId);
}
for (IIdType patientId : myPatientIds) {
myPatientDao.delete(patientId);
}
}
@AfterClass @AfterClass
public static void afterClassClearContext() { public static void afterClassClearContext() {
TestUtil.clearAllStaticFieldsForUnitTest(); TestUtil.clearAllStaticFieldsForUnitTest();

38
hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/LastNParameterHelper.java Normal file
View File

@ -0,0 +1,38 @@
package ca.uhn.fhir.jpa.searchparam.util;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.FhirVersionEnum;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
public class LastNParameterHelper {
public static boolean isLastNParameter(String theParamName, FhirContext theContext) {
if (theParamName == null) {
return false;
}
if (theContext.getVersion().getVersion() == FhirVersionEnum.R5) {
if (theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_PATIENT)
|| theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r5.model.Observation.SP_CODE)) {
return true;
} else {
return false;
}
} else if (theContext.getVersion().getVersion() == FhirVersionEnum.R4) {
if (theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_PATIENT)
|| theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.r4.model.Observation.SP_CODE)) {
return true;
} else {
return false;
}
} else if (theContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) {
if (theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_SUBJECT) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_PATIENT)
|| theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CATEGORY) || theParamName.equals(org.hl7.fhir.dstu3.model.Observation.SP_CODE)) {
return true;
} else {
return false;
}
} else {
throw new InvalidRequestException("$lastn operation is not implemented for FHIR Version " + theContext.getVersion().getVersion().getFhirVersionString());
}
}
}