honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

2838 lastn refactor to use extended lucene index (#3339) 

* Change token and reference search params to 'keyword' field type

* remove normalizer for keyword field

* WIP - refactor lastN operation to use hibernate search generated ES index.

* Add ES native aggregation builder for lastN

* added date search param support for searching resourcetable ES index

* * Added DateSearch parameterized tests to ElasticSearch test suite

* Added more tests cases for Date searches

* WIP refactor LastN test data to central generator class and use it for both hibernate-search and elastic search implementations.

* WIP fix lastNDataGenerator to avoid assertion errors.

* Fixed tests to run test suite on both existing lastN search implementation and newer extended lucene index based search.

* Cleanup json string to avoid carriage return noise

* Fix checkstyle issues

* Fix checkstyle Msg code issues

* Fix checkstyle Msg code issues

* Add more test case for GT, SA precision modifier

* Fix date precision test cases

* Fix Msg Code conflicts

* Add more logging

* Add way more loggging

* even more logging

* fix msg code conflicts

Co-authored-by: Tadgh <garygrantgraham@gmail.com>
This commit is contained in:
Jaison B 2022-02-10 04:05:31 -07:00 committed by GitHub
parent 0f2fc7a882
commit 7794113455
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 967 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java
View File

@ -25,7 +25,7 @@ public final class Msg {
/**
* IMPORTANT: Please update the following comment after you add a new code
* Last code value: 2031
* Last code value: 2034
*/
private Msg() {}

5
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java
View File

@ -41,6 +41,7 @@ import ca.uhn.fhir.jpa.delete.DeleteConflictUtil;
import ca.uhn.fhir.jpa.model.entity.BaseHasResource;
import ca.uhn.fhir.jpa.model.entity.BaseTag;
import ca.uhn.fhir.jpa.model.entity.ForcedId;
import ca.uhn.fhir.jpa.model.entity.PartitionablePartitionId;
import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.entity.TagDefinition;
@ -258,7 +259,9 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
ResourceTable entity = new ResourceTable();
entity.setResourceType(toResourceName(theResource));
entity.setPartitionId(myRequestPartitionHelperService.toStoragePartition(theRequestPartitionId));
PartitionablePartitionId partitionablePartitionId = myRequestPartitionHelperService.toStoragePartition(theRequestPartitionId);
ourLog.info("Setting Entity for resource {} to partition id {} which has date {}", theResource, partitionablePartitionId, partitionablePartitionId.getPartitionDate());
entity.setPartitionId(partitionablePartitionId);
entity.setCreatedByMatchUrl(theIfNoneExist);
entity.setVersion(1);

9
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDaoObservation.java
View File

@ -33,6 +33,7 @@ import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
import ca.uhn.fhir.rest.param.ReferenceOrListParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.springframework.beans.factory.annotation.Autowired;
@ -120,9 +121,11 @@ public abstract class BaseHapiFhirResourceDaoObservation<T extends IBaseResource
theSearchParameterMap.remove(getSubjectParamName());
theSearchParameterMap.remove(getPatientParamName());
for (Long subjectPid : orderedSubjectReferenceMap.keySet()) {
theSearchParameterMap.add(getSubjectParamName(), orderedSubjectReferenceMap.get(subjectPid));
}
// Subject PIDs ordered - so create 'OR' list of subjects for lastN operation
ReferenceOrListParam orList = new ReferenceOrListParam();
orderedSubjectReferenceMap.keySet().forEach(key -> orList.addOr((ReferenceParam) orderedSubjectReferenceMap.get(key)));
theSearchParameterMap.add(getSubjectParamName(), orList);
}
}

16
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java
View File

@ -27,6 +27,7 @@ import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneClauseBuilder;
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneIndexExtractor;
import ca.uhn.fhir.jpa.dao.search.ExtendedLuceneSearchBuilder;
import ca.uhn.fhir.jpa.dao.search.LastNOperation;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.search.ExtendedLuceneIndexData;
import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions;
@ -97,9 +98,7 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
// keep this in sync with the guts of doSearch
boolean requiresHibernateSearchAccess = myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN();
requiresHibernateSearchAccess |=
myDaoConfig.isAdvancedLuceneIndexing() &&
myAdvancedIndexQueryBuilder.isSupportsSomeOf(myParams);
requiresHibernateSearchAccess |= myDaoConfig.isAdvancedLuceneIndexing() && myAdvancedIndexQueryBuilder.isSupportsSomeOf(myParams);
return requiresHibernateSearchAccess;
}
@ -236,8 +235,13 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
ValueSetAutocompleteSearch autocomplete = new ValueSetAutocompleteSearch(myFhirContext, getSearchSession());
IBaseResource result = autocomplete.search(theOptions);
return result;
return autocomplete.search(theOptions);
}
@Override
public List<ResourcePersistentId> lastN(SearchParameterMap theParams, Integer theMaximumResults) {
List<Long> pidList = new LastNOperation(getSearchSession(), myFhirContext, mySearchParamRegistry)
.executeLastN(theParams, theMaximumResults);
return convertLongsToResourcePersistentIds(pidList);
}
}

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java
View File

@ -22,6 +22,7 @@ package ca.uhn.fhir.jpa.dao;
import java.util.List;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.search.ExtendedLuceneIndexData;
import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions;
@ -39,7 +40,7 @@ public interface IFulltextSearchSvc {
* consuming entries from theParams when used to query.
*
* @param theResourceName the resource name to restrict the query.
* @param theParams the full query - modified to return only params unused by the index.
* @param theParams the full query - modified to return only params unused by the index.
* @return the pid list for the matchign resources.
*/
List<ResourcePersistentId> search(String theResourceName, SearchParameterMap theParams);
@ -57,7 +58,7 @@ public interface IFulltextSearchSvc {
ExtendedLuceneIndexData extractLuceneIndexData(IBaseResource theResource, ResourceIndexedSearchParams theNewParams);
boolean supportsSomeOf(SearchParameterMap myParams);
boolean supportsSomeOf(SearchParameterMap myParams);
/**
* Re-publish the resource to the full-text index.
@ -69,4 +70,6 @@ public interface IFulltextSearchSvc {
*/
void reindex(ResourceTable theEntity);
List<ResourcePersistentId> lastN(SearchParameterMap theParams, Integer theMaximumResults);
}

1
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/ObservationLastNIndexPersistSvc.java
View File

@ -134,7 +134,6 @@ public class ObservationLastNIndexPersistSvc {
myElasticsearchSvc.createOrUpdateObservationCodeIndex(codeableConceptField.getCodeableConceptId(), codeableConceptField);
theIndexedObservation.setCode(codeableConceptField);
theIndexedObservation.setCode_concept_id(codeableConceptField.getCodeableConceptId());
}
private void addCategoriesToObservationIndex(List<IBase> observationCategoryCodeableConcepts,

224
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedLuceneClauseBuilder.java
View File

@ -23,12 +23,19 @@ package ca.uhn.fhir.jpa.dao.search;
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.api.TemporalPrecisionEnum;
import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.param.DateParam;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.ParamPrefixEnum;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.util.DateUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.hibernate.search.engine.search.common.BooleanOperator;
import org.hibernate.search.engine.search.predicate.dsl.BooleanPredicateClausesStep;
import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep;
@ -37,8 +44,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import java.time.Instant;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -54,6 +66,8 @@ public class ExtendedLuceneClauseBuilder {
final SearchPredicateFactory myPredicateFactory;
final BooleanPredicateClausesStep<?> myRootClause;
final List<TemporalPrecisionEnum> ordinalSearchPrecisions = Arrays.asList(TemporalPrecisionEnum.YEAR, TemporalPrecisionEnum.MONTH, TemporalPrecisionEnum.DAY);
public ExtendedLuceneClauseBuilder(FhirContext myFhirContext, BooleanPredicateClausesStep<?> myRootClause, SearchPredicateFactory myPredicateFactory) {
this.myFhirContext = myFhirContext;
this.myRootClause = myRootClause;
@ -71,7 +85,7 @@ public class ExtendedLuceneClauseBuilder {
} else if (nextOr instanceof TokenParam) {
TokenParam nextOrToken = (TokenParam) nextOr;
nextValueTrimmed = nextOrToken.getValue();
} else if (nextOr instanceof ReferenceParam){
} else if (nextOr instanceof ReferenceParam) {
ReferenceParam referenceParam = (ReferenceParam) nextOr;
nextValueTrimmed = referenceParam.getValue();
if (nextValueTrimmed.contains("/_history")) {
@ -227,17 +241,203 @@ public class ExtendedLuceneClauseBuilder {
}
}
public void addReferenceUnchainedSearch(String theSearchParamName, List<List<IQueryParameterType>> theReferenceAndOrTerms) {
String fieldPath = "sp." + theSearchParamName + ".reference.value";
for (List<? extends IQueryParameterType> nextAnd : theReferenceAndOrTerms) {
Set<String> terms = extractOrStringParams(nextAnd);
ourLog.trace("reference unchained search {}", terms);
public void addReferenceUnchainedSearch(String theSearchParamName, List<List<IQueryParameterType>> theReferenceAndOrTerms) {
String fieldPath = "sp." + theSearchParamName + ".reference.value";
for (List<? extends IQueryParameterType> nextAnd : theReferenceAndOrTerms) {
Set<String> terms = extractOrStringParams(nextAnd);
ourLog.trace("reference unchained search {}", terms);
List<? extends PredicateFinalStep> orTerms = terms.stream()
.map(s -> myPredicateFactory.match().field(fieldPath).matching(s))
.collect(Collectors.toList());
List<? extends PredicateFinalStep> orTerms = terms.stream()
.map(s -> myPredicateFactory.match().field(fieldPath).matching(s))
.collect(Collectors.toList());
myRootClause.must(orPredicateOrSingle(orTerms));
}
}
myRootClause.must(orPredicateOrSingle(orTerms));
}
}
/**
* Create date clause from date params. The date lower and upper bounds are taken
* into considertion when generating date query ranges
*
* <p>Example 1 ('eq' prefix/empty): <code>http://fhirserver/Observation?date=eq2020</code>
* would generate the following search clause
* <pre>
* {@code
* {
* "bool": {
* "must": [{
* "range": {
* "sp.date.dt.lower-ord": { "gte": "20200101" }
* }
* }, {
* "range": {
* "sp.date.dt.upper-ord": { "lte": "20201231" }
* }
* }]
* }
* }
* }
* </pre>
*
* <p>Example 2 ('gt' prefix): <code>http://fhirserver/Observation?date=gt2020-01-01T08:00:00.000</code>
* <p>No timezone in the query will be taken as localdatetime(for e.g MST/UTC-07:00 in this case) converted to UTC before comparison</p>
* <pre>
* {@code
* {
* "range":{
* "sp.date.dt.upper":{ "gt": "2020-01-01T15:00:00.000000000Z" }
* }
* }
* }
* </pre>
*
* <p>Example 3 between dates: <code>http://fhirserver/Observation?date=ge2010-01-01&date=le2020-01</code></p>
* <pre>
* {@code
* {
* "range":{
* "sp.date.dt.upper-ord":{ "gte":"20100101" }
* },
* "range":{
* "sp.date.dt.lower-ord":{ "lte":"20200101" }
* }
* }
* }
* </pre>
*
* <p>Example 4 not equal: <code>http://fhirserver/Observation?date=ne2021</code></p>
* <pre>
* {@code
* {
* "bool": {
* "should": [{
* "range": {
* "sp.date.dt.upper-ord": { "lt": "20210101" }
* }
* }, {
* "range": {
* "sp.date.dt.lower-ord": { "gt": "20211231" }
* }
* }],
* "minimum_should_match": "1"
* }
* }
* }
* </pre>
*
* @param theSearchParamName
* @param theDateAndOrTerms
*/
public void addDateUnmodifiedSearch(String theSearchParamName, List<List<IQueryParameterType>> theDateAndOrTerms) {
for (List<? extends IQueryParameterType> nextAnd : theDateAndOrTerms) {
// comma separated list of dates(OR list) on a date param is not applicable so grab
// first from default list
if (nextAnd.size() > 1) {
throw new IllegalArgumentException(Msg.code(2032) + "OR (,) searches on DATE search parameters are not supported for ElasticSearch/Lucene");
}
DateParam dateParam = (DateParam) nextAnd.stream().findFirst()
.orElseThrow(() -> new InvalidRequestException("Date param is missing value"));
boolean isOrdinalSearch = ordinalSearchPrecisions.contains(dateParam.getPrecision());
PredicateFinalStep searchPredicate = isOrdinalSearch
? generateDateOrdinalSearchTerms(theSearchParamName, dateParam)
: generateDateInstantSearchTerms(theSearchParamName, dateParam);
myRootClause.must(searchPredicate);
}
}
private PredicateFinalStep generateDateOrdinalSearchTerms(String theSearchParamName, DateParam theDateParam) {
String lowerOrdinalField = "sp." + theSearchParamName + ".dt.lower-ord";
String upperOrdinalField = "sp." + theSearchParamName + ".dt.upper-ord";
int lowerBoundAsOrdinal;
int upperBoundAsOrdinal;
ParamPrefixEnum prefix = theDateParam.getPrefix();
// default when handling 'Day' temporal types
lowerBoundAsOrdinal = upperBoundAsOrdinal = DateUtils.convertDateToDayInteger(theDateParam.getValue());
TemporalPrecisionEnum precision = theDateParam.getPrecision();
// complete the date from 'YYYY' and 'YYYY-MM' temporal types
if (precision == TemporalPrecisionEnum.YEAR || precision == TemporalPrecisionEnum.MONTH) {
Pair<String, String> completedDate = DateUtils.getCompletedDate(theDateParam.getValueAsString());
lowerBoundAsOrdinal = Integer.parseInt(completedDate.getLeft().replace("-", ""));
upperBoundAsOrdinal = Integer.parseInt(completedDate.getRight().replace("-", ""));
}
if (Objects.isNull(prefix) || prefix == ParamPrefixEnum.EQUAL) {
// For equality prefix we would like the date to fall between the lower and upper bound
List<? extends PredicateFinalStep> predicateSteps = Arrays.asList(
myPredicateFactory.range().field(lowerOrdinalField).atLeast(lowerBoundAsOrdinal),
myPredicateFactory.range().field(upperOrdinalField).atMost(upperBoundAsOrdinal)
);
BooleanPredicateClausesStep<?> booleanStep = myPredicateFactory.bool();
predicateSteps.forEach(booleanStep::must);
return booleanStep;
} else if (ParamPrefixEnum.GREATERTHAN == prefix || ParamPrefixEnum.STARTS_AFTER == prefix) {
// TODO JB: more fine tuning needed for STARTS_AFTER
return myPredicateFactory.range().field(upperOrdinalField).greaterThan(upperBoundAsOrdinal);
} else if (ParamPrefixEnum.GREATERTHAN_OR_EQUALS == prefix) {
return myPredicateFactory.range().field(upperOrdinalField).atLeast(upperBoundAsOrdinal);
} else if (ParamPrefixEnum.LESSTHAN == prefix || ParamPrefixEnum.ENDS_BEFORE == prefix) {
// TODO JB: more fine tuning needed for END_BEFORE
return myPredicateFactory.range().field(lowerOrdinalField).lessThan(lowerBoundAsOrdinal);
} else if (ParamPrefixEnum.LESSTHAN_OR_EQUALS == prefix) {
return myPredicateFactory.range().field(lowerOrdinalField).atMost(lowerBoundAsOrdinal);
} else if (ParamPrefixEnum.NOT_EQUAL == prefix) {
List<? extends PredicateFinalStep> predicateSteps = Arrays.asList(
myPredicateFactory.range().field(upperOrdinalField).lessThan(lowerBoundAsOrdinal),
myPredicateFactory.range().field(lowerOrdinalField).greaterThan(upperBoundAsOrdinal)
);
BooleanPredicateClausesStep<?> booleanStep = myPredicateFactory.bool();
predicateSteps.forEach(booleanStep::should);
booleanStep.minimumShouldMatchNumber(1);
return booleanStep;
}
throw new IllegalArgumentException(Msg.code(2025) + "Date search param does not support prefix of type: " + prefix);
}
private PredicateFinalStep generateDateInstantSearchTerms(String theSearchParamName, DateParam theDateParam) {
String lowerInstantField = "sp." + theSearchParamName + ".dt.lower";
String upperInstantField = "sp." + theSearchParamName + ".dt.upper";
ParamPrefixEnum prefix = theDateParam.getPrefix();
if (ParamPrefixEnum.NOT_EQUAL == prefix) {
Instant dateInstant = theDateParam.getValue().toInstant();
List<? extends PredicateFinalStep> predicateSteps = Arrays.asList(
myPredicateFactory.range().field(upperInstantField).lessThan(dateInstant),
myPredicateFactory.range().field(lowerInstantField).greaterThan(dateInstant)
);
BooleanPredicateClausesStep<?> booleanStep = myPredicateFactory.bool();
predicateSteps.forEach(booleanStep::should);
booleanStep.minimumShouldMatchNumber(1);
return booleanStep;
}
// Consider lower and upper bounds for building range predicates
DateRangeParam dateRange = new DateRangeParam(theDateParam);
Instant lowerBoundAsInstant = Optional.ofNullable(dateRange.getLowerBound()).map(param -> param.getValue().toInstant()).orElse(null);
Instant upperBoundAsInstant = Optional.ofNullable(dateRange.getUpperBound()).map(param -> param.getValue().toInstant()).orElse(null);
if (Objects.isNull(prefix) || prefix == ParamPrefixEnum.EQUAL) {
// For equality prefix we would like the date to fall between the lower and upper bound
List<? extends PredicateFinalStep> predicateSteps = Arrays.asList(
myPredicateFactory.range().field(lowerInstantField).atLeast(lowerBoundAsInstant),
myPredicateFactory.range().field(upperInstantField).atMost(upperBoundAsInstant)
);
BooleanPredicateClausesStep<?> booleanStep = myPredicateFactory.bool();
predicateSteps.forEach(booleanStep::must);
return booleanStep;
} else if (ParamPrefixEnum.GREATERTHAN == prefix || ParamPrefixEnum.STARTS_AFTER == prefix) {
return myPredicateFactory.range().field(upperInstantField).greaterThan(lowerBoundAsInstant);
} else if (ParamPrefixEnum.GREATERTHAN_OR_EQUALS == prefix) {
return myPredicateFactory.range().field(upperInstantField).atLeast(lowerBoundAsInstant);
} else if (ParamPrefixEnum.LESSTHAN == prefix || ParamPrefixEnum.ENDS_BEFORE == prefix) {
return myPredicateFactory.range().field(lowerInstantField).lessThan(upperBoundAsInstant);
} else if (ParamPrefixEnum.LESSTHAN_OR_EQUALS == prefix) {
return myPredicateFactory.range().field(lowerInstantField).atMost(upperBoundAsInstant);
}
throw new IllegalArgumentException(Msg.code(2026) + "Date search param does not support prefix of type: " + prefix);
}
}

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedLuceneIndexExtractor.java
View File

@ -36,7 +36,7 @@ import java.util.Map;
/**
* Extract search params for advanced lucene indexing.
*
* <p>
* This class re-uses the extracted JPA entities to build an ExtendedLuceneIndexData instance.
*/
public class ExtendedLuceneIndexExtractor {
@ -59,6 +59,10 @@ public class ExtendedLuceneIndexExtractor {
theNewParams.myTokenParams.forEach(nextParam ->
retVal.addTokenIndexData(nextParam.getParamName(), nextParam.getSystem(), nextParam.getValue()));
theNewParams.myDateParams.forEach(nextParam ->
retVal.addDateIndexData(nextParam.getParamName(), nextParam.getValueLow(), nextParam.getValueLowDateOrdinal(),
nextParam.getValueHigh(), nextParam.getValueHighDateOrdinal()));
if (!theNewParams.myLinks.isEmpty()) {
// awkwardly, links are indexed by jsonpath, not by search param.
@ -86,6 +90,7 @@ public class ExtendedLuceneIndexExtractor {
}
}
}
return retVal;
}
}

13
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedLuceneSearchBuilder.java
View File

@ -24,6 +24,7 @@ import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.param.DateParam;
import ca.uhn.fhir.rest.param.QuantityParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.StringParam;
@ -64,7 +65,7 @@ public class ExtendedLuceneSearchBuilder {
/**
* Do we support this query param type+modifier?
*
* <p>
* NOTE - keep this in sync with addAndConsumeAdvancedQueryClauses() below.
*/
private boolean isParamSupported(IQueryParameterType param) {
@ -103,6 +104,11 @@ public class ExtendedLuceneSearchBuilder {
default:
return false;
}
} else if (param instanceof DateParam) {
if (EMPTY_MODIFIER.equals(modifier)) {
return true;
}
return false;
} else {
return false;
}
@ -153,6 +159,11 @@ public class ExtendedLuceneSearchBuilder {
builder.addReferenceUnchainedSearch(nextParam, referenceAndOrTerms);
break;
case DATE:
List<List<IQueryParameterType>> dateAndOrTerms = theParams.removeByNameUnmodified(nextParam);
builder.addDateUnmodifiedSearch(nextParam, dateAndOrTerms);
break;
default:
// ignore unsupported param types/modifiers. They will be processed up in SearchBuilder.
}

169
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/LastNAggregation.java Normal file
View File

@ -0,0 +1,169 @@
package ca.uhn.fhir.jpa.dao.search;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import org.hibernate.search.engine.search.aggregation.AggregationKey;
import javax.annotation.Nonnull;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
/**
* Builds lastN aggregation, and parse the results
*/
public class LastNAggregation {
static final String SP_SUBJECT = "sp.subject.reference.value";
private static final String SP_CODE_TOKEN_CODE_AND_SYSTEM = "sp.code.token.code-system";
private static final String SP_DATE_DT_UPPER = "sp.date.dt.upper";
private static final String GROUP_BY_CODE_SYSTEM_SUB_AGGREGATION = "group_by_code_system";
private static final String MOST_RECENT_EFFECTIVE_SUB_AGGREGATION = "most_recent_effective";
private final int myLastNMax;
private final boolean myAggregateOnSubject;
private final Gson myJsonParser = new Gson();
public LastNAggregation(int theLastNMax, boolean theAggregateOnSubject) {
myLastNMax = theLastNMax;
myAggregateOnSubject = theAggregateOnSubject;
}
/**
* Aggregation template json.
* <p>
* https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html
*/
public JsonObject toAggregation() {
JsonObject lastNAggregation = myJsonParser.fromJson(
"{" +
" \"terms\":{" +
" \"field\":\"" + SP_CODE_TOKEN_CODE_AND_SYSTEM + "\"," +
" \"size\":100," +
" \"min_doc_count\":1" +
" }," +
" \"aggs\":{" +
" \"" + MOST_RECENT_EFFECTIVE_SUB_AGGREGATION + "\":{" +
" \"top_hits\":{" +
" \"size\":" + myLastNMax + "," +
" \"sort\":[" +
" {" +
" \"" + SP_DATE_DT_UPPER + "\":{" +
" \"order\":\"desc\"" +
" }" +
" }" +
" ]," +
" \"_source\":[" +
" \"myId\"" +
" ]" +
" }" +
" }" +
" }" +
"}", JsonObject.class);
if (myAggregateOnSubject) {
lastNAggregation = myJsonParser.fromJson(
"{" +
" \"terms\": {" +
" \"field\": \"" + SP_SUBJECT + "\"," +
" \"size\": 100," +
" \"min_doc_count\": 1" +
" }," +
" \"aggs\": {" +
" \"" + GROUP_BY_CODE_SYSTEM_SUB_AGGREGATION + "\": " + myJsonParser.toJson(lastNAggregation) + "" +
" }" +
"}", JsonObject.class);
}
return lastNAggregation;
}
/**
* Parses the JSONObject aggregation result from ES to extract observation resource ids
* E.g aggregation result payload
* <pre>
* {@code
* {
* "doc_count_error_upper_bound": 0,
* "sum_other_doc_count": 0,
* "buckets": [
* {
* "key": "http://mycode.com|code0",
* "doc_count": 45,
* "most_recent_effective": {
* "hits": {
* "total": {
* "value": 45,
* "relation": "eq"
* },
* "max_score": null,
* "hits": [
* {
* "_index": "resourcetable-000001",
* "_type": "_doc",
* "_id": "48",
* "_score": null,
* "_source": {
* "myId": 48
* },
* "sort": [
* 1643673125112
* ]
* }
* ]
* }
* }
* },
* {
* "key": "http://mycode.com|code1",
* "doc_count": 30,
* "most_recent_effective": {
* "hits": {
* "total": {
* "value": 30,
* "relation": "eq"
* },
* "max_score": null,
* "hits": [
* {
* "_index": "resourcetable-000001",
* "_type": "_doc",
* "_id": "58",
* "_score": null,
* "_source": {
* "myId": 58
* },
* "sort": [
* 1643673125112
* ]
* }
* ]
* }
* }
* }
* ]
* }
* }
* </pre>
*/
public List<Long> extractResourceIds(@Nonnull JsonObject theAggregationResult) {
Stream<JsonObject> resultBuckets = Stream.of(theAggregationResult);
// was it grouped by subject?
if (myAggregateOnSubject) {
resultBuckets = StreamSupport.stream(theAggregationResult.getAsJsonArray("buckets").spliterator(), false)
.map(bucket -> bucket.getAsJsonObject().getAsJsonObject(GROUP_BY_CODE_SYSTEM_SUB_AGGREGATION));
}
return resultBuckets
.flatMap(grouping -> StreamSupport.stream(grouping.getAsJsonArray("buckets").spliterator(), false))
.flatMap(bucket -> {
JsonArray hits = bucket.getAsJsonObject()
.getAsJsonObject(MOST_RECENT_EFFECTIVE_SUB_AGGREGATION)
.getAsJsonObject("hits")
.getAsJsonArray("hits");
return StreamSupport.stream(hits.spliterator(), false);
})
.map(hit -> hit.getAsJsonObject().getAsJsonObject("_source").get("myId").getAsLong())
.collect(Collectors.toList());
}
}

63
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/LastNOperation.java Normal file
View File

@ -0,0 +1,63 @@
package ca.uhn.fhir.jpa.dao.search;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
import com.google.gson.JsonObject;
import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension;
import org.hibernate.search.engine.search.aggregation.AggregationKey;
import org.hibernate.search.engine.search.query.SearchResult;
import org.hibernate.search.mapper.orm.session.SearchSession;
import java.util.List;
import java.util.Objects;
public class LastNOperation {
public static final String OBSERVATION_RES_TYPE = "Observation";
private final SearchSession mySession;
private final FhirContext myFhirContext;
private final ISearchParamRegistry mySearchParamRegistry;
private final ExtendedLuceneSearchBuilder myExtendedLuceneSearchBuilder = new ExtendedLuceneSearchBuilder();
public LastNOperation(SearchSession theSession, FhirContext theFhirContext, ISearchParamRegistry theSearchParamRegistry) {
mySession = theSession;
myFhirContext = theFhirContext;
mySearchParamRegistry = theSearchParamRegistry;
}
public List<Long> executeLastN(SearchParameterMap theParams, Integer theMaximumResults) {
boolean lastNGroupedBySubject = isLastNGroupedBySubject(theParams);
LastNAggregation lastNAggregation = new LastNAggregation(getLastNMaxParamValue(theParams), lastNGroupedBySubject);
AggregationKey<JsonObject> observationsByCodeKey = AggregationKey.of("lastN_aggregation");
SearchResult<ResourceTable> result = mySession.search(ResourceTable.class)
.extension(ElasticsearchExtension.get())
.where(f -> f.bool(b -> {
// Must match observation type
b.must(f.match().field("myResourceType").matching(OBSERVATION_RES_TYPE));
ExtendedLuceneClauseBuilder builder = new ExtendedLuceneClauseBuilder(myFhirContext, b, f);
myExtendedLuceneSearchBuilder.addAndConsumeAdvancedQueryClauses(builder, OBSERVATION_RES_TYPE, theParams.clone(), mySearchParamRegistry);
}))
.aggregation(observationsByCodeKey, f -> f.fromJson(lastNAggregation.toAggregation()))
.fetch(0);
JsonObject resultAggregation = result.aggregation(observationsByCodeKey);
List<Long> pidList = lastNAggregation.extractResourceIds(resultAggregation);
if (theMaximumResults != null && theMaximumResults <= pidList.size()) {
return pidList.subList(0, theMaximumResults);
}
return pidList;
}
private boolean isLastNGroupedBySubject(SearchParameterMap theParams) {
String patientParamName = LastNParameterHelper.getPatientParamName(myFhirContext);
String subjectParamName = LastNParameterHelper.getSubjectParamName(myFhirContext);
return theParams.containsKey(patientParamName) || theParams.containsKey(subjectParamName);
}
private int getLastNMaxParamValue(SearchParameterMap theParams) {
return Objects.isNull(theParams.getLastNMax()) ? 1 : theParams.getLastNMax();
}
}

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/autocomplete/TokenAutocompleteSearch.java
View File

@ -100,7 +100,7 @@ class TokenAutocompleteSearch {
break;
case "":
default:
throw new IllegalArgumentException(Msg.code(2027) + "Autocomplete only accepts text search for now.");
throw new IllegalArgumentException(Msg.code(2034) + "Autocomplete only accepts text search for now.");
}

31
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java
View File

@ -372,13 +372,23 @@ public class SearchBuilder implements ISearchBuilder {
}
private List<ResourcePersistentId> executeLastNAgainstIndex(Integer theMaximumResults) {
validateLastNIsEnabled();
List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults);
return lastnResourceIds.stream()
.map(lastnResourceId -> myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId))
.collect(Collectors.toList());
// Can we use our hibernate search generated index on resource to support lastN?:
if (myDaoConfig.isAdvancedLuceneIndexing()) {
if (myFulltextSearchSvc == null) {
throw new InvalidRequestException(Msg.code(2027) + "LastN operation is not enabled on this service, can not process this request");
}
return myFulltextSearchSvc.lastN(myParams, theMaximumResults)
.stream().map(lastNResourceId -> myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, String.valueOf(lastNResourceId)))
.collect(Collectors.toList());
} else {
if (myIElasticsearchSvc == null) {
throw new InvalidRequestException(Msg.code(2033) + "LastN operation is not enabled on this service, can not process this request");
}
// use the dedicated observation ES/Lucene index to support lastN query
return myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults).stream()
.map(lastnResourceId -> myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId))
.collect(Collectors.toList());
}
}
private List<ResourcePersistentId> queryLuceneForPIDs(RequestDetails theRequest) {
@ -393,7 +403,6 @@ public class SearchBuilder implements ISearchBuilder {
return pids;
}
private void doCreateChunkedQueries(SearchParameterMap theParams, List<Long> thePids, Integer theOffset, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<SearchQueryExecutor> theQueries) {
if (thePids.size() < getMaximumPageSize()) {
normalizeIdListForLastNInClause(thePids);
@ -1620,12 +1629,6 @@ public class SearchBuilder implements ISearchBuilder {
return thePredicates.toArray(new Predicate[0]);
}
private void validateLastNIsEnabled() {
if (myIElasticsearchSvc == null) {
throw new InvalidRequestException(Msg.code(1199) + "LastN operation is not enabled on this service, can not process this request");
}
}
private void validateFullTextSearchIsEnabled() {
if (myFulltextSearchSvc == null) {
if (myParams.containsKey(Constants.PARAM_TEXT)) {

6
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/ElasticsearchSvcImpl.java
View File

@ -76,6 +76,8 @@ import org.elasticsearch.search.aggregations.metrics.ParsedTopHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import javax.annotation.Nullable;
@ -93,6 +95,8 @@ import static org.apache.commons.lang3.StringUtils.isBlank;
public class ElasticsearchSvcImpl implements IElasticsearchSvc {
private static final Logger ourLog = LoggerFactory.getLogger(ElasticsearchSvcImpl.class);
// Index Constants
public static final String OBSERVATION_INDEX = "observation_index";
public static final String OBSERVATION_CODE_INDEX = "code_index";
@ -223,6 +227,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
}
SearchRequest myLastNRequest = buildObservationsSearchRequest(subject, theSearchParameterMap, theFhirContext,
createObservationSubjectAggregationBuilder(getMaxParameter(theSearchParameterMap), topHitsInclude));
ourLog.debug("ElasticSearch query: {}", myLastNRequest.source().toString());
try {
SearchResponse lastnResponse = executeSearchRequest(myLastNRequest);
searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext,
@ -234,6 +239,7 @@ public class ElasticsearchSvcImpl implements IElasticsearchSvc {
} else {
SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, theFhirContext,
createObservationCodeAggregationBuilder(getMaxParameter(theSearchParameterMap), topHitsInclude));
ourLog.debug("ElasticSearch query: {}", myLastNRequest.source().toString());
try {
SearchResponse lastnResponse = executeSearchRequest(myLastNRequest);
searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext,

16
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/lastn/json/ObservationJson.java
View File

@ -136,6 +136,7 @@ public class ObservationJson {
}
public void setCode(CodeJson theCode) {
myCode_concept_id = theCode.getCodeableConceptId();
myCode_concept_text = theCode.getCodeableConceptText();
// Currently can only support one Coding for Observation Code
myCode_coding_code_system_hash = theCode.getCoding_code_system_hash().get(0);
@ -145,6 +146,17 @@ public class ObservationJson {
}
public CodeJson getCode() {
CodeJson code = new CodeJson();
code.setCodeableConceptId(myCode_concept_id);
code.setCodeableConceptText(myCode_concept_text);
code.getCoding_code_system_hash().add(myCode_coding_code_system_hash);
code.getCoding_code().add(myCode_coding_code);
code.getCoding_display().add(myCode_coding_display);
code.getCoding_system().add(myCode_coding_system);
return code;
}
public String getCode_concept_text() {
return myCode_concept_text;
}
@ -165,10 +177,6 @@ public class ObservationJson {
return myCode_coding_system;
}
public void setCode_concept_id(String theCodeId) {
myCode_concept_id = theCodeId;
}
public String getCode_concept_id() {
return myCode_concept_id;
}

5
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/ElasticsearchWithPrefixConfig.java
View File

@ -3,14 +3,11 @@ package ca.uhn.fhir.jpa.config;
import ca.uhn.fhir.context.ConfigurationException;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.config.BlockLargeNumbersOfParamsListener;
import ca.uhn.fhir.jpa.config.HapiFhirHibernateJpaDialect;
import ca.uhn.fhir.jpa.config.HapiFhirLocalContainerEntityManagerFactoryBean;
import ca.uhn.fhir.jpa.dao.r4.ElasticsearchPrefixTest;
import ca.uhn.fhir.jpa.search.elastic.HapiElasticsearchAnalysisConfigurer;
import ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy;
import ca.uhn.fhir.jpa.search.lastn.ElasticsearchRestClientFactory;
import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchContainerHelper;
import ca.uhn.fhir.jpa.search.elastic.TestElasticsearchContainerHelper;
import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
import net.ttddyy.dsproxy.listener.logging.SLF4JLogLevel;
import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder;

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestHibernateSearchAddInConfig.java
View File

@ -5,8 +5,8 @@ import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
import ca.uhn.fhir.jpa.model.config.PartitionSettings;
import ca.uhn.fhir.jpa.search.HapiLuceneAnalysisConfigurer;
import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder;
import ca.uhn.fhir.jpa.search.elastic.TestElasticsearchContainerHelper;
import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl;
import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchContainerHelper;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import org.hibernate.search.backend.elasticsearch.index.IndexStatus;
import org.hibernate.search.backend.lucene.cfg.LuceneBackendSettings;

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BasePartitioningR4Test.java
View File

@ -229,7 +229,7 @@ public abstract class BasePartitioningR4Test extends BaseJpaR4SystemTest {
assertNotNull(theResource);
assertTrue(!myCreateRequestPartitionIds.isEmpty(), "No create partitions left in interceptor");
RequestPartitionId retVal = myCreateRequestPartitionIds.remove(0);
ourLog.info("Returning partition for create: {}", retVal);
ourLog.info("Returning partition [{}] for create of resource {} with date {}", retVal, theResource, retVal.getPartitionDate());
return retVal;
}

5
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseR4SearchLastN.java
View File

@ -116,6 +116,8 @@ abstract public class BaseR4SearchLastN extends BaseJpaTest {
// Creating this data and the index is time consuming and as such want to avoid having to repeat for each test.
// Normally would use a static @BeforeClass method for this purpose, but Autowired objects cannot be accessed in static methods.
if (!dataLoaded || patient0Id == null) {
// enabled to also create extended lucene index during creation of test data
myDaoConfig.setAdvancedLuceneIndexing(true);
Patient pt = new Patient();
pt.addName().setFamily("Lastn").addGiven("Arthur");
patient0Id = myPatientDao.create(pt, mockSrd()).getId().toUnqualifiedVersionless();
@ -132,7 +134,8 @@ abstract public class BaseR4SearchLastN extends BaseJpaTest {
myElasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
myElasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
// turn off the setting enabled earlier
myDaoConfig.setAdvancedLuceneIndexing(false);
}
}

26
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNUsingExtendedLuceneIndexAsyncIT.java Normal file
View File

@ -0,0 +1,26 @@
package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.test.context.junit.jupiter.SpringExtension;
/**
* Run entire @see {@link FhirResourceDaoR4SearchLastNAsyncIT} test suite this time
* using Extended Lucene index as search target
*/
@ExtendWith(SpringExtension.class)
public class FhirResourceDaoR4SearchLastNUsingExtendedLuceneIndexAsyncIT extends FhirResourceDaoR4SearchLastNAsyncIT {
@BeforeEach
public void enableAdvancedLuceneIndexing() {
myDaoConfig.setAdvancedLuceneIndexing(true);
}
@AfterEach
public void disableAdvancedLuceneIndex() {
myDaoConfig.setAdvancedLuceneIndexing(new DaoConfig().isAdvancedLuceneIndexing());
}
}

26
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchLastNUsingExtendedLuceneIndexIT.java Normal file
View File

@ -0,0 +1,26 @@
package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.test.context.junit.jupiter.SpringExtension;
/**
* Run entire @see {@link FhirResourceDaoR4SearchLastNIT} test suite this time
* using Extended Lucene index as search target
*/
@ExtendWith(SpringExtension.class)
public class FhirResourceDaoR4SearchLastNUsingExtendedLuceneIndexIT extends FhirResourceDaoR4SearchLastNIT {
@BeforeEach
public void enableAdvancedLuceneIndexing() {
myDaoConfig.setAdvancedLuceneIndexing(true);
}
@AfterEach
public void disableAdvancedLuceneIndex() {
myDaoConfig.setAdvancedLuceneIndexing(new DaoConfig().isAdvancedLuceneIndexing());
}
}

52
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java
View File

@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet;
import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
@ -11,12 +12,15 @@ import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportSvc;
import ca.uhn.fhir.jpa.config.TestHibernateSearchAddInConfig;
import ca.uhn.fhir.jpa.config.TestR4Config;
import ca.uhn.fhir.jpa.dao.BaseDateSearchDaoTests;
import ca.uhn.fhir.jpa.dao.BaseJpaTest;
import ca.uhn.fhir.jpa.dao.DaoTestDataBuilder;
import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.partition.SystemRequestDetails;
import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
@ -52,6 +56,7 @@ import org.hl7.fhir.r4.model.StringType;
import org.hl7.fhir.r4.model.ValueSet;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
@ -123,6 +128,8 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
private IBulkDataExportSvc myBulkDataExportSvc;
@Autowired
private ITermCodeSystemStorageSvc myTermCodeSystemStorageSvc;
@Autowired
private DaoRegistry myDaoRegistry;
private boolean myContainsSettings;
@BeforeEach
@ -239,6 +246,7 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
}
}
@Test
public void testResourceCodeTokenSearch() {
IIdType id1, id2, id2b, id3;
@ -307,7 +315,7 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
@Test
public void testResourceCodeTextSearch() {
IIdType id1,id2,id3,id4;
IIdType id1, id2, id3, id4;
{
Observation obs1 = new Observation();
@ -457,7 +465,6 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
}
// run searches
{
@ -499,9 +506,10 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
private void assertObservationSearchMatchesNothing(String message, SearchParameterMap map) {
assertObservationSearchMatches(message,map);
assertObservationSearchMatches(message, map);
}
private void assertObservationSearchMatches(String message, SearchParameterMap map, IIdType ...iIdTypes) {
private void assertObservationSearchMatches(String message, SearchParameterMap map, IIdType... iIdTypes) {
assertThat(message, toUnqualifiedVersionlessIdValues(myObservationDao.search(map)), containsInAnyOrder(toValues(iIdTypes)));
}
@ -529,15 +537,15 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
ValueSet vs = new ValueSet();
ValueSet.ConceptSetComponent include = vs.getCompose().addInclude();
include.setSystem(URL_MY_CODE_SYSTEM);
ValueSet result = myValueSetDao.expand(vs, new ValueSetExpansionOptions().setFilter("child"));
logAndValidateValueSet(result);
String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(result);
ourLog.info(resp);
assertThat(resp, stringContainsInOrder("<code value=\"childCA\"/>","<display value=\"Child CA\"/>"));
assertThat(resp, stringContainsInOrder("<code value=\"childCA\"/>", "<display value=\"Child CA\"/>"));
}
@Test
@ -547,15 +555,15 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
ValueSet vs = new ValueSet();
ValueSet.ConceptSetComponent include = vs.getCompose().addInclude();
include.setSystem(URL_MY_CODE_SYSTEM);
ValueSet result = myValueSetDao.expand(vs, new ValueSetExpansionOptions().setFilter("chi"));
logAndValidateValueSet(result);
String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(result);
ourLog.info(resp);
assertThat(resp, stringContainsInOrder("<code value=\"childCA\"/>","<display value=\"Child CA\"/>"));
assertThat(resp, stringContainsInOrder("<code value=\"childCA\"/>", "<display value=\"Child CA\"/>"));
}
@Test
@ -565,17 +573,17 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
ValueSet vs = new ValueSet();
ValueSet.ConceptSetComponent include = vs.getCompose().addInclude();
include.setSystem(URL_MY_CODE_SYSTEM);
ValueSet result = myValueSetDao.expand(vs, new ValueSetExpansionOptions().setFilter("hil"));
logAndValidateValueSet(result);
String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(result);
ourLog.info(resp);
assertThat(resp, not(stringContainsInOrder("<code value=\"childCA\"/>","<display value=\"Child CA\"/>")));
assertThat(resp, not(stringContainsInOrder("<code value=\"childCA\"/>", "<display value=\"Child CA\"/>")));
}
@Test
public void testExpandVsWithMultiInclude_All() throws IOException {
CodeSystem cs = loadResource(myFhirCtx, CodeSystem.class, "/r4/expand-multi-cs.json");
@ -701,4 +709,14 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
}
/*@Nested
public class DateSearchTests extends BaseDateSearchDaoTests {
@Override
protected Fixture getFixture() {
DaoTestDataBuilder testDataBuilder = new DaoTestDataBuilder(myFhirCtx, myDaoRegistry, new SystemRequestDetails());
return new TestDataBuilderFixture<>(testDataBuilder, myObservationDao);
}
}*/
}

6
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java
View File

@ -623,7 +623,8 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
public void testCreateInTransaction_ServerId_WithPartition() {
createUniqueCompositeSp();
createRequestId();
ourLog.info("Starting testCreateInTransaction_ServerId_WithPartition");
ourLog.info("Setting up partitionId {} with date {}", myPartitionId, myPartitionDate);
addCreatePartition(myPartitionId, myPartitionDate);
addCreatePartition(myPartitionId, myPartitionDate);
addCreatePartition(myPartitionId, myPartitionDate);
@ -657,6 +658,9 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
runInTransaction(() -> {
// HFJ_RESOURCE
ResourceTable resourceTable = myResourceTableDao.findById(patientId).orElseThrow(IllegalArgumentException::new);
ourLog.info("Found resourceTable {}, which contains partition id {} with date {}", resourceTable.getId(), resourceTable.getPartitionId(), resourceTable.getPartitionId().getPartitionDate());
ourLog.info("in test: myPartitionDate = {}", myPartitionDate);
ourLog.info("in test: resourceTablePartDate = {}", resourceTable.getPartitionId().getPartitionDate());
assertEquals(myPartitionId, resourceTable.getPartitionId().getPartitionId().intValue());
assertEquals(myPartitionDate, resourceTable.getPartitionId().getPartitionDate());
});

7
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/config/TestElasticsearchContainerHelper.java → hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/elastic/TestElasticsearchContainerHelper.java
View File

@ -1,10 +1,5 @@
package ca.uhn.fhir.jpa.search.lastn.config;
package ca.uhn.fhir.jpa.search.elastic;
import com.github.dockerjava.api.exception.InternalServerErrorException;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import java.time.Duration;

160
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcMultipleObservationsIT.java
View File

@ -2,7 +2,7 @@ package ca.uhn.fhir.jpa.search.lastn;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.model.config.PartitionSettings;
import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchContainerHelper;
import ca.uhn.fhir.jpa.search.elastic.TestElasticsearchContainerHelper;
import ca.uhn.fhir.jpa.search.lastn.json.CodeJson;
import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
@ -18,13 +18,8 @@ import ca.uhn.fhir.rest.param.TokenOrListParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.param.TokenParamModifier;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.hl7.fhir.r4.model.Observation;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@ -35,14 +30,15 @@ import org.testcontainers.junit.jupiter.Testcontainers;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.TEST_BASELINE_TIMESTAMP;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@ -54,10 +50,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@Testcontainers
public class LastNElasticsearchSvcMultipleObservationsIT {
static private final Calendar baseObservationDate = new GregorianCalendar();
@Container
public static ElasticsearchContainer elasticsearchContainer = TestElasticsearchContainerHelper.getEmbeddedElasticSearch();
private static ObjectMapper ourMapperNonPrettyPrint;
private static boolean indexLoaded = false;
private final Map<String, Map<String, List<Date>>> createdPatientObservationMap = new HashMap<>();
private final FhirContext myFhirContext = FhirContext.forR4Cached();
@ -85,26 +79,10 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
// execute Observation ID search (Composite Aggregation) last 3 observations for each patient
SearchParameterMap searchParameterMap = new SearchParameterMap();
ReferenceParam subjectParam = new ReferenceParam("Patient", "", "0");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
subjectParam = new ReferenceParam("Patient", "", "1");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
subjectParam = new ReferenceParam("Patient", "", "2");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
subjectParam = new ReferenceParam("Patient", "", "3");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
subjectParam = new ReferenceParam("Patient", "", "4");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
subjectParam = new ReferenceParam("Patient", "", "5");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
subjectParam = new ReferenceParam("Patient", "", "6");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
subjectParam = new ReferenceParam("Patient", "", "7");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
subjectParam = new ReferenceParam("Patient", "", "8");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
subjectParam = new ReferenceParam("Patient", "", "9");
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
IntStream.range(0, 10).forEach(index -> {
ReferenceParam subjectParam = new ReferenceParam("Patient", "", String.valueOf(index));
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
});
searchParameterMap.setLastNMax(3);
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFieldsForTest(searchParameterMap, myFhirContext);
@ -322,7 +300,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
ReferenceParam validPatientParam = new ReferenceParam("Patient", "", "9");
TokenParam validCategoryCodeParam = new TokenParam("http://mycodes.org/fhir/observation-category", "test-heart-rate");
TokenParam validObservationCodeParam = new TokenParam("http://mycodes.org/fhir/observation-code", "test-code-1");
DateParam validDateParam = new DateParam(ParamPrefixEnum.EQUAL, new Date(baseObservationDate.getTimeInMillis() - (9 * 3600 * 1000)));
DateParam validDateParam = new DateParam(ParamPrefixEnum.EQUAL, new Date(TEST_BASELINE_TIMESTAMP - (9 * 3600 * 1000)));
// Ensure that valid parameters are indeed valid
SearchParameterMap searchParameterMap = new SearchParameterMap();
@ -387,7 +365,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(validPatientParam));
searchParameterMap.add(Observation.SP_CATEGORY, buildTokenAndListParam(validCategoryCodeParam));
searchParameterMap.add(Observation.SP_CODE, buildTokenAndListParam(validObservationCodeParam));
searchParameterMap.add(Observation.SP_DATE, new DateParam(ParamPrefixEnum.GREATERTHAN, baseObservationDate.getTime()));
searchParameterMap.add(Observation.SP_DATE, new DateParam(ParamPrefixEnum.GREATERTHAN, TEST_BASELINE_TIMESTAMP));
searchParameterMap.setLastNMax(100);
observations = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100);
assertEquals(0, observations.size());
@ -396,8 +374,8 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
@Test
public void testLastNEffectiveDates() {
Date highDate = new Date(baseObservationDate.getTimeInMillis() - (3600 * 1000));
Date lowDate = new Date(baseObservationDate.getTimeInMillis() - (10 * 3600 * 1000));
Date highDate = new Date(TEST_BASELINE_TIMESTAMP - (3600 * 1000));
Date lowDate = new Date(TEST_BASELINE_TIMESTAMP - (10 * 3600 * 1000));
SearchParameterMap searchParameterMap = new SearchParameterMap();
ReferenceParam subjectParam = new ReferenceParam("Patient", "", "3");
@ -458,7 +436,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
searchParameterMap = new SearchParameterMap();
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
DateParam startDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(baseObservationDate.getTimeInMillis() - (4 * 3600 * 1000)));
DateParam startDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(TEST_BASELINE_TIMESTAMP - (4 * 3600 * 1000)));
DateAndListParam dateAndListParam = new DateAndListParam();
dateAndListParam.addAnd(new DateOrListParam().addOr(startDateParam));
dateParam = new DateParam(ParamPrefixEnum.LESSTHAN_OR_EQUALS, highDate);
@ -470,7 +448,7 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
searchParameterMap = new SearchParameterMap();
searchParameterMap.add(Observation.SP_SUBJECT, buildReferenceAndListParam(subjectParam));
startDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(baseObservationDate.getTimeInMillis() - (4 * 3600 * 1000)));
startDateParam = new DateParam(ParamPrefixEnum.GREATERTHAN, new Date(TEST_BASELINE_TIMESTAMP - (4 * 3600 * 1000)));
searchParameterMap.add(Observation.SP_DATE, startDateParam);
dateParam = new DateParam(ParamPrefixEnum.LESSTHAN, lowDate);
searchParameterMap.add(Observation.SP_DATE, dateParam);
@ -481,89 +459,34 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
}
private void createMultiplePatientsAndObservations() throws IOException {
// Create CodeableConcepts for two Codes, each with three codings.
String codeableConceptId1 = UUID.randomUUID().toString();
CodeJson codeJson1 = new CodeJson();
codeJson1.setCodeableConceptText("Test Codeable Concept Field for First Code");
codeJson1.setCodeableConceptId(codeableConceptId1);
codeJson1.addCoding("http://mycodes.org/fhir/observation-code", "test-code-1", "1-Observation Code1");
List<Integer> patientIds = IntStream.range(0, 10).boxed().collect(Collectors.toList());
List<ObservationJson> observations = LastNTestDataGenerator.createMultipleObservationJson(patientIds);
String codeableConceptId2 = UUID.randomUUID().toString();
CodeJson codeJson2 = new CodeJson();
codeJson2.setCodeableConceptText("Test Codeable Concept Field for Second Code");
codeJson2.setCodeableConceptId(codeableConceptId1);
codeJson2.addCoding("http://mycodes.org/fhir/observation-code", "test-code-2", "2-Observation Code2");
observations.forEach(observation -> {
CodeJson codeJson = observation.getCode();
assertTrue(elasticsearchSvc.createOrUpdateObservationCodeIndex(codeJson.getCodeableConceptId(), codeJson));
assertTrue(elasticsearchSvc.createOrUpdateObservationIndex(observation.getIdentifier(), observation));
// Create CodeableConcepts for two categories, each with three codings.
// Create three codings and first category CodeableConcept
List<CodeJson> categoryConcepts1 = new ArrayList<>();
CodeJson categoryCodeableConcept1 = new CodeJson();
categoryCodeableConcept1.setCodeableConceptText("Test Codeable Concept Field for first category");
categoryCodeableConcept1.addCoding("http://mycodes.org/fhir/observation-category", "test-heart-rate", "Test Heart Rate");
categoryCodeableConcept1.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-heart-rate", "Test Heartrate");
categoryCodeableConcept1.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-heart-rate", "Test Heart-Rate");
categoryConcepts1.add(categoryCodeableConcept1);
// Create three codings and second category CodeableConcept
List<CodeJson> categoryConcepts2 = new ArrayList<>();
CodeJson categoryCodeableConcept2 = new CodeJson();
categoryCodeableConcept2.setCodeableConceptText("Test Codeable Concept Field for second category");
categoryCodeableConcept2.addCoding("http://mycodes.org/fhir/observation-category", "test-vital-signs", "Test Vital Signs");
categoryCodeableConcept2.addCoding("http://myalternatecodes.org/fhir/observation-category", "test-alt-vitals", "Test Vital-Signs");
categoryCodeableConcept2.addCoding("http://mysecondaltcodes.org/fhir/observation-category", "test-2nd-alt-vitals", "Test Vitals");
categoryConcepts2.add(categoryCodeableConcept2);
for (int patientCount = 0; patientCount < 10; patientCount++) {
String subject = String.valueOf(patientCount);
for (int entryCount = 0; entryCount < 10; entryCount++) {
ObservationJson observationJson = new ObservationJson();
String identifier = String.valueOf((entryCount + patientCount * 10L));
observationJson.setIdentifier(identifier);
observationJson.setSubject(subject);
if (entryCount % 2 == 1) {
observationJson.setCategories(categoryConcepts1);
observationJson.setCode(codeJson1);
observationJson.setCode_concept_id(codeableConceptId1);
assertTrue(elasticsearchSvc.createOrUpdateObservationCodeIndex(codeableConceptId1, codeJson1));
} else {
observationJson.setCategories(categoryConcepts2);
observationJson.setCode(codeJson2);
observationJson.setCode_concept_id(codeableConceptId2);
assertTrue(elasticsearchSvc.createOrUpdateObservationCodeIndex(codeableConceptId2, codeJson2));
}
Date effectiveDtm = new Date(baseObservationDate.getTimeInMillis() - ((10L - entryCount) * 3600L * 1000L));
observationJson.setEffectiveDtm(effectiveDtm);
assertTrue(elasticsearchSvc.createOrUpdateObservationIndex(identifier, observationJson));
if (createdPatientObservationMap.containsKey(subject)) {
Map<String, List<Date>> observationCodeMap = createdPatientObservationMap.get(subject);
if (observationCodeMap.containsKey(observationJson.getCode_concept_id())) {
List<Date> observationDates = observationCodeMap.get(observationJson.getCode_concept_id());
// Want dates to be sorted in descending order
observationDates.add(0, effectiveDtm);
// Only keep the three most recent dates for later check.
if (observationDates.size() > 3) {
observationDates.remove(3);
}
} else {
ArrayList<Date> observationDates = new ArrayList<>();
observationDates.add(effectiveDtm);
observationCodeMap.put(observationJson.getCode_concept_id(), observationDates);
}
String subject = observation.getSubject();
if (createdPatientObservationMap.containsKey(subject)) {
Map<String, List<Date>> observationCodeMap = createdPatientObservationMap.get(subject);
if (observationCodeMap.containsKey(observation.getCode_concept_id())) {
List<Date> observationDates = observationCodeMap.get(observation.getCode_concept_id());
observationDates.add(observation.getEffectiveDtm());
observationDates.sort(Collections.reverseOrder());
} else {
ArrayList<Date> observationDates = new ArrayList<>();
observationDates.add(effectiveDtm);
Map<String, List<Date>> codeObservationMap = new HashMap<>();
codeObservationMap.put(observationJson.getCode_concept_id(), observationDates);
createdPatientObservationMap.put(subject, codeObservationMap);
observationDates.add(observation.getEffectiveDtm());
observationCodeMap.put(observation.getCode_concept_id(), observationDates);
}
} else {
ArrayList<Date> observationDates = new ArrayList<>();
observationDates.add(observation.getEffectiveDtm());
Map<String, List<Date>> codeObservationMap = new HashMap<>();
codeObservationMap.put(observation.getCode_concept_id(), observationDates);
createdPatientObservationMap.put(subject, codeObservationMap);
}
}
});
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
@ -585,13 +508,4 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
}
@BeforeAll
public static void beforeClass() {
ourMapperNonPrettyPrint = new ObjectMapper();
ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL);
ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT);
ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
}
}

159
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNElasticsearchSvcSingleObservationIT.java
View File

@ -3,7 +3,7 @@ package ca.uhn.fhir.jpa.search.lastn;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.model.config.PartitionSettings;
import ca.uhn.fhir.jpa.model.util.CodeSystemHash;
import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchContainerHelper;
import ca.uhn.fhir.jpa.search.elastic.TestElasticsearchContainerHelper;
import ca.uhn.fhir.jpa.search.lastn.json.CodeJson;
import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
@ -17,12 +17,7 @@ import ca.uhn.fhir.rest.param.TokenAndListParam;
import ca.uhn.fhir.rest.param.TokenOrListParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@ -32,11 +27,41 @@ import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.CATEGORYFIRSTCODINGSYSTEM;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.CATEGORYSECONDCODINGSYSTEM;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.CATEGORYTHIRDCODINGSYSTEM;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.CODEFIRSTCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.CODEFIRSTCODINGDISPLAY;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.CODEFIRSTCODINGSYSTEM;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.FIRSTCATEGORYFIRSTCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.FIRSTCATEGORYFIRSTCODINGDISPLAY;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.FIRSTCATEGORYSECONDCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.FIRSTCATEGORYSECONDCODINGDISPLAY;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.FIRSTCATEGORYTEXT;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.FIRSTCATEGORYTHIRDCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.FIRSTCATEGORYTHIRDCODINGDISPLAY;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.OBSERVATIONSINGLECODEID;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.OBSERVATION_CODE_CONCEPT_TEXT_1;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.SECONDCATEGORYFIRSTCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.SECONDCATEGORYFIRSTCODINGDISPLAY;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.SECONDCATEGORYSECONDCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.SECONDCATEGORYSECONDCODINGDISPLAY;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.SECONDCATEGORYTEXT;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.SECONDCATEGORYTHIRDCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.SECONDCATEGORYTHIRDCODINGDISPLAY;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.SINGLE_OBSERVATION_RESOURCE_PID;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.SINGLE_OBSERVATION_SUBJECT_ID;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.TEST_BASELINE_TIMESTAMP;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.THIRDCATEGORYFIRSTCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.THIRDCATEGORYFIRSTCODINGDISPLAY;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.THIRDCATEGORYSECONDCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.THIRDCATEGORYSECONDCODINGDISPLAY;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.THIRDCATEGORYTEXT;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.THIRDCATEGORYTHIRDCODINGCODE;
import static ca.uhn.fhir.jpa.search.lastn.LastNTestDataGenerator.THIRDCATEGORYTHIRDCODINGDISPLAY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -45,40 +70,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@Testcontainers
public class LastNElasticsearchSvcSingleObservationIT {
static ObjectMapper ourMapperNonPrettyPrint;
final String RESOURCEPID = "123";
final String SUBJECTID = "Patient/4567";
final Date EFFECTIVEDTM = new Date();
final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category";
final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category";
final String CATEGORYSECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-category";
final String CATEGORYTHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-category";
final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate";
final String FIRSTCATEGORYFIRSTCODINGDISPLAY = "test-heart-rate display";
final String FIRSTCATEGORYSECONDCODINGCODE = "test-alt-heart-rate";
final String FIRSTCATEGORYSECONDCODINGDISPLAY = "test-alt-heart-rate display";
final String FIRSTCATEGORYTHIRDCODINGCODE = "test-2nd-alt-heart-rate";
final String FIRSTCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-heart-rate display";
final String SECONDCATEGORYTEXT = "Test Codeable Concept Field for for second category";
final String SECONDCATEGORYFIRSTCODINGCODE = "test-vital-signs";
final String SECONDCATEGORYFIRSTCODINGDISPLAY = "test-vital-signs display";
final String SECONDCATEGORYSECONDCODINGCODE = "test-alt-vitals";
final String SECONDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display";
final String SECONDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals";
final String SECONDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals display";
final String THIRDCATEGORYTEXT = "Test Codeable Concept Field for third category";
final String THIRDCATEGORYFIRSTCODINGCODE = "test-vital-panel";
final String THIRDCATEGORYFIRSTCODINGDISPLAY = "test-vitals-panel display";
final String THIRDCATEGORYSECONDCODINGCODE = "test-alt-vitals-panel";
final String THIRDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display";
final String THIRDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals-panel";
final String THIRDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals-panel display";
final String OBSERVATIONSINGLECODEID = UUID.randomUUID().toString();
final String OBSERVATIONCODETEXT = "Test Codeable Concept Field for Code";
final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code";
final String CODEFIRSTCODINGCODE = "test-code";
final String CODEFIRSTCODINGDISPLAY = "test-code display";
final FhirContext myFhirContext = FhirContext.forR4Cached();
private final FhirContext myFhirContext = FhirContext.forR4Cached();
ElasticsearchSvcImpl elasticsearchSvc;
@ -104,16 +96,22 @@ public class LastNElasticsearchSvcSingleObservationIT {
@Test
public void testSingleObservationQuery() throws IOException {
createSingleObservation();
// Create test observation
ObservationJson indexedObservation = LastNTestDataGenerator.createSingleObservationJson();
assertTrue(elasticsearchSvc.createOrUpdateObservationIndex(SINGLE_OBSERVATION_RESOURCE_PID, indexedObservation));
assertTrue(elasticsearchSvc.createOrUpdateObservationCodeIndex(OBSERVATIONSINGLECODEID, indexedObservation.getCode()));
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
SearchParameterMap searchParameterMap = new SearchParameterMap();
ReferenceParam subjectParam = new ReferenceParam("Patient", "", SUBJECTID);
ReferenceParam subjectParam = new ReferenceParam("Patient", "", SINGLE_OBSERVATION_SUBJECT_ID);
searchParameterMap.add(Observation.SP_SUBJECT, new ReferenceAndListParam().addAnd(new ReferenceOrListParam().addOr(subjectParam)));
TokenParam categoryParam = new TokenParam(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE);
searchParameterMap.add(Observation.SP_CATEGORY, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(categoryParam)));
TokenParam codeParam = new TokenParam(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE);
searchParameterMap.add(Observation.SP_CODE, new TokenAndListParam().addAnd(new TokenOrListParam().addOr(codeParam)));
searchParameterMap.add(Observation.SP_DATE, new DateParam(ParamPrefixEnum.EQUAL, EFFECTIVEDTM));
searchParameterMap.add(Observation.SP_DATE, new DateParam(ParamPrefixEnum.EQUAL, new Date(TEST_BASELINE_TIMESTAMP)));
searchParameterMap.setLastNMax(3);
@ -121,7 +119,7 @@ public class LastNElasticsearchSvcSingleObservationIT {
List<String> observationIdsOnly = elasticsearchSvc.executeLastN(searchParameterMap, myFhirContext, 100);
assertEquals(1, observationIdsOnly.size());
assertEquals(RESOURCEPID, observationIdsOnly.get(0));
assertEquals(SINGLE_OBSERVATION_RESOURCE_PID, observationIdsOnly.get(0));
// execute Observation search for all search fields
List<ObservationJson> observations = elasticsearchSvc.executeLastNWithAllFieldsForTest(searchParameterMap, myFhirContext);
@ -133,11 +131,11 @@ public class LastNElasticsearchSvcSingleObservationIT {
assertEquals(1, observations.size());
ObservationJson observation = observations.get(0);
assertEquals(RESOURCEPID, observation.getIdentifier());
assertEquals(SINGLE_OBSERVATION_RESOURCE_PID, observation.getIdentifier());
assertEquals(SUBJECTID, observation.getSubject());
assertEquals(RESOURCEPID, observation.getIdentifier());
assertEquals(EFFECTIVEDTM, observation.getEffectiveDtm());
assertEquals(SINGLE_OBSERVATION_SUBJECT_ID, observation.getSubject());
assertEquals(SINGLE_OBSERVATION_RESOURCE_PID, observation.getIdentifier());
assertEquals(new Date(TEST_BASELINE_TIMESTAMP), observation.getEffectiveDtm());
assertEquals(OBSERVATIONSINGLECODEID, observation.getCode_concept_id());
List<String> category_concept_text_values = observation.getCategory_concept_text();
@ -217,7 +215,7 @@ public class LastNElasticsearchSvcSingleObservationIT {
assertEquals(String.valueOf(CodeSystemHash.hashCodeSystem(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE)), category_coding_code_system_hashes.get(2));
String code_concept_text_values = observation.getCode_concept_text();
assertEquals(OBSERVATIONCODETEXT, code_concept_text_values);
assertEquals(OBSERVATION_CODE_CONCEPT_TEXT_1, code_concept_text_values);
String code_coding_systems = observation.getCode_coding_system();
assertEquals(CODEFIRSTCODINGSYSTEM, code_coding_systems);
@ -239,7 +237,7 @@ public class LastNElasticsearchSvcSingleObservationIT {
String persistedCodeConceptID = persistedObservationCode.getCodeableConceptId();
assertEquals(OBSERVATIONSINGLECODEID, persistedCodeConceptID);
String persistedCodeConceptText = persistedObservationCode.getCodeableConceptText();
assertEquals(OBSERVATIONCODETEXT, persistedCodeConceptText);
assertEquals(OBSERVATION_CODE_CONCEPT_TEXT_1, persistedCodeConceptText);
List<String> persistedCodeCodingSystems = persistedObservationCode.getCoding_system();
assertEquals(1, persistedCodeCodingSystems.size());
@ -260,62 +258,5 @@ public class LastNElasticsearchSvcSingleObservationIT {
}
private void createSingleObservation() throws IOException {
ObservationJson indexedObservation = new ObservationJson();
indexedObservation.setIdentifier(RESOURCEPID);
indexedObservation.setSubject(SUBJECTID);
indexedObservation.setEffectiveDtm(EFFECTIVEDTM);
// Add three CodeableConcepts for category
List<CodeJson> categoryConcepts = new ArrayList<>();
// Create three codings and first category CodeableConcept
CodeJson categoryCodeableConcept1 = new CodeJson();
categoryCodeableConcept1.setCodeableConceptText(FIRSTCATEGORYTEXT);
categoryCodeableConcept1.addCoding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY);
categoryCodeableConcept1.addCoding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY);
categoryCodeableConcept1.addCoding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY);
categoryConcepts.add(categoryCodeableConcept1);
// Create three codings and second category CodeableConcept
CodeJson categoryCodeableConcept2 = new CodeJson();
categoryCodeableConcept2.setCodeableConceptText(SECONDCATEGORYTEXT);
categoryCodeableConcept2.addCoding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY);
categoryCodeableConcept2.addCoding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY);
categoryCodeableConcept2.addCoding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY);
categoryConcepts.add(categoryCodeableConcept2);
// Create three codings and third category CodeableConcept
CodeJson categoryCodeableConcept3 = new CodeJson();
categoryCodeableConcept3.setCodeableConceptText(THIRDCATEGORYTEXT);
categoryCodeableConcept3.addCoding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY);
categoryCodeableConcept3.addCoding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY);
categoryCodeableConcept3.addCoding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY);
categoryConcepts.add(categoryCodeableConcept3);
indexedObservation.setCategories(categoryConcepts);
// Create CodeableConcept for Code with three codings.
indexedObservation.setCode_concept_id(OBSERVATIONSINGLECODEID);
CodeJson codeableConceptField = new CodeJson();
codeableConceptField.setCodeableConceptText(OBSERVATIONCODETEXT);
codeableConceptField.addCoding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY);
indexedObservation.setCode(codeableConceptField);
assertTrue(elasticsearchSvc.createOrUpdateObservationIndex(RESOURCEPID, indexedObservation));
codeableConceptField.setCodeableConceptId(OBSERVATIONSINGLECODEID);
assertTrue(elasticsearchSvc.createOrUpdateObservationCodeIndex(OBSERVATIONSINGLECODEID, codeableConceptField));
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
elasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_CODE_INDEX);
}
@BeforeAll
public static void beforeClass() {
ourMapperNonPrettyPrint = new ObjectMapper();
ourMapperNonPrettyPrint.setSerializationInclusion(JsonInclude.Include.NON_NULL);
ourMapperNonPrettyPrint.disable(SerializationFeature.INDENT_OUTPUT);
ourMapperNonPrettyPrint.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
}
}

146
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/lastn/LastNTestDataGenerator.java Normal file
View File

@ -0,0 +1,146 @@
package ca.uhn.fhir.jpa.search.lastn;
import ca.uhn.fhir.jpa.search.lastn.json.CodeJson;
import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
public class LastNTestDataGenerator {
public static final long TEST_BASELINE_TIMESTAMP = new Date().toInstant().toEpochMilli();
public static final String SINGLE_OBSERVATION_RESOURCE_PID = "123";
public static final String SINGLE_OBSERVATION_SUBJECT_ID = "Patient/4567";
public static final String FIRSTCATEGORYTEXT = "Test Codeable Concept Field for first category";
public static final String CATEGORYFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-category";
public static final String CATEGORYSECONDCODINGSYSTEM = "http://myalternatecodes.org/fhir/observation-category";
public static final String CATEGORYTHIRDCODINGSYSTEM = "http://mysecondaltcodes.org/fhir/observation-category";
public static final String FIRSTCATEGORYFIRSTCODINGCODE = "test-heart-rate";
public static final String FIRSTCATEGORYFIRSTCODINGDISPLAY = "Test Heart Rate";
public static final String FIRSTCATEGORYSECONDCODINGCODE = "test-alt-heart-rate";
public static final String FIRSTCATEGORYSECONDCODINGDISPLAY = "Test HeartRate";
public static final String FIRSTCATEGORYTHIRDCODINGCODE = "test-2nd-alt-heart-rate";
public static final String FIRSTCATEGORYTHIRDCODINGDISPLAY = "Test Heart-Rate";
public static final String SECONDCATEGORYTEXT = "Test Codeable Concept Field for for second category";
public static final String SECONDCATEGORYFIRSTCODINGCODE = "test-vital-signs";
public static final String SECONDCATEGORYFIRSTCODINGDISPLAY = "Test Vital Signs";
public static final String SECONDCATEGORYSECONDCODINGCODE = "test-alt-vitals";
public static final String SECONDCATEGORYSECONDCODINGDISPLAY = "Test Vital-Signs";
public static final String SECONDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals";
public static final String SECONDCATEGORYTHIRDCODINGDISPLAY = "Test Vitals";
public static final String THIRDCATEGORYTEXT = "Test Codeable Concept Field for third category";
public static final String THIRDCATEGORYFIRSTCODINGCODE = "test-vital-panel";
public static final String THIRDCATEGORYFIRSTCODINGDISPLAY = "test-vitals-panel display";
public static final String THIRDCATEGORYSECONDCODINGCODE = "test-alt-vitals-panel";
public static final String THIRDCATEGORYSECONDCODINGDISPLAY = "test-alt-vitals display";
public static final String THIRDCATEGORYTHIRDCODINGCODE = "test-2nd-alt-vitals-panel";
public static final String THIRDCATEGORYTHIRDCODINGDISPLAY = "test-2nd-alt-vitals-panel display";
public static final String OBSERVATIONSINGLECODEID = UUID.randomUUID().toString();
public static final String OBSERVATION_CODE_CONCEPT_TEXT_1 = "Test Codeable Concept Field for First Code";
public static final String OBSERVATION_CODE_CONCEPT_TEXT_2 = "Test Codeable Concept Field for Second Code";
public static final String CODEFIRSTCODINGSYSTEM = "http://mycodes.org/fhir/observation-code";
public static final String CODEFIRSTCODINGCODE = "test-code-1";
public static final String CODEFIRSTCODINGDISPLAY = "1-Observation Code1";
public static final String CODE_SECOND_CODING_SYSTEM = "http://mycodes.org/fhir/observation-code";
public static final String CODE_SECOND_CODING_CODE = "test-code-2";
public static final String CODE_SECOND_CODING_DISPLAY = "2-Observation Code2";
public static ObservationJson createSingleObservationJson() {
ObservationJson indexedObservation = new ObservationJson();
indexedObservation.setIdentifier(SINGLE_OBSERVATION_RESOURCE_PID);
indexedObservation.setSubject(SINGLE_OBSERVATION_SUBJECT_ID);
indexedObservation.setEffectiveDtm(new Date(TEST_BASELINE_TIMESTAMP));
indexedObservation.setCategories(createCategoryCodeableConcepts());
// Create CodeableConcept for Code
CodeJson codeableConceptField = new CodeJson();
codeableConceptField.setCodeableConceptId(OBSERVATIONSINGLECODEID);
codeableConceptField.setCodeableConceptText(OBSERVATION_CODE_CONCEPT_TEXT_1);
codeableConceptField.addCoding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY);
indexedObservation.setCode(codeableConceptField);
return indexedObservation;
}
private static List<CodeJson> createCategoryCodeableConcepts() {
CodeJson categoryCodeableConcept1 = new CodeJson();
categoryCodeableConcept1.setCodeableConceptText(FIRSTCATEGORYTEXT);
categoryCodeableConcept1.addCoding(CATEGORYFIRSTCODINGSYSTEM, FIRSTCATEGORYFIRSTCODINGCODE, FIRSTCATEGORYFIRSTCODINGDISPLAY);
categoryCodeableConcept1.addCoding(CATEGORYSECONDCODINGSYSTEM, FIRSTCATEGORYSECONDCODINGCODE, FIRSTCATEGORYSECONDCODINGDISPLAY);
categoryCodeableConcept1.addCoding(CATEGORYTHIRDCODINGSYSTEM, FIRSTCATEGORYTHIRDCODINGCODE, FIRSTCATEGORYTHIRDCODINGDISPLAY);
CodeJson categoryCodeableConcept2 = new CodeJson();
categoryCodeableConcept2.setCodeableConceptText(SECONDCATEGORYTEXT);
categoryCodeableConcept2.addCoding(CATEGORYFIRSTCODINGSYSTEM, SECONDCATEGORYFIRSTCODINGCODE, SECONDCATEGORYFIRSTCODINGDISPLAY);
categoryCodeableConcept2.addCoding(CATEGORYSECONDCODINGSYSTEM, SECONDCATEGORYSECONDCODINGCODE, SECONDCATEGORYSECONDCODINGDISPLAY);
categoryCodeableConcept2.addCoding(CATEGORYTHIRDCODINGSYSTEM, SECONDCATEGORYTHIRDCODINGCODE, SECONDCATEGORYTHIRDCODINGDISPLAY);
CodeJson categoryCodeableConcept3 = new CodeJson();
categoryCodeableConcept3.setCodeableConceptText(THIRDCATEGORYTEXT);
categoryCodeableConcept3.addCoding(CATEGORYFIRSTCODINGSYSTEM, THIRDCATEGORYFIRSTCODINGCODE, THIRDCATEGORYFIRSTCODINGDISPLAY);
categoryCodeableConcept3.addCoding(CATEGORYSECONDCODINGSYSTEM, THIRDCATEGORYSECONDCODINGCODE, THIRDCATEGORYSECONDCODINGDISPLAY);
categoryCodeableConcept3.addCoding(CATEGORYTHIRDCODINGSYSTEM, THIRDCATEGORYTHIRDCODINGCODE, THIRDCATEGORYTHIRDCODINGDISPLAY);
return Arrays.asList(categoryCodeableConcept1, categoryCodeableConcept2, categoryCodeableConcept3);
}
public static List<ObservationJson> createMultipleObservationJson(List<Integer> thePatientIds) {
// CodeableConcept 1 - with 3 codings
String codeableConceptId1 = UUID.randomUUID().toString();
CodeJson codeJson1 = new CodeJson();
codeJson1.setCodeableConceptId(codeableConceptId1);
codeJson1.setCodeableConceptText(OBSERVATION_CODE_CONCEPT_TEXT_1);
codeJson1.addCoding(CODEFIRSTCODINGSYSTEM, CODEFIRSTCODINGCODE, CODEFIRSTCODINGDISPLAY);
// CodeableConcept 2 - with 3 codings
String codeableConceptId2 = UUID.randomUUID().toString();
CodeJson codeJson2 = new CodeJson();
codeJson2.setCodeableConceptId(codeableConceptId2);
codeJson2.setCodeableConceptText(OBSERVATION_CODE_CONCEPT_TEXT_2);
codeJson2.addCoding(CODE_SECOND_CODING_SYSTEM, CODE_SECOND_CODING_CODE, CODE_SECOND_CODING_DISPLAY);
List<CodeJson> categoryCodeableConcepts = createCategoryCodeableConcepts();
// CategoryCodeableConcept 1 - with 3 codings
List<CodeJson> categoryConcepts1 = Collections.singletonList(categoryCodeableConcepts.get(0));
// CateogryCodeableConcept 2 - with 3 codings
List<CodeJson> categoryConcepts2 = Collections.singletonList(categoryCodeableConcepts.get(1));
// Pair CodeableConcept 1 + CategoryCodeableConcept 1 for odd numbered observation
// Pair CodeableConcept 2 + CategoryCodeableConcept 2 for even numbered observation
// For each patient - create 10 observations
return thePatientIds.stream()
.flatMap(patientId -> IntStream.range(0, 10)
.mapToObj(index -> {
ObservationJson observationJson = new ObservationJson();
String identifier = String.valueOf((index + patientId * 10L));
observationJson.setIdentifier(identifier);
observationJson.setSubject(String.valueOf(patientId));
if (index % 2 == 1) {
observationJson.setCategories(categoryConcepts1);
observationJson.setCode(codeJson1);
} else {
observationJson.setCategories(categoryConcepts2);
observationJson.setCode(codeJson2);
}
Date effectiveDtm = new Date(TEST_BASELINE_TIMESTAMP - ((10L - index) * 3600L * 1000L));
observationJson.setEffectiveDtm(effectiveDtm);
return observationJson;
}))
.collect(Collectors.toList());
}
}

11
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/BasePartitionable.java
View File

@ -21,6 +21,7 @@ package ca.uhn.fhir.jpa.model.entity;
*/
import ca.uhn.fhir.interceptor.model.RequestPartitionId;
import org.slf4j.Logger;
import javax.annotation.Nullable;
import javax.persistence.Column;
@ -28,9 +29,12 @@ import javax.persistence.Embedded;
import javax.persistence.MappedSuperclass;
import java.io.Serializable;
import static org.slf4j.LoggerFactory.getLogger;
@MappedSuperclass
public class BasePartitionable implements Serializable {
@Embedded
private PartitionablePartitionId myPartitionId;
@ -50,4 +54,11 @@ public class BasePartitionable implements Serializable {
myPartitionId = thePartitionId;
}
@Override
public String toString() {
return "BasePartitionable{" +
"myPartitionId=" + myPartitionId +
", myPartitionIdValue=" + myPartitionIdValue +
'}';
}
}

7
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/PartitionablePartitionId.java
View File

@ -104,9 +104,10 @@ public class PartitionablePartitionId implements Cloneable {
@Override
public String toString() {
return "[" +
getPartitionId() +
"]";
return "PartitionablePartitionId{" +
"myPartitionId=" + myPartitionId +
", myPartitionDate=" + myPartitionDate +
'}';
}
@Nonnull

33
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/DateSearchIndexData.java Normal file
View File

@ -0,0 +1,33 @@
package ca.uhn.fhir.jpa.model.search;
import java.util.Date;
class DateSearchIndexData {
private final Date myLowerBoundDate;
private final int myLowerBoundOrdinal;
private final Date myUpperBoundDate;
private final int myUpperBoundOrdinal;
DateSearchIndexData(Date theLowerBoundDate, int theLowerBoundOrdinal, Date theUpperBoundDate, int theUpperBoundOrdinal) {
myLowerBoundDate = theLowerBoundDate;
myLowerBoundOrdinal = theLowerBoundOrdinal;
myUpperBoundDate = theUpperBoundDate;
myUpperBoundOrdinal = theUpperBoundOrdinal;
}
public Date getLowerBoundDate() {
return myLowerBoundDate;
}
public int getLowerBoundOrdinal() {
return myLowerBoundOrdinal;
}
public Date getUpperBoundDate() {
return myUpperBoundDate;
}
public int getUpperBoundOrdinal() {
return myUpperBoundOrdinal;
}
}

10
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/ExtendedLuceneIndexData.java
View File

@ -28,6 +28,8 @@ import org.hibernate.search.engine.backend.document.DocumentElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Date;
/**
* Collects our lucene extended indexing data.
*
@ -39,6 +41,7 @@ public class ExtendedLuceneIndexData {
final SetMultimap<String, String> mySearchParamStrings = HashMultimap.create();
final SetMultimap<String, TokenParam> mySearchParamTokens = HashMultimap.create();
final SetMultimap<String, String> mySearchParamLinks = HashMultimap.create();
final SetMultimap<String, DateSearchIndexData> mySearchParamDates = HashMultimap.create();
public ExtendedLuceneIndexData(FhirContext theFhirContext) {
this.myFhirContext = theFhirContext;
@ -51,6 +54,7 @@ public class ExtendedLuceneIndexData {
mySearchParamStrings.forEach(indexWriter::writeStringIndex);
mySearchParamTokens.forEach(indexWriter::writeTokenIndex);
mySearchParamLinks.forEach(indexWriter::writeReferenceIndex);
mySearchParamDates.forEach(indexWriter::writeDateIndex);
}
public void addStringIndexData(String theSpName, String theText) {
@ -61,7 +65,11 @@ public class ExtendedLuceneIndexData {
mySearchParamTokens.put(theSpName, new TokenParam(theSystem, theValue));
}
public void addResourceLinkIndexData(String theSpName, String theTargetResourceId){
public void addResourceLinkIndexData(String theSpName, String theTargetResourceId) {
mySearchParamLinks.put(theSpName, theTargetResourceId);
}
public void addDateIndexData(String theSpName, Date theLowerBound, int theLowerBoundOrdinal, Date theUpperBound, int theUpperBoundOrdinal) {
mySearchParamDates.put(theSpName, new DateSearchIndexData(theLowerBound, theLowerBoundOrdinal, theUpperBound, theUpperBoundOrdinal));
}
}

21
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/HibernateSearchIndexWriter.java
View File

@ -67,9 +67,20 @@ public class HibernateSearchIndexWriter {
ourLog.debug("Adding Search Param Token: {} -- {}", theSearchParam, theValue);
}
public void writeReferenceIndex(String theSearchParam, String theValue) {
DocumentElement referenceIndexNode = getSearchParamIndexNode(theSearchParam, "reference");
referenceIndexNode.addValue("value", theValue);
ourLog.trace("Adding Search Param Reference: {} -- {}", theSearchParam, theValue);
}
public void writeReferenceIndex(String theSearchParam, String theValue) {
DocumentElement referenceIndexNode = getSearchParamIndexNode(theSearchParam, "reference");
referenceIndexNode.addValue("value", theValue);
ourLog.trace("Adding Search Param Reference: {} -- {}", theSearchParam, theValue);
}
public void writeDateIndex(String theSearchParam, DateSearchIndexData theValue) {
DocumentElement dateIndexNode = getSearchParamIndexNode(theSearchParam, "dt");
// Lower bound
dateIndexNode.addValue("lower-ord", theValue.getLowerBoundOrdinal());
dateIndexNode.addValue("lower", theValue.getLowerBoundDate().toInstant());
// Upper bound
dateIndexNode.addValue("upper-ord", theValue.getUpperBoundOrdinal());
dateIndexNode.addValue("upper", theValue.getUpperBoundDate().toInstant());
ourLog.trace("Adding Search Param Reference: {} -- {}", theSearchParam, theValue);
}
}

22
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/SearchParamTextPropertyBinder.java
View File

@ -26,7 +26,9 @@ import org.hibernate.search.engine.backend.document.model.dsl.IndexSchemaObjectF
import org.hibernate.search.engine.backend.types.Aggregable;
import org.hibernate.search.engine.backend.types.ObjectStructure;
import org.hibernate.search.engine.backend.types.Projectable;
import org.hibernate.search.engine.backend.types.Sortable;
import org.hibernate.search.engine.backend.types.dsl.IndexFieldTypeFactory;
import org.hibernate.search.engine.backend.types.dsl.StandardIndexFieldTypeOptionsStep;
import org.hibernate.search.engine.backend.types.dsl.StringIndexFieldTypeOptionsStep;
import org.hibernate.search.mapper.pojo.bridge.PropertyBridge;
import org.hibernate.search.mapper.pojo.bridge.binding.PropertyBindingContext;
@ -35,6 +37,9 @@ import org.hibernate.search.mapper.pojo.bridge.runtime.PropertyBridgeWriteContex
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Instant;
import java.time.LocalDateTime;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_EXACT;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_NORMALIZED;
import static ca.uhn.fhir.jpa.model.search.HibernateSearchIndexWriter.IDX_STRING_TEXT;
@ -88,6 +93,13 @@ public class SearchParamTextPropertyBinder implements PropertyBinder, PropertyBr
.projectable(Projectable.NO)
.aggregable(Aggregable.YES);
StandardIndexFieldTypeOptionsStep<?, Instant> dateTimeFieldType = indexFieldTypeFactory.asInstant()
.projectable(Projectable.NO)
.sortable(Sortable.YES);
StandardIndexFieldTypeOptionsStep<?, Integer> dateTimeOrdinalFieldType = indexFieldTypeFactory.asInteger()
.projectable(Projectable.NO)
.sortable(Sortable.YES);
// the old style for _text and _contains
indexSchemaElement
@ -127,9 +139,19 @@ public class SearchParamTextPropertyBinder implements PropertyBinder, PropertyBr
spfield.fieldTemplate("token-code-system", keywordFieldType).matchingPathGlob(tokenPathGlob + ".code-system").multiValued();
spfield.fieldTemplate("token-system", keywordFieldType).matchingPathGlob(tokenPathGlob + ".system").multiValued();
// reference
// reference
spfield.fieldTemplate("reference-value", keywordFieldType).matchingPathGlob("*.reference.value").multiValued();
// date
String dateTimePathGlob = "*.dt";
spfield.objectFieldTemplate("datetimeIndex", ObjectStructure.FLATTENED).matchingPathGlob(dateTimePathGlob);
spfield.fieldTemplate("datetime-lower-ordinal", dateTimeOrdinalFieldType).matchingPathGlob(dateTimePathGlob + ".lower-ord");
spfield.fieldTemplate("datetime-lower-value", dateTimeFieldType).matchingPathGlob(dateTimePathGlob + ".lower");
spfield.fieldTemplate("datetime-upper-ordinal", dateTimeOrdinalFieldType).matchingPathGlob(dateTimePathGlob + ".upper-ord");
spfield.fieldTemplate("datetime-upper-value", dateTimeFieldType).matchingPathGlob(dateTimePathGlob + ".upper");
// last, since the globs are matched in declaration order, and * matches even nested nodes.
spfield.objectFieldTemplate("spObject", ObjectStructure.FLATTENED).matchingPathGlob("*");
}

11
hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java
View File

@ -115,12 +115,13 @@ public class SearchParameterMap implements Serializable {
map.setSearchContainedMode(getSearchContainedMode());
for (Map.Entry<String, List<List<IQueryParameterType>>> entry : mySearchParameterMap.entrySet()) {
List<List<IQueryParameterType>> params = entry.getValue();
for (List<IQueryParameterType> p : params) {
for (IQueryParameterType t : p) {
map.add(entry.getKey(), t);
}
List<List<IQueryParameterType>> andParams = entry.getValue();
List<List<IQueryParameterType>> newAndParams = new ArrayList<>();
for(List<IQueryParameterType> orParams: andParams) {
List<IQueryParameterType> newOrParams = new ArrayList<>(orParams);
newAndParams.add(newOrParams);
}
map.put(entry.getKey(), newAndParams);
}
return map;

9
hapi-fhir-test-utilities/src/main/resources/ca/uhn/fhir/jpa/conformance/DateSearchTestCase.csv
View File

@ -159,3 +159,12 @@
2019-12-31T00:00:00.000,lt2020-01-01T08:00:00.000Z, True
2020-01-01T08:00:00.000Z,lt2020-01-01T08:00:00.000Z, False
2019-01-01T12:00:00.000Z,lt2020-01-01T08:00:00.000Z, True
2019-01-01T01:00:00.000Z,ne2019-01-01T01:00:00.000Z, False
2019-01-01T01:00:00.000Z,ne2019-01-01T01:59:00.000Z, True
2020-01-01,gt2019-12-31T23:59:59.999Z, True
2020-01-01,lt2021-01-01T00:00:00.000Z, True
2019-12-31T23:59:59.999Z,eq2019-12-31T23:59:59.999Z, True
2019-12-31T23:59:59.999Z,gt2019, False
2019-12-31T23:59:59.999Z,lt2020, True
2019-12-31T23:59:59.999Z,le2019-12-31T23:59:59.998Z, False
2019-12-31T23:59:59.999Z,gt2019-12-31T23:59:59.998Z, True

1 #ObservationDate, Query, Result, Comment (ignored)
159 2019-12-31T00:00:00.000,lt2020-01-01T08:00:00.000Z, True
160 2020-01-01T08:00:00.000Z,lt2020-01-01T08:00:00.000Z, False
161 2019-01-01T12:00:00.000Z,lt2020-01-01T08:00:00.000Z, True
162 2019-01-01T01:00:00.000Z,ne2019-01-01T01:00:00.000Z, False
163 2019-01-01T01:00:00.000Z,ne2019-01-01T01:59:00.000Z, True
164 2020-01-01,gt2019-12-31T23:59:59.999Z, True
165 2020-01-01,lt2021-01-01T00:00:00.000Z, True
166 2019-12-31T23:59:59.999Z,eq2019-12-31T23:59:59.999Z, True
167 2019-12-31T23:59:59.999Z,gt2019, False
168 2019-12-31T23:59:59.999Z,lt2020, True
169 2019-12-31T23:59:59.999Z,le2019-12-31T23:59:59.998Z, False
170 2019-12-31T23:59:59.999Z,gt2019-12-31T23:59:59.998Z, True