Implement :below qualifier for tag parameters (#3613)

* Initial implementation

* Implement job cancellation

* Implement :below qualifier for tag parameters

* Add changelog

* Add tests

Co-authored-by: juan.marchionatto <juan.marchionatto@smilecdr.com>

hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java

@@ -25,7 +25,7 @@ public final class Msg {
 
 	/**
 	 * IMPORTANT: Please update the following comment after you add a new code
-	 * Last code value: 2078
+	 * Last code value: 2079
 	 */
 
 	private Msg() {}

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3612-below-modifier-not-working-with-tag-parameters.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 3612
+jira: SMILE-4112
+title: "Search with tag parameters using `:below` qualifier was not working. This is now fixed."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java

@@ -31,11 +31,12 @@ import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity;
 import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity;
 import org.apache.commons.lang3.Validate;
 import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Sort;
 
 import javax.annotation.Nonnull;
 import javax.transaction.Transactional;
+import java.util.Collection;
 import java.util.Date;
-import java.util.EnumSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
@@ -113,7 +114,15 @@ public class JpaJobPersistenceImpl implements IJobPersistence {
 
 	@Override
 	public List<JobInstance> fetchInstances(int thePageSize, int thePageIndex) {
-		return myJobInstanceRepository.fetchAll(PageRequest.of(thePageIndex, thePageSize)).stream().map(t -> toInstance(t)).collect(Collectors.toList());
+		// default sort is myCreateTime Asc
+		PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.ASC, "myCreateTime");
+		return myJobInstanceRepository.findAll(pageRequest).stream().map(t -> toInstance(t)).collect(Collectors.toList());
+	}
+
+	@Override
+	public Collection<JobInstance> fetchRecentInstances(int thePageSize, int thePageIndex) {
+		PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.DESC, "myCreateTime");
+		return myJobInstanceRepository.findAll(pageRequest).stream().map(this::toInstance).collect(Collectors.toList());
 	}
 
 	private WorkChunk toChunk(Batch2WorkChunkEntity theEntity, boolean theIncludeData) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2JobInstanceRepository.java

@@ -22,23 +22,17 @@ package ca.uhn.fhir.jpa.dao.data;
 
 import ca.uhn.fhir.batch2.model.StatusEnum;
 import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity;
-import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 
-import java.util.List;
-
 public interface IBatch2JobInstanceRepository extends JpaRepository<Batch2JobInstanceEntity, String>, IHapiFhirJpaRepository {
 
 	@Modifying
 	@Query("UPDATE Batch2JobInstanceEntity e SET e.myStatus = :status WHERE e.myId = :id")
 	void updateInstanceStatus(@Param("id") String theInstanceId, @Param("status") StatusEnum theInProgress);
 
-	@Query("SELECT e FROM Batch2JobInstanceEntity e ORDER BY e.myCreateTime ASC")
-	List<Batch2JobInstanceEntity> fetchAll(Pageable thePageRequest);
-
 	@Modifying
 	@Query("UPDATE Batch2JobInstanceEntity e SET e.myCancelled = :cancelled WHERE e.myId = :id")
 	void updateInstanceCancelled(@Param("id") String theInstanceId, @Param("cancelled") boolean theCancelled);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/QueryStack.java

@@ -101,7 +101,7 @@ import org.apache.commons.collections4.bidimap.UnmodifiableBidiMap;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.Triple;
 import org.hl7.fhir.instance.model.api.IAnyResource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -1221,52 +1221,11 @@ public class QueryStack {
 
 		List<Condition> andPredicates = new ArrayList<>();
 		for (List<? extends IQueryParameterType> nextAndParams : theList) {
-			boolean haveTags = false;
+			if ( ! checkHaveTags(nextAndParams, theParamName)) { continue; }
-			for (IQueryParameterType nextParamUncasted : nextAndParams) {
-				if (nextParamUncasted instanceof TokenParam) {
-					TokenParam nextParam = (TokenParam) nextParamUncasted;
-					if (isNotBlank(nextParam.getValue())) {
-						haveTags = true;
-					} else if (isNotBlank(nextParam.getSystem())) {
-						throw new InvalidRequestException(Msg.code(1218) + "Invalid " + theParamName + " parameter (must supply a value/code and not just a system): " + nextParam.getValueAsQueryToken(myFhirContext));
-					}
-				} else {
-					UriParam nextParam = (UriParam) nextParamUncasted;
-					if (isNotBlank(nextParam.getValue())) {
-						haveTags = true;
-					}
-				}
-			}
-			if (!haveTags) {
-				continue;
-			}
 
-			boolean paramInverted = false;
+			List<Triple<String, String, String>> tokens = Lists.newArrayList();
-			List<Pair<String, String>> tokens = Lists.newArrayList();
+			boolean paramInverted = populateTokens(tokens, nextAndParams);
-			for (IQueryParameterType nextOrParams : nextAndParams) {
+			if (tokens.isEmpty()) { continue; }
-				String code;
-				String system;
-				if (nextOrParams instanceof TokenParam) {
-					TokenParam nextParam = (TokenParam) nextOrParams;
-					code = nextParam.getValue();
-					system = nextParam.getSystem();
-					if (nextParam.getModifier() == TokenParamModifier.NOT) {
-						paramInverted = true;
-					}
-				} else {
-					UriParam nextParam = (UriParam) nextOrParams;
-					code = nextParam.getValue();
-					system = null;
-				}
-
-				if (isNotBlank(code)) {
-					tokens.add(Pair.of(system, code));
-				}
-			}
-
-			if (tokens.isEmpty()) {
-				continue;
-			}
 
 			Condition tagPredicate;
 			BaseJoiningPredicateBuilder join;
@@ -1296,6 +1255,50 @@ public class QueryStack {
 		return toAndPredicate(andPredicates);
 	}
 
+	private boolean populateTokens(List<Triple<String, String, String>> theTokens, List<? extends IQueryParameterType> theAndParams) {
+		boolean paramInverted = false;
+
+		for (IQueryParameterType nextOrParam : theAndParams) {
+			String code;
+			String system;
+			if (nextOrParam instanceof TokenParam) {
+				TokenParam nextParam = (TokenParam) nextOrParam;
+				code = nextParam.getValue();
+				system = nextParam.getSystem();
+				if (nextParam.getModifier() == TokenParamModifier.NOT) {
+					paramInverted = true;
+				}
+			} else {
+				UriParam nextParam = (UriParam) nextOrParam;
+				code = nextParam.getValue();
+				system = null;
+			}
+
+			if (isNotBlank(code)) {
+				theTokens.add(Triple.of(system, nextOrParam.getQueryParameterQualifier(), code));
+			}
+		}
+		return paramInverted;
+	}
+
+	private boolean checkHaveTags(List<? extends IQueryParameterType> theParams, String theParamName) {
+		for (IQueryParameterType nextParamUncasted : theParams) {
+			if (nextParamUncasted instanceof TokenParam) {
+				TokenParam nextParam = (TokenParam) nextParamUncasted;
+				if (isNotBlank(nextParam.getValue())) { return true; }
+				if (isNotBlank(nextParam.getSystem())) {
+					throw new InvalidRequestException(Msg.code(1218) + "Invalid " + theParamName +
+						" parameter (must supply a value/code and not just a system): " + nextParam.getValueAsQueryToken(myFhirContext));
+				}
+			}
+
+			UriParam nextParam = (UriParam) nextParamUncasted;
+			if (isNotBlank(nextParam.getValue())) { return true; }
+		}
+
+		return false;
+	}
+
 	public Condition createPredicateToken(@Nullable DbColumn theSourceJoinColumn, String theResourceName,
 													  String theSpnamePrefix, RuntimeSearchParam theSearchParam, List<? extends IQueryParameterType> theList,
 													  SearchFilterParser.CompareOperation theOperation, RequestPartitionId theRequestPartitionId) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/TagPredicateBuilder.java

@@ -24,17 +24,19 @@ import ca.uhn.fhir.interceptor.model.RequestPartitionId;
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
 import ca.uhn.fhir.jpa.model.entity.TagTypeEnum;
 import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder;
+import ca.uhn.fhir.rest.param.UriParamQualifierEnum;
 import com.google.common.collect.Lists;
 import com.healthmarketscience.sqlbuilder.BinaryCondition;
 import com.healthmarketscience.sqlbuilder.ComboCondition;
 import com.healthmarketscience.sqlbuilder.Condition;
-import com.healthmarketscience.sqlbuilder.UnaryCondition;
 import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn;
 import com.healthmarketscience.sqlbuilder.dbspec.basic.DbTable;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.Triple;
 
 import java.util.List;
+import java.util.Objects;
 
+import static ca.uhn.fhir.jpa.search.builder.predicate.StringPredicateBuilder.createLeftMatchLikeExpression;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 public class TagPredicateBuilder extends BaseJoiningPredicateBuilder {
@@ -61,24 +63,28 @@ public class TagPredicateBuilder extends BaseJoiningPredicateBuilder {
 	}
 
 
-	public Condition createPredicateTag(TagTypeEnum theTagType, List<Pair<String, String>> theTokens, String theParamName, RequestPartitionId theRequestPartitionId) {
+	public Condition createPredicateTag(TagTypeEnum theTagType, List<Triple<String, String, String>> theTokens, String theParamName, RequestPartitionId theRequestPartitionId) {
 		addJoin(getTable(), myTagDefinitionTable, myColumnTagId, myTagDefinitionColumnTagId);
 		return createPredicateTagList(theTagType, theTokens);
 	}
 
-	private Condition createPredicateTagList(TagTypeEnum theTagType, List<Pair<String, String>> theTokens) {
+	private Condition createPredicateTagList(TagTypeEnum theTagType, List<Triple<String, String, String>> theTokens) {
 		Condition typePredicate = BinaryCondition.equalTo(myTagDefinitionColumnTagType, generatePlaceholder(theTagType.ordinal()));
 
 		List<Condition> orPredicates = Lists.newArrayList();
-		for (Pair<String, String> next : theTokens) {
+		for (Triple<String, String, String> next : theTokens) {
 			String system = next.getLeft();
 			String code = next.getRight();
+			String qualifier = next.getMiddle();
 
 			if (theTagType == TagTypeEnum.PROFILE) {
 				system = BaseHapiFhirDao.NS_JPA_PROFILE;
 			}
 
-			Condition codePredicate = BinaryCondition.equalTo(myTagDefinitionColumnTagCode, generatePlaceholder(code));
+			Condition codePredicate = Objects.equals(qualifier, UriParamQualifierEnum.BELOW.getValue())
+				? BinaryCondition.like(myTagDefinitionColumnTagCode, generatePlaceholder(createLeftMatchLikeExpression(code)))
+				: BinaryCondition.equalTo(myTagDefinitionColumnTagCode, generatePlaceholder(code));
+
 			if (isNotBlank(system)) {
 				Condition systemPredicate = BinaryCondition.equalTo(myTagDefinitionColumnTagSystem, generatePlaceholder(system));
 				orPredicates.add(ComboCondition.and(typePredicate, systemPredicate, codePredicate));

hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java

@@ -101,6 +101,7 @@ import org.hl7.fhir.r4.model.Location;
 import org.hl7.fhir.r4.model.Medication;
 import org.hl7.fhir.r4.model.MedicationAdministration;
 import org.hl7.fhir.r4.model.MedicationRequest;
+import org.hl7.fhir.r4.model.Meta;
 import org.hl7.fhir.r4.model.MolecularSequence;
 import org.hl7.fhir.r4.model.Observation;
 import org.hl7.fhir.r4.model.Observation.ObservationStatus;
@@ -132,6 +133,7 @@ import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.ArgumentCaptor;
@@ -157,6 +159,9 @@ import java.util.Set;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
 
+import static ca.uhn.fhir.rest.api.Constants.PARAM_PROFILE;
+import static ca.uhn.fhir.rest.api.Constants.PARAM_SECURITY;
+import static ca.uhn.fhir.rest.api.Constants.PARAM_TAG;
 import static ca.uhn.fhir.rest.api.Constants.PARAM_TYPE;
 import static org.apache.commons.lang3.StringUtils.countMatches;
 import static org.apache.commons.lang3.StringUtils.leftPad;
@@ -5641,6 +5646,96 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
 		}
 	}
 
+
+	@Nested
+	public class TagBelowTests {
+
+		@Test
+		public void testTagProfile() {
+			Patient p1 = new Patient();
+			p1.setActive(true);
+			p1.setMeta(new Meta().addProfile("http://acme.com/some-profile|1.0"));
+			IIdType p1Id = myPatientDao.create(p1).getId().toUnqualifiedVersionless();
+
+			Patient p2 = new Patient();
+			p2.setActive(true);
+			p2.setMeta(new Meta().addProfile("http://acme.com/some-profile|1.1"));
+			IIdType p2Id = myPatientDao.create(p2).getId().toUnqualifiedVersionless();
+
+			Patient p3 = new Patient();
+			p3.setActive(true);
+			p3.setMeta(new Meta().addProfile("http://acme.com/some-profile|2.0"));
+			IIdType p3Id = myPatientDao.create(p3).getId().toUnqualifiedVersionless();
+
+			SearchParameterMap params = new SearchParameterMap();
+			params.setLoadSynchronous(true);
+			params.add(PARAM_PROFILE, new UriParam(
+				"http://acme.com/some-profile|1").setQualifier(UriParamQualifierEnum.BELOW));
+			IBundleProvider results = myPatientDao.search(params);
+			List<String> values = toUnqualifiedVersionlessIdValues(results);
+
+			assertThat(values.toString(), values, containsInAnyOrder(p1Id.getValue(), p2Id.getValue()));
+		}
+
+		@Test
+		public void testTagTag() {
+			Patient p1 = new Patient();
+			p1.setActive(true);
+			p1.setMeta(new Meta().addTag("http://acme.com", "some-code", "some-display"));
+			IIdType p1Id = myPatientDao.create(p1).getId().toUnqualifiedVersionless();
+
+			Patient p2 = new Patient();
+			p2.setActive(true);
+			p2.setMeta(new Meta().addTag("http://acme.com", "some-code-2", "some-display-2"));
+			IIdType p2Id = myPatientDao.create(p2).getId().toUnqualifiedVersionless();
+
+			Patient p3 = new Patient();
+			p3.setActive(true);
+			p3.setMeta(new Meta().addTag("http://acme.com", "another-code", "another-display"));
+			IIdType p3Id = myPatientDao.create(p3).getId().toUnqualifiedVersionless();
+
+			SearchParameterMap params = new SearchParameterMap();
+			params.setLoadSynchronous(true);
+			params.add(PARAM_TAG, new TokenParam("http://acme.com", "some").setModifier(TokenParamModifier.BELOW));
+			IBundleProvider results = myPatientDao.search(params);
+			List<String> values = toUnqualifiedVersionlessIdValues(results);
+
+			assertThat(values.toString(), values, containsInAnyOrder(p1Id.getValue(), p2Id.getValue()));
+		}
+
+		@Test
+		public void testSecurityLabelTag() {
+			Patient p1 = new Patient();
+			p1.setActive(true);
+			p1.setMeta(new Meta().addSecurity(
+				"http://terminology.hl7.org/CodeSystem/v3-Confidentiality", "DELAU", "delete after use"));
+			IIdType p1Id = myPatientDao.create(p1).getId().toUnqualifiedVersionless();
+
+			Patient p2 = new Patient();
+			p2.setActive(true);
+			p2.setMeta(new Meta().addSecurity(
+				"http://terminology.hl7.org/CodeSystem/v3-Confidentiality", "DELBU", "delete before use"));
+			IIdType p2Id = myPatientDao.create(p2).getId().toUnqualifiedVersionless();
+
+			Patient p3 = new Patient();
+			p3.setActive(true);
+			p3.setMeta(new Meta().addSecurity(
+				"http://terminology.hl7.org/CodeSystem/v3-Confidentiality", "R", "restricted"));
+			IIdType p3Id = myPatientDao.create(p3).getId().toUnqualifiedVersionless();
+
+			SearchParameterMap params = new SearchParameterMap();
+			params.setLoadSynchronous(true);
+			params.add(PARAM_SECURITY, new TokenParam(
+				"http://terminology.hl7.org/CodeSystem/v3-Confidentiality", "DEL").setModifier(TokenParamModifier.BELOW));
+			IBundleProvider results = myPatientDao.search(params);
+			List<String> values = toUnqualifiedVersionlessIdValues(results);
+
+			assertThat(values.toString(), values, containsInAnyOrder(p1Id.getValue(), p2Id.getValue()));
+		}
+
+	}
+
+
 	private String toStringMultiline(List<?> theResults) {
 		StringBuilder b = new StringBuilder();
 		for (Object next : theResults) {

hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobCoordinator.java

@@ -52,6 +52,8 @@ public interface IJobCoordinator {
 	 */
 	List<JobInstance> getInstances(int thePageSize, int thePageIndex);
 
+	List<JobInstance> getRecentInstances(int thePageSize, int thePageIndex);
+
 	void cancelInstance(String theInstanceId) throws ResourceNotFoundException;
 
 }

hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java

@@ -24,6 +24,7 @@ import ca.uhn.fhir.batch2.impl.BatchWorkChunk;
 import ca.uhn.fhir.batch2.model.JobInstance;
 import ca.uhn.fhir.batch2.model.WorkChunk;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 
@@ -68,6 +69,11 @@ public interface IJobPersistence {
 	 */
 	List<JobInstance> fetchInstances(int thePageSize, int thePageIndex);
 
+	/**
+	 * Fetch instance in 'myCreateTime' descending order
+	 */
+	Collection<JobInstance> fetchRecentInstances(int thePageSize, int thePageIndex);
+
 	/**
 	 * Fetch a given instance and update the stored status
 	 * * to {@link ca.uhn.fhir.batch2.model.StatusEnum#IN_PROGRESS}

hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/impl/JobCoordinatorImpl.java

@@ -161,6 +161,12 @@ public class JobCoordinatorImpl extends BaseJobService implements IJobCoordinato
 		return myJobPersistence.fetchInstances(thePageSize, thePageIndex).stream().map(t -> massageInstanceForUserAccess(t)).collect(Collectors.toList());
 	}
 
+	@Override
+	public List<JobInstance> getRecentInstances(int thePageSize, int thePageIndex) {
+		return myJobPersistence.fetchRecentInstances(thePageSize, thePageIndex).stream()
+			.map(this::massageInstanceForUserAccess).collect(Collectors.toList());
+	}
+
 	@Override
 	public void cancelInstance(String theInstanceId) throws ResourceNotFoundException {
 		myJobPersistence.cancelInstance(theInstanceId);

hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/impl/SynchronizedJobPersistenceWrapper.java

@@ -22,10 +22,9 @@ package ca.uhn.fhir.batch2.impl;
 
 import ca.uhn.fhir.batch2.api.IJobPersistence;
 import ca.uhn.fhir.batch2.model.JobInstance;
-import ca.uhn.fhir.batch2.model.StatusEnum;
 import ca.uhn.fhir.batch2.model.WorkChunk;
 
-import java.util.EnumSet;
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 
@@ -65,6 +64,11 @@ public class SynchronizedJobPersistenceWrapper implements IJobPersistence {
 		return myWrap.fetchInstances(thePageSize, thePageIndex);
 	}
 
+	@Override
+	public Collection<JobInstance> fetchRecentInstances(int thePageSize, int thePageIndex) {
+		return myWrap.fetchRecentInstances(thePageSize, thePageIndex);
+	}
+
 	@Override
 	public synchronized Optional<JobInstance> fetchInstanceAndMarkInProgress(String theInstanceId) {
 		return myWrap.fetchInstanceAndMarkInProgress(theInstanceId);