Move query count tests to consolidate them, and avoid an accidental

rewrite of existing indexes in some conditions

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/gclient/StringClientParam.java

@@ -31,7 +31,7 @@ import java.util.List;
  * @author james
  *
  */
-public class StringClientParam extends BaseClientParam implements IParam {
+public class  StringClientParam extends BaseClientParam implements IParam {
 
 	private final String myParamName;
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java

@@ -2086,6 +2086,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 		 */
 		if (thePerformIndexing) {
 
+			calculateHashes(stringParams);
 			for (ResourceIndexedSearchParamString next : removeCommon(existingStringParams, stringParams)) {
 				next.setDaoConfig(myConfig);
 				myEntityManager.remove(next);
@@ -2095,6 +2096,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 				myEntityManager.persist(next);
 			}
 
+			calculateHashes(tokenParams);
 			for (ResourceIndexedSearchParamToken next : removeCommon(existingTokenParams, tokenParams)) {
 				myEntityManager.remove(next);
 				theEntity.getParamsToken().remove(next);
@@ -2103,6 +2105,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 				myEntityManager.persist(next);
 			}
 
+			calculateHashes(numberParams);
 			for (ResourceIndexedSearchParamNumber next : removeCommon(existingNumberParams, numberParams)) {
 				myEntityManager.remove(next);
 				theEntity.getParamsNumber().remove(next);
@@ -2111,6 +2114,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 				myEntityManager.persist(next);
 			}
 
+			calculateHashes(quantityParams);
 			for (ResourceIndexedSearchParamQuantity next : removeCommon(existingQuantityParams, quantityParams)) {
 				myEntityManager.remove(next);
 				theEntity.getParamsQuantity().remove(next);
@@ -2120,6 +2124,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 			}
 
 			// Store date SP's
+			calculateHashes(dateParams);
 			for (ResourceIndexedSearchParamDate next : removeCommon(existingDateParams, dateParams)) {
 				myEntityManager.remove(next);
 				theEntity.getParamsDate().remove(next);
@@ -2129,6 +2134,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 			}
 
 			// Store URI SP's
+			calculateHashes(uriParams);
 			for (ResourceIndexedSearchParamUri next : removeCommon(existingUriParams, uriParams)) {
 				myEntityManager.remove(next);
 				theEntity.getParamsUri().remove(next);
@@ -2138,6 +2144,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 			}
 
 			// Store Coords SP's
+			calculateHashes(coordsParams);
 			for (ResourceIndexedSearchParamCoords next : removeCommon(existingCoordsParams, coordsParams)) {
 				myEntityManager.remove(next);
 				theEntity.getParamsCoords().remove(next);
@@ -2187,6 +2194,12 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 		return theEntity;
 	}
 
+	private void calculateHashes(Collection<? extends BaseResourceIndexedSearchParam> theStringParams) {
+		for (BaseResourceIndexedSearchParam next : theStringParams) {
+			next.calculateHashes();
+		}
+	}
+
 	protected ResourceTable updateEntity(RequestDetails theRequest, IBaseResource theResource, ResourceTable
 		entity, Date theDeletedTimestampOrNull, Date theUpdateTime) {
 		return updateEntity(theRequest, theResource, entity, theDeletedTimestampOrNull, true, true, theUpdateTime, false, true);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java

@@ -155,6 +155,7 @@ public class DaoConfig {
 	private boolean myValidateSearchParameterExpressionsOnSave = true;
 	private List<Integer> mySearchPreFetchThresholds = Arrays.asList(500, 2000, -1);
 	private List<WarmCacheEntry> myWarmCacheEntries = new ArrayList<>();
+	private boolean myDisableHashBasedSearches;
 
 	/**
 	 * Constructor
@@ -1383,6 +1384,34 @@ public class DaoConfig {
 		return mySearchPreFetchThresholds;
 	}
 
+	/**
+	 * If set to <code>true</code> (default is false) the server will not use
+	 * hash based searches. These searches were introduced in HAPI FHIR 3.5.0
+	 * and are the new default way of searching. However they require a very
+	 * large data migration if an existing system has a large amount of data
+	 * so this setting can be used to use the old search mechanism while data
+	 * is migrated.
+	 *
+	 * @since 3.6.0
+	 */
+	public boolean getDisableHashBasedSearches() {
+		return myDisableHashBasedSearches;
+	}
+
+	/**
+	 * If set to <code>true</code> (default is false) the server will not use
+	 * hash based searches. These searches were introduced in HAPI FHIR 3.5.0
+	 * and are the new default way of searching. However they require a very
+	 * large data migration if an existing system has a large amount of data
+	 * so this setting can be used to use the old search mechanism while data
+	 * is migrated.
+	 *
+	 * @since 3.6.0
+	 */
+	public void setDisableHashBasedSearches(boolean theDisableHashBasedSearches) {
+		myDisableHashBasedSearches = theDisableHashBasedSearches;
+	}
+
 	public enum IndexEnabledEnum {
 		ENABLED,
 		DISABLED

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java

@@ -55,6 +55,7 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
@@ -94,6 +95,7 @@ public class SearchBuilder implements ISearchBuilder {
 	private static SearchParameterMap ourLastHandlerParamsForUnitTest;
 	private static String ourLastHandlerThreadForUnitTest;
 	private static boolean ourTrackHandlersForUnitTest;
+	private final boolean myDontUseHashesForSearch;
 	protected IResourceTagDao myResourceTagDao;
 	private IResourceSearchViewDao myResourceSearchViewDao;
 	private List<Long> myAlsoIncludePids;
@@ -130,6 +132,7 @@ public class SearchBuilder implements ISearchBuilder {
 		myEntityManager = theEntityManager;
 		myFulltextSearchSvc = theFulltextSearchSvc;
 		myCallingDao = theDao;
+		myDontUseHashesForSearch = theDao.getConfig().getDisableHashBasedSearches();
 		myResourceIndexedSearchParamUriDao = theResourceIndexedSearchParamUriDao;
 		myForcedIdDao = theForcedIdDao;
 		myTerminologySvc = theTerminologySvc;
@@ -304,6 +307,15 @@ public class SearchBuilder implements ISearchBuilder {
 	}
 
 	private void addPredicateParamMissing(String theResourceName, String theParamName, boolean theMissing) {
+//		if (myDontUseHashesForSearch) {
+//			Join<ResourceTable, SearchParamPresent> paramPresentJoin = myResourceTableRoot.join("mySearchParamPresents", JoinType.LEFT);
+//			Join<Object, Object> paramJoin = paramPresentJoin.join("mySearchParam", JoinType.LEFT);
+//
+//			myPredicates.add(myBuilder.equal(paramJoin.get("myResourceName"), theResourceName));
+//			myPredicates.add(myBuilder.equal(paramJoin.get("myParamName"), theParamName));
+//			myPredicates.add(myBuilder.equal(paramPresentJoin.get("myPresent"), !theMissing));
+//		}
+
 		Join<ResourceTable, SearchParamPresent> paramPresentJoin = myResourceTableRoot.join("mySearchParamPresents", JoinType.LEFT);
 
 		Expression<Long> hashPresence = paramPresentJoin.get("myHashPresence").as(Long.class);
@@ -841,10 +853,18 @@ public class SearchBuilder implements ISearchBuilder {
 
 				} else {
 
-					long hashUri = ResourceIndexedSearchParamUri.calculateHashUri(theResourceName, theParamName, value);
+					if (myDontUseHashesForSearch) {
-					Predicate hashPredicate = myBuilder.equal(join.get("myHashUri"), hashUri);
-					codePredicates.add(hashPredicate);
 
+						Predicate predicate = myBuilder.equal(join.get("myUri").as(String.class), value);
+						codePredicates.add(predicate);
+
+					} else {
+
+						long hashUri = ResourceIndexedSearchParamUri.calculateHashUri(theResourceName, theParamName, value);
+						Predicate hashPredicate = myBuilder.equal(join.get("myHashUri"), hashUri);
+						codePredicates.add(hashPredicate);
+
+					}
 				}
 
 			} else {
@@ -868,6 +888,13 @@ public class SearchBuilder implements ISearchBuilder {
 	}
 
 	private Predicate combineParamIndexPredicateWithParamNamePredicate(String theResourceName, String theParamName, From<?, ? extends BaseResourceIndexedSearchParam> theFrom, Predicate thePredicate) {
+		if (myDontUseHashesForSearch) {
+			Predicate resourceTypePredicate = myBuilder.equal(theFrom.get("myResourceType"), theResourceName);
+			Predicate paramNamePredicate = myBuilder.equal(theFrom.get("myParamName"), theParamName);
+			Predicate outerPredicate = myBuilder.and(resourceTypePredicate, paramNamePredicate, thePredicate);
+			return outerPredicate;
+		}
+
 		long hashIdentity = BaseResourceIndexedSearchParam.calculateHashIdentity(theResourceName, theParamName);
 		Predicate hashIdentityPredicate = myBuilder.equal(theFrom.get("myHashIdentity"), hashIdentity);
 		return myBuilder.and(hashIdentityPredicate, thePredicate);
@@ -1079,6 +1106,37 @@ public class SearchBuilder implements ISearchBuilder {
 			throw new IllegalArgumentException("Invalid quantity type: " + theParam.getClass());
 		}
 
+		if (myDontUseHashesForSearch) {
+			Predicate system = null;
+			if (!isBlank(systemValue)) {
+				system = theBuilder.equal(theFrom.get("mySystem"), systemValue);
+			}
+
+			Predicate code = null;
+			if (!isBlank(unitsValue)) {
+				code = theBuilder.equal(theFrom.get("myUnits"), unitsValue);
+			}
+
+			cmpValue = ObjectUtils.defaultIfNull(cmpValue, ParamPrefixEnum.EQUAL);
+			final Expression<BigDecimal> path = theFrom.get("myValue");
+			String invalidMessageName = "invalidQuantityPrefix";
+
+			Predicate num = createPredicateNumeric(theResourceName, null, theFrom, theBuilder, theParam, cmpValue, valueValue, path, invalidMessageName);
+
+			Predicate singleCode;
+			if (system == null && code == null) {
+				singleCode = num;
+			} else if (system == null) {
+				singleCode = theBuilder.and(code, num);
+			} else if (code == null) {
+				singleCode = theBuilder.and(system, num);
+			} else {
+				singleCode = theBuilder.and(system, code, num);
+			}
+
+			return combineParamIndexPredicateWithParamNamePredicate(theResourceName, theParamName, theFrom, singleCode);
+		}
+
 		Predicate hashPredicate;
 		if (!isBlank(systemValue) && !isBlank(unitsValue)) {
 			long hash = ResourceIndexedSearchParamQuantity.calculateHashSystemAndUnits(theResourceName, theParamName, systemValue, unitsValue);
@@ -1130,6 +1188,31 @@ public class SearchBuilder implements ISearchBuilder {
 				+ ResourceIndexedSearchParamString.MAX_LENGTH + "): " + rawSearchTerm);
 		}
 
+		if (myDontUseHashesForSearch) {
+			String likeExpression = BaseHapiFhirDao.normalizeString(rawSearchTerm);
+			if (myCallingDao.getConfig().isAllowContainsSearches()) {
+				if (theParameter instanceof StringParam) {
+					if (((StringParam) theParameter).isContains()) {
+						likeExpression = createLeftAndRightMatchLikeExpression(likeExpression);
+					} else {
+						likeExpression = createLeftMatchLikeExpression(likeExpression);
+					}
+				} else {
+					likeExpression = createLeftMatchLikeExpression(likeExpression);
+				}
+			} else {
+				likeExpression = createLeftMatchLikeExpression(likeExpression);
+			}
+
+			Predicate singleCode = theBuilder.like(theFrom.get("myValueNormalized").as(String.class), likeExpression);
+			if (theParameter instanceof StringParam && ((StringParam) theParameter).isExact()) {
+				Predicate exactCode = theBuilder.equal(theFrom.get("myValueExact"), rawSearchTerm);
+				singleCode = theBuilder.and(singleCode, exactCode);
+			}
+
+			return combineParamIndexPredicateWithParamNamePredicate(theResourceName, theParamName, theFrom, singleCode);
+		}
+
 		boolean exactMatch = theParameter instanceof StringParam && ((StringParam) theParameter).isExact();
 		if (exactMatch) {
 
@@ -1234,6 +1317,92 @@ public class SearchBuilder implements ISearchBuilder {
 			return new BooleanStaticAssertionPredicate((CriteriaBuilderImpl) theBuilder, false);
 		}
 
+		if (myDontUseHashesForSearch) {
+			ArrayList<Predicate> singleCodePredicates = new ArrayList<Predicate>();
+			if (codes != null) {
+
+				List<Predicate> orPredicates = new ArrayList<Predicate>();
+				Map<String, List<VersionIndependentConcept>> map = new HashMap<String, List<VersionIndependentConcept>>();
+				for (VersionIndependentConcept nextCode : codes) {
+					List<VersionIndependentConcept> systemCodes = map.get(nextCode.getSystem());
+					if (null == systemCodes) {
+						systemCodes = new ArrayList<>();
+						map.put(nextCode.getSystem(), systemCodes);
+					}
+					systemCodes.add(nextCode);
+				}
+				// Use "in" in case of large numbers of codes due to param modifiers
+				final Path<String> systemExpression = theFrom.get("mySystem");
+				final Path<String> valueExpression = theFrom.get("myValue");
+				for (Map.Entry<String, List<VersionIndependentConcept>> entry : map.entrySet()) {
+					CriteriaBuilder.In<String> codePredicate = theBuilder.in(valueExpression);
+					boolean haveAtLeastOneCode = false;
+					for (VersionIndependentConcept nextCode : entry.getValue()) {
+						if (isNotBlank(nextCode.getCode())) {
+							codePredicate.value(nextCode.getCode());
+							haveAtLeastOneCode = true;
+						}
+					}
+
+					if (entry.getKey() != null) {
+						Predicate systemPredicate = theBuilder.equal(systemExpression, entry.getKey());
+						if (haveAtLeastOneCode) {
+							orPredicates.add(theBuilder.and(systemPredicate, codePredicate));
+						} else {
+							orPredicates.add(systemPredicate);
+						}
+					} else {
+						orPredicates.add(codePredicate);
+					}
+				}
+
+				Predicate or = theBuilder.or(orPredicates.toArray(new Predicate[0]));
+				if (modifier == TokenParamModifier.NOT) {
+					or = theBuilder.not(or);
+				}
+				singleCodePredicates.add(or);
+
+			} else {
+
+				/*
+				 * Ok, this is a normal query
+				 */
+
+				if (StringUtils.isNotBlank(system)) {
+					if (modifier != null && modifier == TokenParamModifier.NOT) {
+						singleCodePredicates.add(theBuilder.notEqual(theFrom.get("mySystem"), system));
+					} else {
+						singleCodePredicates.add(theBuilder.equal(theFrom.get("mySystem"), system));
+					}
+				} else if (system == null) {
+					// don't check the system
+				} else {
+					// If the system is "", we only match on null systems
+					singleCodePredicates.add(theBuilder.isNull(theFrom.get("mySystem")));
+				}
+
+				if (StringUtils.isNotBlank(code)) {
+					if (modifier != null && modifier == TokenParamModifier.NOT) {
+						singleCodePredicates.add(theBuilder.notEqual(theFrom.get("myValue"), code));
+					} else {
+						singleCodePredicates.add(theBuilder.equal(theFrom.get("myValue"), code));
+					}
+				} else {
+					/*
+					 * As of HAPI FHIR 1.5, if the client searched for a token with a system but no specified value this means to
+					 * match all tokens with the given value.
+					 *
+					 * I'm not sure I agree with this, but hey.. FHIR-I voted and this was the result :)
+					 */
+					// singleCodePredicates.add(theBuilder.isNull(theFrom.get("myValue")));
+				}
+			}
+
+			Predicate singleCode = theBuilder.and(toArray(singleCodePredicates));
+			return combineParamIndexPredicateWithParamNamePredicate(theResourceName, theParamName, theFrom, singleCode);
+		}
+
+
 		/*
 		 * Note: A null system value means "match any system", but
 		 * an empty-string system value means "match values that
@@ -1607,9 +1776,14 @@ public class SearchBuilder implements ISearchBuilder {
 			if (param.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
 				thePredicates.add(join.get("mySourcePath").as(String.class).in(param.getPathsSplit()));
 			} else {
-				Long hashIdentity = BaseResourceIndexedSearchParam.calculateHashIdentity(myResourceName, theSort.getParamName());
+				if (myDontUseHashesForSearch) {
-				Predicate joinParam1 = theBuilder.equal(join.get("myHashIdentity"), hashIdentity);
+					Predicate joinParam1 = theBuilder.equal(join.get("myParamName"), theSort.getParamName());
-				thePredicates.add(joinParam1);
+					thePredicates.add(joinParam1);
+				} else {
+					Long hashIdentity = BaseResourceIndexedSearchParam.calculateHashIdentity(myResourceName, theSort.getParamName());
+					Predicate joinParam1 = theBuilder.equal(join.get("myHashIdentity"), hashIdentity);
+					thePredicates.add(joinParam1);
+				}
 			}
 		} else {
 			ourLog.debug("Reusing join for {}", theSort.getParamName());
@@ -1668,7 +1842,7 @@ public class SearchBuilder implements ISearchBuilder {
 		//-- preload all tags with tag definition if any
 		Map<Long, Collection<ResourceTag>> tagMap = getResourceTagMap(resourceSearchViewList);
 
-		Long resourceId = null;
+		Long resourceId;
 		for (ResourceSearchView next : resourceSearchViewList) {
 
 			Class<? extends IBaseResource> resourceType = context.getResourceDefinition(next.getResourceType()).getImplementingClass();
@@ -1706,7 +1880,7 @@ public class SearchBuilder implements ISearchBuilder {
 
 	private Map<Long, Collection<ResourceTag>> getResourceTagMap(Collection<ResourceSearchView> theResourceSearchViewList) {
 
-		List<Long> idList = new ArrayList<Long>(theResourceSearchViewList.size());
+		List<Long> idList = new ArrayList<>(theResourceSearchViewList.size());
 
 		//-- find all resource has tags
 		for (ResourceSearchView resource : theResourceSearchViewList) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/BaseResourceIndexedSearchParam.java

@@ -129,6 +129,8 @@ public abstract class BaseResourceIndexedSearchParam implements Serializable {
 
 	public abstract IQueryParameterType toQueryParameterType();
 
+	public abstract void calculateHashes();
+
 	public static long calculateHashIdentity(String theResourceType, String theParamName) {
 		return hash(theResourceType, theParamName);
 	}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamCoords.java

@@ -67,6 +67,7 @@ public class ResourceIndexedSearchParamCoords extends BaseResourceIndexedSearchP
 		setLongitude(theLongitude);
 	}
 
+	@Override
 	@PrePersist
 	public void calculateHashes() {
 		if (myHashIdentity == null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamDate.java

@@ -83,6 +83,7 @@ public class ResourceIndexedSearchParamDate extends BaseResourceIndexedSearchPar
 		myOriginalValue = theOriginalValue;
 	}
 
+	@Override
 	@PrePersist
 	public void calculateHashes() {
 		if (myHashIdentity == null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamNumber.java

@@ -69,6 +69,7 @@ public class ResourceIndexedSearchParamNumber extends BaseResourceIndexedSearchP
 		setValue(theValue);
 	}
 
+	@Override
 	@PrePersist
 	public void calculateHashes() {
 		if (myHashIdentity == null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamQuantity.java

@@ -95,6 +95,7 @@ public class ResourceIndexedSearchParamQuantity extends BaseResourceIndexedSearc
 		setUnits(theUnits);
 	}
 
+	@Override
 	@PrePersist
 	public void calculateHashes() {
 		if (myHashIdentity == null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamString.java

@@ -161,6 +161,7 @@ public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchP
 		myHashIdentity = theHashIdentity;
 	}
 
+	@Override
 	@PrePersist
 	public void calculateHashes() {
 		if (myHashNormalizedPrefix == null && myDaoConfig != null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamToken.java

@@ -108,6 +108,7 @@ public class ResourceIndexedSearchParamToken extends BaseResourceIndexedSearchPa
 		setValue(theValue);
 	}
 
+	@Override
 	@PrePersist
 	public void calculateHashes() {
 		if (myHashSystem == null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamUri.java

@@ -82,6 +82,7 @@ public class ResourceIndexedSearchParamUri extends BaseResourceIndexedSearchPara
 		setUri(theUri);
 	}
 
+	@Override
 	@PrePersist
 	public void calculateHashes() {
 		if (myHashUri == null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/SubscriptionTriggeringProvider.java

@@ -39,6 +39,7 @@ import ca.uhn.fhir.rest.api.server.IBundleProvider;
 import ca.uhn.fhir.rest.param.StringParam;
 import ca.uhn.fhir.rest.param.UriParam;
 import ca.uhn.fhir.rest.server.IResourceProvider;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
 import ca.uhn.fhir.util.ParametersUtil;
@@ -307,7 +308,7 @@ public class SubscriptionTriggeringProvider implements IResourceProvider, Applic
 			ourLog.info("Triggering job[{}] search {} requesting resources {} - {}", theJobDetails.getJobId(), theJobDetails.getCurrentSearchUuid(), fromIndex, toIndex);
 			List<Long> resourceIds = mySearchCoordinatorSvc.getResources(theJobDetails.getCurrentSearchUuid(), fromIndex, toIndex);
 
-			ourLog.info("Triggering job[{}] delivering {} resources", theJobDetails.getJobId(), theJobDetails.getCurrentSearchUuid(), fromIndex, toIndex);
+			ourLog.info("Triggering job[{}] delivering {} resources", theJobDetails.getJobId(), resourceIds.size());
 			int highestIndexSubmitted = theJobDetails.getCurrentSearchLastUploadedIndex();
 
 			for (Long next : resourceIds) {
@@ -374,9 +375,22 @@ public class SubscriptionTriggeringProvider implements IResourceProvider, Applic
 		msg.setNewPayload(myFhirContext, theResourceToTrigger);
 
 		return myExecutorService.submit(()->{
-			for (BaseSubscriptionInterceptor<?> next : mySubscriptionInterceptorList) {
+			for (int i = 0; ; i++) {
-				next.submitResourceModified(msg);
+				try {
+					for (BaseSubscriptionInterceptor<?> next : mySubscriptionInterceptorList) {
+						next.submitResourceModified(msg);
+					}
+					break;
+				} catch (Exception e) {
+					if (i >= 3) {
+						throw new InternalErrorException(e);
+					}
+
+					ourLog.warn("Exception while retriggering subscriptions (going to sleep and retry): {}", e.toString());
+					Thread.sleep(1000);
+				}
 			}
+
 			return null;
 		});
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java

@@ -74,7 +74,6 @@ public class StaleSearchDeletingSvcImpl implements IStaleSearchDeletingSvc {
 
 	private void deleteSearch(final Long theSearchPid) {
 		mySearchDao.findById(theSearchPid).ifPresent(searchToDelete -> {
-			ourLog.info("Deleting search {}/{} - Created[{}] -- Last returned[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()), new InstantType(searchToDelete.getSearchLastReturned()));
 			mySearchIncludeDao.deleteForSearch(searchToDelete.getId());
 
 			/*
@@ -93,7 +92,10 @@ public class StaleSearchDeletingSvcImpl implements IStaleSearchDeletingSvc {
 
 			// Only delete if we don't have results left in this search
 			if (resultPids.getNumberOfElements() < max) {
+				ourLog.info("Deleting search {}/{} - Created[{}] -- Last returned[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()), new InstantType(searchToDelete.getSearchLastReturned()));
 				mySearchDao.deleteByPid(searchToDelete.getId());
+			} else {
+				ourLog.info("Purged {} search results for deleted search {}/{}", resultPids.getSize(), searchToDelete.getId(), searchToDelete.getUuid());
 			}
 		});
 	}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java

@@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.config;
 import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor;
 import ca.uhn.fhir.validation.ResultSeverityEnum;
+import net.ttddyy.dsproxy.listener.SingleQueryCountHolder;
 import net.ttddyy.dsproxy.listener.ThreadQueryCountHolder;
 import net.ttddyy.dsproxy.listener.logging.SLF4JLogLevel;
 import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder;
@@ -107,12 +108,18 @@ public class TestR4Config extends BaseJavaConfigR4 {
 			.create(retVal)
 			.logQueryBySlf4j(SLF4JLogLevel.INFO, "SQL")
 			.logSlowQueryBySlf4j(10, TimeUnit.SECONDS)
-			.countQuery(new ThreadQueryCountHolder())
+//			.countQuery(new ThreadQueryCountHolder())
+			.countQuery(singleQueryCountHolder())
 			.build();
 
 		return dataSource;
 	}
 
+	@Bean
+	public SingleQueryCountHolder singleQueryCountHolder() {
+		return new SingleQueryCountHolder();
+	}
+
 	@Override
 	@Bean()
 	public LocalContainerEntityManagerFactoryBean entityManagerFactory() {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4CreateTest.java

@@ -4,7 +4,6 @@ import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.dao.SearchParameterMap;
 import ca.uhn.fhir.rest.param.StringParam;
 import ca.uhn.fhir.util.TestUtil;
-import net.ttddyy.dsproxy.listener.ThreadQueryCountHolder;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r4.model.Bundle;
 import org.hl7.fhir.r4.model.IdType;
@@ -18,10 +17,9 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
-import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.*;
-import static org.hamcrest.Matchers.empty;
+import static org.junit.Assert.assertEquals;
-import static org.hamcrest.Matchers.matchesPattern;
+import static org.junit.Assert.assertThat;
-import static org.junit.Assert.*;
 
 public class FhirResourceDaoR4CreateTest extends BaseJpaR4Test {
 	private static final Logger ourLog = LoggerFactory.getLogger(FhirResourceDaoR4CreateTest.class);
@@ -37,22 +35,22 @@ public class FhirResourceDaoR4CreateTest extends BaseJpaR4Test {
 		Patient p = myFhirCtx.newXmlParser().parseResource(Patient.class, input);
 		String id = myPatientDao.create(p).getId().toUnqualifiedVersionless().getValue();
 
-		SearchParameterMap map= new SearchParameterMap();
+		SearchParameterMap map = new SearchParameterMap();
 		map.setLoadSynchronous(true);
 		map.add(Patient.SP_FAMILY, new StringParam("김"));
 		assertThat(toUnqualifiedVersionlessIdValues(myPatientDao.search(map)), contains(id));
 
-		map= new SearchParameterMap();
+		map = new SearchParameterMap();
 		map.setLoadSynchronous(true);
 		map.add(Patient.SP_GIVEN, new StringParam("준"));
 		assertThat(toUnqualifiedVersionlessIdValues(myPatientDao.search(map)), contains(id));
 
-		map= new SearchParameterMap();
+		map = new SearchParameterMap();
 		map.setLoadSynchronous(true);
 		map.add(Patient.SP_GIVEN, new StringParam("준수"));
 		assertThat(toUnqualifiedVersionlessIdValues(myPatientDao.search(map)), contains(id));
 
-		map= new SearchParameterMap();
+		map = new SearchParameterMap();
 		map.setLoadSynchronous(true);
 		map.add(Patient.SP_GIVEN, new StringParam("수")); // rightmost character only
 		assertThat(toUnqualifiedVersionlessIdValues(myPatientDao.search(map)), empty());
@@ -60,7 +58,7 @@ public class FhirResourceDaoR4CreateTest extends BaseJpaR4Test {
 	}
 
 	@Test
-	public void testCreateWithUuidResourceStrategy() throws Exception {
+	public void testCreateWithUuidResourceStrategy() {
 		myDaoConfig.setResourceServerIdStrategy(DaoConfig.IdStrategyEnum.UUID);
 
 		Patient p = new Patient();
@@ -110,26 +108,6 @@ public class FhirResourceDaoR4CreateTest extends BaseJpaR4Test {
 		assertThat(output.getEntry().get(1).getResponse().getLocation(), matchesPattern("Patient/[a-z0-9]{8}-.*"));
 
 
-	}
-
-	@Test
-	public void testWritesPerformMinimalSqlStatements() {
-		Patient p = new Patient();
-		p.addIdentifier().setSystem("sys1").setValue("val1");
-		p.addIdentifier().setSystem("sys2").setValue("val2");
-
-		ourLog.info("** About to perform write");
-		new ThreadQueryCountHolder().getOrCreateQueryCount("").setInsert(0);
-		new ThreadQueryCountHolder().getOrCreateQueryCount("").setUpdate(0);
-
-		myPatientDao.create(p);
-
-		ourLog.info("** Done performing write");
-
-		ourLog.info("Inserts: {}", new ThreadQueryCountHolder().getOrCreateQueryCount("").getInsert());
-		ourLog.info("Updates: {}", new ThreadQueryCountHolder().getOrCreateQueryCount("").getUpdate());
-
-
 	}
 
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java

@@ -1,14 +1,19 @@
 package ca.uhn.fhir.jpa.dao.r4;
 
 import ca.uhn.fhir.jpa.dao.DaoConfig;
+import ca.uhn.fhir.jpa.dao.SearchParameterMap;
+import ca.uhn.fhir.rest.api.server.IBundleProvider;
 import ca.uhn.fhir.util.TestUtil;
+import net.ttddyy.dsproxy.QueryCount;
 import net.ttddyy.dsproxy.QueryCountHolder;
+import net.ttddyy.dsproxy.listener.SingleQueryCountHolder;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r4.model.DateTimeType;
 import org.hl7.fhir.r4.model.Patient;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.context.TestPropertySource;
 
 import static org.junit.Assert.assertEquals;
@@ -18,6 +23,8 @@ import static org.junit.Assert.assertEquals;
 })
 public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4QueryCountTest.class);
+	@Autowired
+	private SingleQueryCountHolder myCountHolder;
 
 	@After
 	public void afterResetDao() {
@@ -25,22 +32,87 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test {
 		myDaoConfig.setIndexMissingFields(new DaoConfig().getIndexMissingFields());
 	}
 
+	@Test
+	public void testWritesPerformMinimalSqlStatements() {
+		Patient p = new Patient();
+		p.addIdentifier().setSystem("sys1").setValue("val1");
+		p.addIdentifier().setSystem("sys2").setValue("val2");
+
+		ourLog.info("** About to perform write");
+		myCountHolder.clear();
+
+		IIdType id = myPatientDao.create(p).getId().toUnqualifiedVersionless();
+
+		ourLog.info("** Done performing write");
+
+		assertEquals(6, getQueryCount().getInsert());
+		assertEquals(0, getQueryCount().getUpdate());
+
+		/*
+		 * Not update the value
+		 */
+
+		p = new Patient();
+		p.setId(id);
+		p.addIdentifier().setSystem("sys1").setValue("val3");
+		p.addIdentifier().setSystem("sys2").setValue("val4");
+
+		ourLog.info("** About to perform write 2");
+		myCountHolder.clear();
+
+		myPatientDao.update(p).getId().toUnqualifiedVersionless();
+
+		ourLog.info("** Done performing write 2");
+
+		assertEquals(2, getQueryCount().getInsert());
+		assertEquals(1, getQueryCount().getUpdate());
+		assertEquals(1, getQueryCount().getDelete());
+	}
+
+	@Test
+	public void testSearch() {
+
+		for (int i = 0; i < 20; i++) {
+			Patient p = new Patient();
+			p.addIdentifier().setSystem("sys1").setValue("val" + i);
+			myPatientDao.create(p);
+		}
+
+		myCountHolder.clear();
+
+		ourLog.info("** About to perform search");
+		IBundleProvider search = myPatientDao.search(new SearchParameterMap());
+		ourLog.info("** About to retrieve resources");
+		search.getResources(0, 20);
+		ourLog.info("** Done retrieving resources");
+
+		assertEquals(4, getQueryCount().getSelect());
+		assertEquals(2, getQueryCount().getInsert());
+		assertEquals(1, getQueryCount().getUpdate());
+		assertEquals(0, getQueryCount().getDelete());
+
+	}
+
+	private QueryCount getQueryCount() {
+		return myCountHolder.getQueryCountMap().get("");
+	}
+
 	@Test
 	public void testCreateClientAssignedId() {
 		myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.DISABLED);
 
-		QueryCountHolder.clear();
+		myCountHolder.clear();
 		ourLog.info("** Starting Update Non-Existing resource with client assigned ID");
 		Patient p = new Patient();
 		p.setId("A");
 		p.getPhotoFirstRep().setCreationElement(new DateTimeType("2011")); // non-indexed field
 		myPatientDao.update(p).getId().toUnqualifiedVersionless();
 
-		assertEquals(1, QueryCountHolder.getGrandTotal().getSelect());
+		assertEquals(1, getQueryCount().getSelect());
-		assertEquals(4, QueryCountHolder.getGrandTotal().getInsert());
+		assertEquals(4, getQueryCount().getInsert());
-		assertEquals(0, QueryCountHolder.getGrandTotal().getDelete());
+		assertEquals(0, getQueryCount().getDelete());
 		// Because of the forced ID's bidirectional link HFJ_RESOURCE <-> HFJ_FORCED_ID
-		assertEquals(1, QueryCountHolder.getGrandTotal().getUpdate());
+		assertEquals(1, getQueryCount().getUpdate());
 		runInTransaction(() -> {
 			assertEquals(1, myResourceTableDao.count());
 			assertEquals(1, myResourceHistoryTableDao.count());
@@ -50,17 +122,17 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test {
 
 		// Ok how about an update
 
-		QueryCountHolder.clear();
+		myCountHolder.clear();
 		ourLog.info("** Starting Update Existing resource with client assigned ID");
 		p = new Patient();
 		p.setId("A");
 		p.getPhotoFirstRep().setCreationElement(new DateTimeType("2012")); // non-indexed field
 		myPatientDao.update(p).getId().toUnqualifiedVersionless();
 
-		assertEquals(5, QueryCountHolder.getGrandTotal().getSelect());
+		assertEquals(5, getQueryCount().getSelect());
-		assertEquals(1, QueryCountHolder.getGrandTotal().getInsert());
+		assertEquals(1, getQueryCount().getInsert());
-		assertEquals(0, QueryCountHolder.getGrandTotal().getDelete());
+		assertEquals(0, getQueryCount().getDelete());
-		assertEquals(1, QueryCountHolder.getGrandTotal().getUpdate());
+		assertEquals(1, getQueryCount().getUpdate());
 		runInTransaction(() -> {
 			assertEquals(1, myResourceTableDao.count());
 			assertEquals(2, myResourceHistoryTableDao.count());
@@ -75,24 +147,24 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test {
 	public void testOneRowPerUpdate() {
 		myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.DISABLED);
 
-		QueryCountHolder.clear();
+		myCountHolder.clear();
 		Patient p = new Patient();
 		p.getPhotoFirstRep().setCreationElement(new DateTimeType("2011")); // non-indexed field
 		IIdType id = myPatientDao.create(p).getId().toUnqualifiedVersionless();
 
-		assertEquals(3, QueryCountHolder.getGrandTotal().getInsert());
+		assertEquals(3, getQueryCount().getInsert());
 		runInTransaction(() -> {
 			assertEquals(1, myResourceTableDao.count());
 			assertEquals(1, myResourceHistoryTableDao.count());
 		});
 
-		QueryCountHolder.clear();
+		myCountHolder.clear();
 		p = new Patient();
 		p.setId(id);
 		p.getPhotoFirstRep().setCreationElement(new DateTimeType("2012")); // non-indexed field
 		myPatientDao.update(p).getId().toUnqualifiedVersionless();
 
-		assertEquals(1, QueryCountHolder.getGrandTotal().getInsert());
+		assertEquals(1, getQueryCount().getInsert());
 		runInTransaction(() -> {
 			assertEquals(1, myResourceTableDao.count());
 			assertEquals(2, myResourceHistoryTableDao.count());
@@ -101,6 +173,34 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test {
 	}
 
 
+	@Test
+	public void testUpdateReusesIndexes() {
+		myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.DISABLED);
+
+		myCountHolder.clear();
+
+		Patient pt = new Patient();
+		pt.setActive(true);
+		pt.addName().setFamily("FAMILY1").addGiven("GIVEN1A").addGiven("GIVEN1B");
+		IIdType id = myPatientDao.create(pt).getId().toUnqualifiedVersionless();
+
+		ourLog.info("Now have {} deleted", getQueryCount().getDelete());
+		ourLog.info("Now have {} inserts", getQueryCount().getInsert());
+		myCountHolder.clear();
+
+		ourLog.info("** About to update");
+
+		pt.setId(id);
+		pt.getNameFirstRep().addGiven("GIVEN1C");
+		myPatientDao.update(pt);
+
+		ourLog.info("Now have {} deleted", getQueryCount().getDelete());
+		ourLog.info("Now have {} inserts", getQueryCount().getInsert());
+		assertEquals(0, getQueryCount().getDelete());
+		assertEquals(2, getQueryCount().getInsert());
+	}
+
+
 	@AfterClass
 	public static void afterClassClearContext() {
 		TestUtil.clearAllStaticFieldsForUnitTest();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4UpdateTest.java

@@ -668,33 +668,6 @@ public class FhirResourceDaoR4UpdateTest extends BaseJpaR4Test {
 
 	}
 
-	@Test
-	public void testUpdateReusesIndexes() {
-		myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.DISABLED);
-
-		QueryCountHolder.clear();
-
-		Patient pt = new Patient();
-		pt.setActive(true);
-		pt.addName().setFamily("FAMILY1").addGiven("GIVEN1A").addGiven("GIVEN1B");
-		IIdType id = myPatientDao.create(pt).getId().toUnqualifiedVersionless();
-
-		ourLog.info("Now have {} deleted", QueryCountHolder.getGrandTotal().getDelete());
-		ourLog.info("Now have {} inserts", QueryCountHolder.getGrandTotal().getInsert());
-		QueryCountHolder.clear();
-
-		ourLog.info("** About to update");
-
-		pt.setId(id);
-		pt.getNameFirstRep().addGiven("GIVEN1C");
-		myPatientDao.update(pt);
-
-		ourLog.info("Now have {} deleted", QueryCountHolder.getGrandTotal().getDelete());
-		ourLog.info("Now have {} inserts", QueryCountHolder.getGrandTotal().getInsert());
-		assertEquals(0, QueryCountHolder.getGrandTotal().getDelete());
-		assertEquals(4, QueryCountHolder.getGrandTotal().getInsert());
-	}
-
 	@Test
 	public void testUpdateUnknownNumericIdFails() {
 		Patient p = new Patient();

src/changes/changes.xml

@@ -116,6 +116,11 @@
 				<![CDATA[<a href="https://github.com/hapifhir/hapi-fhir-jpaserver-starter/blob/master/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerConfig.java#L62">the example project</a>]]>
 				if they are not already.
 			</action>
+			<action type="fix">
+				When updating resources in the JPA server, a bug caused index table entries to be refreshed
+				sometimes even though the index value hadn't changed. This issue did not cause incorrect search
+				results but had an effect on write performance. This has been corrected.
+			</action>
 		</release>
 
 		<release version="3.5.0" date="2018-09-17">