Custom structs in progress

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/FhirTerser.java

@@ -22,6 +22,7 @@ import static org.apache.commons.lang3.StringUtils.defaultString;
  * #L%
  */
 import java.util.*;
+import java.util.function.Predicate;
 
 import org.apache.commons.lang3.Validate;
 import org.hl7.fhir.instance.model.api.*;
@@ -212,10 +213,43 @@ public class FhirTerser {
 	@SuppressWarnings("unchecked")
 	private <T> List<T> getValues(BaseRuntimeElementCompositeDefinition<?> theCurrentDef, Object theCurrentObj, List<String> theSubList, Class<T> theWantedClass) {
 		String name = theSubList.get(0);
-				
+		List<T> retVal = new ArrayList<>();
+
+		if (name.startsWith("extension('")) {
+			String extensionUrl = name.substring("extension('".length());
+			int endIndex = extensionUrl.indexOf('\'');
+			if (endIndex != -1) {
+				extensionUrl = extensionUrl.substring(0, endIndex);
+			}
+
+			List<ExtensionDt> extensions= Collections.emptyList();
+			if (theCurrentObj instanceof ISupportsUndeclaredExtensions) {
+				extensions = ((ISupportsUndeclaredExtensions) theCurrentObj).getUndeclaredExtensionsByUrl(extensionUrl);
+			} else if (theCurrentObj instanceof IBaseExtension) {
+				extensions = ((IBaseExtension)theCurrentObj).getExtension();
+			}
+
+			for (ExtensionDt next : extensions) {
+				if (theWantedClass.isAssignableFrom(next.getClass())) {
+					retVal.add((T) next);
+				}
+			}
+
+			if (theSubList.size() > 1) {
+				List<T> values = retVal;
+				retVal = new ArrayList<>();
+				for (T nextElement : values) {
+					BaseRuntimeElementCompositeDefinition<?> nextChildDef = (BaseRuntimeElementCompositeDefinition<?>) myContext.getElementDefinition((Class<? extends IBase>) nextElement.getClass());
+					List<T> foundValues = getValues(nextChildDef, nextElement, theSubList.subList(1, theSubList.size()), theWantedClass);
+					retVal.addAll(foundValues);
+				}
+			}
+
+			return retVal;
+		}
+
 		BaseRuntimeChildDefinition nextDef = theCurrentDef.getChildByNameOrThrowDataFormatException(name);
 		List<? extends IBase> values = nextDef.getAccessor().getValues(theCurrentObj);
-		List<T> retVal = new ArrayList<T>();
 
 		if (theSubList.size() == 1) {
 			if (nextDef instanceof RuntimeChildChoiceDefinition) {
@@ -268,7 +302,25 @@ public class FhirTerser {
 	}
 
 	private List<String> parsePath(BaseRuntimeElementCompositeDefinition<?> theElementDef, String thePath) {
-		List<String> parts = Arrays.asList(thePath.split("\\."));
+		List<String> parts = new ArrayList<>();
+
+		int currentStart = 0;
+		boolean inSingleQuote = false;
+		for (int i = 0; i < thePath.length(); i++) {
+			switch (thePath.charAt(i)) {
+				case '\'':
+					inSingleQuote = !inSingleQuote;
+					break;
+				case '.':
+					if (!inSingleQuote) {
+						parts.add(thePath.substring(currentStart, i));
+						currentStart = i + 1;
+					}
+					break;
+			}
+		}
+
+		parts.add(thePath.substring(currentStart));
 
 		if (theElementDef instanceof RuntimeResourceDefinition) {
 			if (parts.size() > 0 && parts.get(0).equals(theElementDef.getName())) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java

@@ -56,8 +56,12 @@ import org.hl7.fhir.instance.model.api.*;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Required;
 import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionStatus;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
 
 import javax.annotation.PostConstruct;
 import javax.persistence.NoResultException;
@@ -89,6 +93,8 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 	private Class<T> myResourceType;
 	private String mySecondaryPrimaryKeyParamName;
 
+	@Autowired
+	private ISearchParamRegistry mySearchParamRegistry;
 
 	@Override
 	public void addTag(IIdType theId, TagTypeEnum theTagType, String theScheme, String theTerm, String theLabel) {
@@ -371,8 +377,8 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 		updateEntity(theResource, entity, null, thePerformIndexing, thePerformIndexing, theUpdateTime, false, thePerformIndexing);
 		theResource.setId(entity.getIdDt());
 
-		
+
-		/* 
+		/*
 		 * If we aren't indexing (meaning we're probably executing a sub-operation within a transaction),
 		 * we'll manually increase the version. This is important because we want the updated version number
 		 * to be reflected in the resource shared with interceptors
@@ -553,6 +559,26 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 		return false;
 	}
 
+	protected void markResourcesMatchingExpressionAsNeedingReindexing(String theExpression) {
+		if (isNotBlank(theExpression)) {
+			final String resourceType = theExpression.substring(0, theExpression.indexOf('.'));
+			ourLog.info("Marking all resources of type {} for reindexing due to updated search parameter with path: {}", resourceType, theExpression);
+
+			TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager);
+			txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
+			int updatedCount = txTemplate.execute(new TransactionCallback<Integer>() {
+				@Override
+				public Integer doInTransaction(TransactionStatus theStatus) {
+					return myResourceTableDao.markResourcesOfTypeAsRequiringReindexing(resourceType);
+				}
+			});
+
+			ourLog.info("Marked {} resources for reindexing", updatedCount);
+		}
+
+		mySearchParamRegistry.forceRefresh();
+	}
+
 	@Override
 	public <MT extends IBaseMetaType> MT metaAddOperation(IIdType theResourceId, MT theMetaAdd, RequestDetails theRequestDetails) {
 		// Notify interceptors

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseSearchParamExtractor.java

@@ -27,6 +27,8 @@ import java.util.List;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.ObjectUtils;
+import org.hl7.fhir.instance.model.api.IBaseDatatype;
+import org.hl7.fhir.instance.model.api.IBaseExtension;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -80,7 +82,17 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
 		for (String nextPath : nextPathsSplit) {
 			String nextPathTrimmed = nextPath.trim();
 			try {
-				values.addAll(t.getValues(theResource, nextPathTrimmed));
+				List<Object> allValues = t.getValues(theResource, nextPathTrimmed);
+				for (Object next : allValues) {
+					if (next instanceof IBaseExtension) {
+						IBaseDatatype value = ((IBaseExtension) next).getValue();
+						if (value != null) {
+							values.add(value);
+						}
+					} else {
+						values.add(next);
+					}
+				}
 			} catch (Exception e) {
 				RuntimeResourceDefinition def = myContext.getResourceDefinition(theResource);
 				ourLog.warn("Failed to index values from path[{}] in resource type[{}]: {}", new Object[] { nextPathTrimmed, def.getName(), e.toString(), e } );

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FhirResourceDaoSearchParameterDstu2.java

@@ -20,19 +20,34 @@ package ca.uhn.fhir.jpa.dao;
  * #L%
  */
 
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.dao.r4.FhirResourceDaoSearchParameterR4;
+import ca.uhn.fhir.jpa.entity.ResourceTable;
+import ca.uhn.fhir.model.dstu2.composite.MetaDt;
+import ca.uhn.fhir.model.dstu2.resource.Bundle;
+import ca.uhn.fhir.model.dstu2.resource.SearchParameter;
+import ca.uhn.fhir.model.dstu2.valueset.ResourceTypeEnum;
+import ca.uhn.fhir.model.dstu2.valueset.SearchParamTypeEnum;
+import ca.uhn.fhir.model.primitive.BoundCodeDt;
+import ca.uhn.fhir.model.primitive.CodeDt;
 import org.apache.commons.lang3.time.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Scheduled;
 
-import ca.uhn.fhir.model.dstu2.composite.MetaDt;
+import java.util.ArrayList;
-import ca.uhn.fhir.model.dstu2.resource.Bundle;
+import java.util.Arrays;
-import ca.uhn.fhir.model.dstu2.resource.SearchParameter;
+import java.util.Collections;
+import java.util.List;
 
-public class FhirResourceDaoSearchParameterDstu2 extends FhirResourceDaoDstu2<SearchParameter>implements IFhirResourceDaoSearchParameter<SearchParameter> {
+public class FhirResourceDaoSearchParameterDstu2 extends FhirResourceDaoDstu2<SearchParameter> implements IFhirResourceDaoSearchParameter<SearchParameter> {
 
 	@Autowired
 	private IFhirSystemDao<Bundle, MetaDt> mySystemDao;
-	
+
+	protected void markAffectedResources(SearchParameter theResource) {
+		markResourcesMatchingExpressionAsNeedingReindexing(theResource != null ? theResource.getXpath() : null);
+	}
+
 	/**
 	 * This method is called once per minute to perform any required re-indexing. During most passes this will
 	 * just check and find that there are no resources requiring re-indexing. In that case the method just returns
@@ -40,7 +55,7 @@ public class FhirResourceDaoSearchParameterDstu2 extends FhirResourceDaoDstu2<Se
 	 * reindexing and then return.
 	 */
 	@Override
-	@Scheduled(fixedDelay=DateUtils.MILLIS_PER_MINUTE)
+	@Scheduled(fixedDelay = DateUtils.MILLIS_PER_MINUTE)
 	public void performReindexingPass() {
 		if (getConfig().isSchedulingDisabled()) {
 			return;
@@ -55,7 +70,42 @@ public class FhirResourceDaoSearchParameterDstu2 extends FhirResourceDaoDstu2<Se
 				break;
 			}
 		}
-		
+
 	}
-	
+
+	@Override
+	protected void postPersist(ResourceTable theEntity, SearchParameter theResource) {
+		super.postPersist(theEntity, theResource);
+		markAffectedResources(theResource);
+	}
+
+	@Override
+	protected void postUpdate(ResourceTable theEntity, SearchParameter theResource) {
+		super.postUpdate(theEntity, theResource);
+		markAffectedResources(theResource);
+	}
+
+	@Override
+	protected void preDelete(SearchParameter theResourceToDelete, ResourceTable theEntityToDelete) {
+		super.preDelete(theResourceToDelete, theEntityToDelete);
+		markAffectedResources(theResourceToDelete);
+	}
+
+	@Override
+	protected void validateResourceForStorage(SearchParameter theResource, ResourceTable theEntityToSave) {
+		super.validateResourceForStorage(theResource, theEntityToSave);
+
+		Enum<?> status = theResource.getStatusElement().getValueAsEnum();
+		List<BoundCodeDt<ResourceTypeEnum>> base = Collections.emptyList();
+		if (theResource.getBase() != null) {
+			base = Arrays.asList(theResource.getBaseElement());
+		}
+		String expression = theResource.getXpath();
+		FhirContext context = getContext();
+		SearchParamTypeEnum type = theResource.getTypeElement().getValueAsEnum();
+
+		FhirResourceDaoSearchParameterR4.validateSearchParam(type, status, base, expression, context);
+	}
+
+
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchParamExtractorDstu2.java

@@ -227,6 +227,15 @@ public class SearchParamExtractorDstu2 extends BaseSearchParamExtractor implemen
 					ResourceIndexedSearchParamNumber nextEntity = new ResourceIndexedSearchParamNumber(resourceName, new BigDecimal(nextValue.getValue()));
 					nextEntity.setResource(theEntity);
 					retVal.add(nextEntity);
+				} else if (nextObject instanceof DecimalDt) {
+					DecimalDt nextValue = (DecimalDt) nextObject;
+					if (nextValue.getValue() == null) {
+						continue;
+					}
+
+					ResourceIndexedSearchParamNumber nextEntity = new ResourceIndexedSearchParamNumber(resourceName, nextValue.getValue());
+					nextEntity.setResource(theEntity);
+					retVal.add(nextEntity);
 				} else {
 					if (!multiType) {
 						throw new ConfigurationException("Search param " + resourceName + " is of unexpected datatype: " + nextObject.getClass());

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchParamRegistryDstu2.java

@@ -20,9 +20,234 @@ package ca.uhn.fhir.jpa.dao;
  * #L%
  */
 
+import ca.uhn.fhir.context.RuntimeSearchParam;
+import ca.uhn.fhir.jpa.dao.dstu3.SearchParamRegistryDstu3;
+import ca.uhn.fhir.jpa.search.JpaRuntimeSearchParam;
+import ca.uhn.fhir.jpa.util.JpaConstants;
+import ca.uhn.fhir.jpa.util.StopWatch;
+import ca.uhn.fhir.model.api.ExtensionDt;
+import ca.uhn.fhir.model.dstu2.resource.SearchParameter;
+import ca.uhn.fhir.model.primitive.CodeDt;
+import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
+import ca.uhn.fhir.rest.api.server.IBundleProvider;
+import org.apache.commons.lang3.time.DateUtils;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.instance.model.api.IPrimitiveType;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.*;
+
+import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
 public class SearchParamRegistryDstu2 extends BaseSearchParamRegistry {
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchParamRegistryDstu3.class);
+	public static final int MAX_MANAGED_PARAM_COUNT = 10000;
+
+	private volatile Map<String, Map<String, RuntimeSearchParam>> myActiveSearchParams;
+
+	@Autowired
+	private DaoConfig myDaoConfig;
+
+	private volatile long myLastRefresh;
+
+	@Autowired
+	private IFhirResourceDao<SearchParameter> mySpDao;
+
 	@Override
-	protected void refreshCacheIfNecessary() {
+	public void forceRefresh() {
-		// nothing yet
+		synchronized (this) {
+			myLastRefresh = 0;
+		}
 	}
+
+	@Override
+	public Map<String, Map<String, RuntimeSearchParam>> getActiveSearchParams() {
+		refreshCacheIfNecessary();
+		return myActiveSearchParams;
+	}
+
+	@Override
+	public Map<String, RuntimeSearchParam> getActiveSearchParams(String theResourceName) {
+		refreshCacheIfNecessary();
+		return myActiveSearchParams.get(theResourceName);
+	}
+
+	private Map<String, RuntimeSearchParam> getSearchParamMap(Map<String, Map<String, RuntimeSearchParam>> searchParams, String theResourceName) {
+		Map<String, RuntimeSearchParam> retVal = searchParams.get(theResourceName);
+		if (retVal == null) {
+			retVal = new HashMap<>();
+			searchParams.put(theResourceName, retVal);
+		}
+		return retVal;
+	}
+
+	protected void refreshCacheIfNecessary() {
+		long refreshInterval = 60 * DateUtils.MILLIS_PER_MINUTE;
+		if (System.currentTimeMillis() - refreshInterval > myLastRefresh) {
+			synchronized (this) {
+				if (System.currentTimeMillis() - refreshInterval > myLastRefresh) {
+					StopWatch sw = new StopWatch();
+
+					Map<String, Map<String, RuntimeSearchParam>> searchParams = new HashMap<>();
+					for (Map.Entry<String, Map<String, RuntimeSearchParam>> nextBuiltInEntry : getBuiltInSearchParams().entrySet()) {
+						for (RuntimeSearchParam nextParam : nextBuiltInEntry.getValue().values()) {
+							String nextResourceName = nextBuiltInEntry.getKey();
+							getSearchParamMap(searchParams, nextResourceName).put(nextParam.getName(), nextParam);
+						}
+					}
+
+					SearchParameterMap params = new SearchParameterMap();
+					params.setLoadSynchronousUpTo(MAX_MANAGED_PARAM_COUNT);
+
+					IBundleProvider allSearchParamsBp = mySpDao.search(params);
+					int size = allSearchParamsBp.size();
+
+					// Just in case..
+					if (size > MAX_MANAGED_PARAM_COUNT) {
+						ourLog.warn("Unable to support >" + MAX_MANAGED_PARAM_COUNT + " search params!");
+						size = MAX_MANAGED_PARAM_COUNT;
+					}
+
+					List<IBaseResource> allSearchParams = allSearchParamsBp.getResources(0, size);
+					for (IBaseResource nextResource : allSearchParams) {
+						SearchParameter nextSp = (SearchParameter) nextResource;
+						JpaRuntimeSearchParam runtimeSp = toRuntimeSp(nextSp);
+						if (runtimeSp == null) {
+							continue;
+						}
+
+						CodeDt nextBaseName = nextSp.getBaseElement();
+							String resourceType = nextBaseName.getValue();
+							if (isBlank(resourceType)) {
+								continue;
+							}
+
+							Map<String, RuntimeSearchParam> searchParamMap = getSearchParamMap(searchParams, resourceType);
+							String name = runtimeSp.getName();
+							if (myDaoConfig.isDefaultSearchParamsCanBeOverridden() || !searchParamMap.containsKey(name)) {
+								searchParamMap.put(name, runtimeSp);
+							}
+
+					}
+
+					Map<String, Map<String, RuntimeSearchParam>> activeSearchParams = new HashMap<>();
+					for (Map.Entry<String, Map<String, RuntimeSearchParam>> nextEntry : searchParams.entrySet()) {
+						for (RuntimeSearchParam nextSp : nextEntry.getValue().values()) {
+							String nextName = nextSp.getName();
+							if (nextSp.getStatus() != RuntimeSearchParam.RuntimeSearchParamStatusEnum.ACTIVE) {
+								nextSp = null;
+							}
+
+							if (!activeSearchParams.containsKey(nextEntry.getKey())) {
+								activeSearchParams.put(nextEntry.getKey(), new HashMap<String, RuntimeSearchParam>());
+							}
+							if (activeSearchParams.containsKey(nextEntry.getKey())) {
+								ourLog.debug("Replacing existing/built in search param {}:{} with new one", nextEntry.getKey(), nextName);
+							}
+
+							if (nextSp != null) {
+								activeSearchParams.get(nextEntry.getKey()).put(nextName, nextSp);
+							} else {
+								activeSearchParams.get(nextEntry.getKey()).remove(nextName);
+							}
+						}
+					}
+
+					myActiveSearchParams = activeSearchParams;
+
+					super.populateActiveSearchParams(activeSearchParams);
+
+					myLastRefresh = System.currentTimeMillis();
+					ourLog.info("Refreshed search parameter cache in {}ms", sw.getMillis());
+				}
+			}
+		}
+	}
+
+	private JpaRuntimeSearchParam toRuntimeSp(SearchParameter theNextSp) {
+		String name = theNextSp.getCode();
+		String description = theNextSp.getDescription();
+		String path = theNextSp.getXpath();
+		RestSearchParameterTypeEnum paramType = null;
+		RuntimeSearchParam.RuntimeSearchParamStatusEnum status = null;
+		switch (theNextSp.getTypeElement().getValueAsEnum()) {
+			case COMPOSITE:
+				paramType = RestSearchParameterTypeEnum.COMPOSITE;
+				break;
+			case DATE_DATETIME:
+				paramType = RestSearchParameterTypeEnum.DATE;
+				break;
+			case NUMBER:
+				paramType = RestSearchParameterTypeEnum.NUMBER;
+				break;
+			case QUANTITY:
+				paramType = RestSearchParameterTypeEnum.QUANTITY;
+				break;
+			case REFERENCE:
+				paramType = RestSearchParameterTypeEnum.REFERENCE;
+				break;
+			case STRING:
+				paramType = RestSearchParameterTypeEnum.STRING;
+				break;
+			case TOKEN:
+				paramType = RestSearchParameterTypeEnum.TOKEN;
+				break;
+			case URI:
+				paramType = RestSearchParameterTypeEnum.URI;
+				break;
+		}
+		if (theNextSp.getStatus() != null) {
+			switch (theNextSp.getStatusElement().getValueAsEnum()) {
+				case ACTIVE:
+					status = RuntimeSearchParam.RuntimeSearchParamStatusEnum.ACTIVE;
+					break;
+				case DRAFT:
+					status = RuntimeSearchParam.RuntimeSearchParamStatusEnum.DRAFT;
+					break;
+				case RETIRED:
+					status = RuntimeSearchParam.RuntimeSearchParamStatusEnum.RETIRED;
+					break;
+			}
+		}
+		Set<String> providesMembershipInCompartments = Collections.emptySet();
+		Set<String> targets = toStrings(theNextSp.getTarget());
+
+		if (isBlank(name) || isBlank(path) || paramType == null) {
+			if (paramType != RestSearchParameterTypeEnum.COMPOSITE) {
+				return null;
+			}
+		}
+
+		IIdType id = theNextSp.getIdElement();
+		String uri = "";
+		boolean unique = false;
+
+		List<ExtensionDt> uniqueExts = theNextSp.getUndeclaredExtensionsByUrl(JpaConstants.EXT_SP_UNIQUE);
+		if (uniqueExts.size() > 0) {
+			IPrimitiveType<?> uniqueExtsValuePrimitive = uniqueExts.get(0).getValueAsPrimitive();
+			if (uniqueExtsValuePrimitive != null) {
+				if ("true".equalsIgnoreCase(uniqueExtsValuePrimitive.getValueAsString())) {
+					unique = true;
+				}
+			}
+		}
+
+		List<JpaRuntimeSearchParam.Component> components = Collections.emptyList();
+		Collection<? extends IPrimitiveType<String>> base = Arrays.asList(theNextSp.getBaseElement());
+		JpaRuntimeSearchParam retVal = new JpaRuntimeSearchParam(id, uri, name, description, path, paramType, providesMembershipInCompartments, targets, status, unique, components, base);
+		return retVal;
+	}
+
+	private Set<String> toStrings(List<? extends CodeDt> theTarget) {
+		HashSet<String> retVal = new HashSet<String>();
+		for (CodeDt next : theTarget) {
+			if (isNotBlank(next.getValue())) {
+				retVal.add(next.getValue());
+			}
+		}
+		return retVal;
+	}
+
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoSearchParameterDstu3.java

@@ -1,27 +1,23 @@
 package ca.uhn.fhir.jpa.dao.dstu3;
 
+import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.dao.BaseSearchParamExtractor;
 import ca.uhn.fhir.jpa.dao.IFhirResourceDaoSearchParameter;
 import ca.uhn.fhir.jpa.dao.IFhirSystemDao;
 import ca.uhn.fhir.jpa.dao.ISearchParamRegistry;
+import ca.uhn.fhir.jpa.dao.r4.FhirResourceDaoSearchParameterR4;
 import ca.uhn.fhir.jpa.entity.ResourceTable;
 import ca.uhn.fhir.parser.DataFormatException;
 import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
 import ca.uhn.fhir.util.ElementUtil;
 import org.apache.commons.lang3.time.DateUtils;
-import org.hl7.fhir.dstu3.model.Bundle;
+import org.hl7.fhir.dstu3.model.*;
-import org.hl7.fhir.dstu3.model.Enumerations;
-import org.hl7.fhir.dstu3.model.Meta;
-import org.hl7.fhir.dstu3.model.SearchParameter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.transaction.TransactionDefinition;
+
-import org.springframework.transaction.TransactionStatus;
+import java.util.List;
-import org.springframework.transaction.support.TransactionCallback;
-import org.springframework.transaction.support.TransactionTemplate;
 
 import static org.apache.commons.lang3.StringUtils.isBlank;
-import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 /*
  * #%L
@@ -54,26 +50,7 @@ public class FhirResourceDaoSearchParameterDstu3 extends FhirResourceDaoDstu3<Se
 	private IFhirSystemDao<Bundle, Meta> mySystemDao;
 
 	protected void markAffectedResources(SearchParameter theResource) {
-		if (theResource != null) {
+		markResourcesMatchingExpressionAsNeedingReindexing(theResource != null ? theResource.getExpression() : null);
-			String expression = theResource.getExpression();
-			if (isNotBlank(expression)) {
-				final String resourceType = expression.substring(0, expression.indexOf('.'));
-				ourLog.info("Marking all resources of type {} for reindexing due to updated search parameter with path: {}", resourceType, expression);
-
-				TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager);
-				txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
-				int updatedCount = txTemplate.execute(new TransactionCallback<Integer>() {
-					@Override
-					public Integer doInTransaction(TransactionStatus theStatus) {
-						return myResourceTableDao.markResourcesOfTypeAsRequiringReindexing(resourceType);
-					}
-				});
-
-				ourLog.info("Marked {} resources for reindexing", updatedCount);
-			}
-		}
-
-		mySearchParamRegistry.forceRefresh();
 	}
 
 	/**
@@ -123,57 +100,13 @@ public class FhirResourceDaoSearchParameterDstu3 extends FhirResourceDaoDstu3<Se
 	protected void validateResourceForStorage(SearchParameter theResource, ResourceTable theEntityToSave) {
 		super.validateResourceForStorage(theResource, theEntityToSave);
 
-		if (theResource.getStatus() == null) {
+		Enum<?> status = theResource.getStatus();
-			throw new UnprocessableEntityException("SearchParameter.status is missing or invalid: " + theResource.getStatusElement().getValueAsString());
+		List<CodeType> base = theResource.getBase();
-		}
-
-		if (ElementUtil.isEmpty(theResource.getBase())) {
-			throw new UnprocessableEntityException("SearchParameter.base is missing");
-		}
-
 		String expression = theResource.getExpression();
-		if (theResource.getType() == Enumerations.SearchParamType.COMPOSITE && isBlank(expression)) {
+		FhirContext context = getContext();
-
+		Enumerations.SearchParamType type = theResource.getType();
-			// this is ok
-
-		} else if (isBlank(expression)) {
-
-			throw new UnprocessableEntityException("SearchParameter.expression is missing");
-
-		} else {
-
-			expression = expression.trim();
-			theResource.setExpression(expression);
-
-			String[] expressionSplit = BaseSearchParamExtractor.SPLIT.split(expression);
-			String allResourceName = null;
-			for (String nextPath : expressionSplit) {
-				nextPath = nextPath.trim();
-
-				int dotIdx = nextPath.indexOf('.');
-				if (dotIdx == -1) {
-					throw new UnprocessableEntityException("Invalid SearchParameter.expression value \"" + nextPath + "\". Must start with a resource name");
-				}
-
-				String resourceName = nextPath.substring(0, dotIdx);
-				try {
-					getContext().getResourceDefinition(resourceName);
-				} catch (DataFormatException e) {
-					throw new UnprocessableEntityException("Invalid SearchParameter.expression value \"" + nextPath + "\": " + e.getMessage());
-				}
-
-				if (allResourceName == null) {
-					allResourceName = resourceName;
-				} else {
-					if (!allResourceName.equals(resourceName)) {
-						throw new UnprocessableEntityException("Invalid SearchParameter.expression value \"" + nextPath + "\". All paths in a single SearchParameter must match the same resource type");
-					}
-				}
-
-			}
-
-		} // if have expression
 
+		FhirResourceDaoSearchParameterR4.validateSearchParam(type, status, base, expression, context);
 	}
 
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoSearchParameterR4.java

@@ -1,7 +1,22 @@
 package ca.uhn.fhir.jpa.dao.r4;
 
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.dao.BaseSearchParamExtractor;
+import ca.uhn.fhir.jpa.dao.IFhirResourceDaoSearchParameter;
+import ca.uhn.fhir.jpa.dao.IFhirSystemDao;
+import ca.uhn.fhir.jpa.entity.ResourceTable;
+import ca.uhn.fhir.parser.DataFormatException;
+import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
+import ca.uhn.fhir.util.ElementUtil;
+import org.apache.commons.lang3.time.DateUtils;
+import org.hl7.fhir.instance.model.api.IPrimitiveType;
+import org.hl7.fhir.r4.model.*;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+
+import java.util.List;
+
 import static org.apache.commons.lang3.StringUtils.isBlank;
-import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 /*
  * #%L
@@ -12,9 +27,9 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -23,59 +38,16 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
  * #L%
  */
 
-import org.apache.commons.lang3.time.DateUtils;
-import org.hl7.fhir.r4.model.Bundle;
-import org.hl7.fhir.r4.model.Enumerations;
-import org.hl7.fhir.r4.model.Meta;
-import org.hl7.fhir.r4.model.SearchParameter;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.transaction.TransactionDefinition;
-import org.springframework.transaction.TransactionStatus;
-import org.springframework.transaction.annotation.Propagation;
-import org.springframework.transaction.support.TransactionCallback;
-import org.springframework.transaction.support.TransactionTemplate;
-
-import ca.uhn.fhir.jpa.dao.BaseSearchParamExtractor;
-import ca.uhn.fhir.jpa.dao.IFhirResourceDaoSearchParameter;
-import ca.uhn.fhir.jpa.dao.IFhirSystemDao;
-import ca.uhn.fhir.jpa.dao.ISearchParamRegistry;
-import ca.uhn.fhir.jpa.entity.ResourceTable;
-import ca.uhn.fhir.parser.DataFormatException;
-import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
-import ca.uhn.fhir.util.ElementUtil;
-
 public class FhirResourceDaoSearchParameterR4 extends FhirResourceDaoR4<SearchParameter> implements IFhirResourceDaoSearchParameter<SearchParameter> {
 
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoSearchParameterR4.class);
 
-	@Autowired
-	private ISearchParamRegistry mySearchParamRegistry;
 
 	@Autowired
 	private IFhirSystemDao<Bundle, Meta> mySystemDao;
 
 	protected void markAffectedResources(SearchParameter theResource) {
-		if (theResource != null) {
+		markResourcesMatchingExpressionAsNeedingReindexing(theResource != null ? theResource.getExpression() : null);
-			String expression = theResource.getExpression();
-			if (isNotBlank(expression)) {
-				final String resourceType = expression.substring(0, expression.indexOf('.'));
-				ourLog.info("Marking all resources of type {} for reindexing due to updated search parameter with path: {}", resourceType, expression);
-
-				TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager);
-				txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
-				int updatedCount = txTemplate.execute(new TransactionCallback<Integer>() {
-					@Override
-					public Integer doInTransaction(TransactionStatus theStatus) {
-						return myResourceTableDao.markResourcesOfTypeAsRequiringReindexing(resourceType);
-					}
-				});
-
-				ourLog.info("Marked {} resources for reindexing", updatedCount);
-			}
-		}
-
-		mySearchParamRegistry.forceRefresh();
 	}
 
 	/**
@@ -125,29 +97,37 @@ public class FhirResourceDaoSearchParameterR4 extends FhirResourceDaoR4<SearchPa
 	protected void validateResourceForStorage(SearchParameter theResource, ResourceTable theEntityToSave) {
 		super.validateResourceForStorage(theResource, theEntityToSave);
 
-		if (theResource.getStatus() == null) {
+		Enum<?> status = theResource.getStatus();
-			throw new UnprocessableEntityException("SearchParameter.status is missing or invalid: " + theResource.getStatusElement().getValueAsString());
+		List<CodeType> base = theResource.getBase();
+		String expression = theResource.getExpression();
+		FhirContext context = getContext();
+		Enum<?> type = theResource.getType();
+		
+		FhirResourceDaoSearchParameterR4.validateSearchParam(type, status, base, expression, context);
+	}
+
+	public static void validateSearchParam(Enum<?> theType, Enum<?> theStatus, List<? extends IPrimitiveType> theBase, String theExpression, FhirContext theContext) {
+		if (theStatus == null) {
+			throw new UnprocessableEntityException("SearchParameter.status is missing or invalid");
 		}
 
-		if (ElementUtil.isEmpty(theResource.getBase())) {
+		if (ElementUtil.isEmpty(theBase)) {
 			throw new UnprocessableEntityException("SearchParameter.base is missing");
 		}
 
-		String expression = theResource.getExpression();
+		if (theType != null && theType.name().equals(Enumerations.SearchParamType.COMPOSITE.name()) && isBlank(theExpression)) {
-		if (theResource.getType() == Enumerations.SearchParamType.COMPOSITE && isBlank(expression)) {
 
 			// this is ok
 
-		} else if (isBlank(expression)) {
+		} else if (isBlank(theExpression)) {
 
 			throw new UnprocessableEntityException("SearchParameter.expression is missing");
 
 		} else {
 
-			expression = expression.trim();
+			theExpression = theExpression.trim();
-			theResource.setExpression(expression);
 
-			String[] expressionSplit = BaseSearchParamExtractor.SPLIT.split(expression);
+			String[] expressionSplit = BaseSearchParamExtractor.SPLIT.split(theExpression);
 			String allResourceName = null;
 			for (String nextPath : expressionSplit) {
 				nextPath = nextPath.trim();
@@ -159,7 +139,7 @@ public class FhirResourceDaoSearchParameterR4 extends FhirResourceDaoR4<SearchPa
 
 				String resourceName = nextPath.substring(0, dotIdx);
 				try {
-					getContext().getResourceDefinition(resourceName);
+					theContext.getResourceDefinition(resourceName);
 				} catch (DataFormatException e) {
 					throw new UnprocessableEntityException("Invalid SearchParameter.expression value \"" + nextPath + "\": " + e.getMessage());
 				}
@@ -175,7 +155,6 @@ public class FhirResourceDaoSearchParameterR4 extends FhirResourceDaoR4<SearchPa
 			}
 
 		} // if have expression
-
 	}
 
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/subscription/email/SubscriptionDeliveringEmailSubscriber.java

@@ -54,16 +54,13 @@ public class SubscriptionDeliveringEmailSubscriber extends BaseSubscriptionDeliv
 		List<String> destinationAddresses = new ArrayList<>();
 		String[] destinationAddressStrings = StringUtils.split(endpointUrl, ",");
 		for (String next : destinationAddressStrings) {
-			next = trim(defaultString(next));
+			next = processEmailAddressUri(next);
-			if (next.startsWith("mailto:")) {
-				next = next.substring("mailto:".length());
-			}
 			if (isNotBlank(next)) {
 				destinationAddresses.add(next);
 			}
 		}
 
-		String from = defaultString(subscription.getEmailDetails().getFrom(), mySubscriptionEmailInterceptor.getDefaultFromAddress());
+		String from = processEmailAddressUri(defaultString(subscription.getEmailDetails().getFrom(), mySubscriptionEmailInterceptor.getDefaultFromAddress()));
 		String subjectTemplate = defaultString(subscription.getEmailDetails().getSubjectTemplate(), provideDefaultSubjectTemplate());
 
 		EmailDetails details = new EmailDetails();
@@ -77,6 +74,14 @@ public class SubscriptionDeliveringEmailSubscriber extends BaseSubscriptionDeliv
 		emailSender.send(details);
 	}
 
+	private String processEmailAddressUri(String next) {
+		next = trim(defaultString(next));
+		if (next.startsWith("mailto:")) {
+         next = next.substring("mailto:".length());
+      }
+		return next;
+	}
+
 
 	private String provideDefaultSubjectTemplate() {
 		return "HAPI FHIR Subscriptions";

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/BaseJpaDstu2Test.java

@@ -45,12 +45,17 @@ import static org.mockito.Mockito.mock;
 @RunWith(SpringJUnit4ClassRunner.class)
 @ContextConfiguration(classes = {TestDstu2Config.class})
 public abstract class BaseJpaDstu2Test extends BaseJpaTest {
+	@Autowired
+	protected ISearchParamRegistry mySearchParamRegsitry;
 	@Autowired
 	protected ApplicationContext myAppCtx;
 	@Autowired
 	@Qualifier("myAppointmentDaoDstu2")
 	protected IFhirResourceDao<Appointment> myAppointmentDao;
 	@Autowired
+	@Qualifier("mySearchParameterDaoDstu2")
+	protected IFhirResourceDao<SearchParameter> mySearchParameterDao;
+	@Autowired
 	@Qualifier("myBundleDaoDstu2")
 	protected IFhirResourceDao<Bundle> myBundleDao;
 	@Autowired

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchCustomSearchParamTest.java

@@ -0,0 +1,867 @@
+package ca.uhn.fhir.jpa.dao.dstu2;
+
+import ca.uhn.fhir.jpa.dao.SearchParameterMap;
+import ca.uhn.fhir.jpa.dao.dstu3.BaseJpaDstu3Test;
+import ca.uhn.fhir.model.api.ExtensionDt;
+import ca.uhn.fhir.model.api.Include;
+import ca.uhn.fhir.model.dstu2.composite.CodeableConceptDt;
+import ca.uhn.fhir.model.dstu2.composite.CodingDt;
+import ca.uhn.fhir.model.dstu2.composite.ResourceReferenceDt;
+import ca.uhn.fhir.model.dstu2.resource.Appointment;
+import ca.uhn.fhir.model.dstu2.resource.Patient;
+import ca.uhn.fhir.model.dstu2.resource.Practitioner;
+import ca.uhn.fhir.model.dstu2.resource.SearchParameter;
+import ca.uhn.fhir.model.dstu2.valueset.*;
+import ca.uhn.fhir.model.primitive.*;
+import ca.uhn.fhir.rest.api.server.IBundleProvider;
+import ca.uhn.fhir.rest.param.*;
+import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
+import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
+import ca.uhn.fhir.util.TestUtil;
+import org.hl7.fhir.instance.model.api.IIdType;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.*;
+import static org.junit.Assert.*;
+
+public class FhirResourceDaoDstu2SearchCustomSearchParamTest extends BaseJpaDstu2Test {
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoDstu2SearchCustomSearchParamTest.class);
+
+	@Before
+	public void beforeDisableResultReuse() {
+		myDaoConfig.setReuseCachedSearchResultsForMillis(null);
+	}
+
+	@Test
+	public void testCreateInvalidParamInvalidResourceName() {
+		SearchParameter fooSp = new SearchParameter();
+		fooSp.setBase(ResourceTypeEnum.PATIENT);
+		fooSp.setCode("foo");
+		fooSp.setType(SearchParamTypeEnum.TOKEN);
+		fooSp.setXpath("PatientFoo.gender");
+		fooSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		fooSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		try {
+			mySearchParameterDao.create(fooSp, mySrd);
+			fail();
+		} catch (UnprocessableEntityException e) {
+			assertEquals("Invalid SearchParameter.expression value \"PatientFoo.gender\": Unknown resource name \"PatientFoo\" (this name is not known in FHIR version \"DSTU2\")", e.getMessage());
+		}
+	}
+
+	@Test
+	public void testCreateInvalidNoBase() {
+		SearchParameter fooSp = new SearchParameter();
+		fooSp.setCode("foo");
+		fooSp.setType(SearchParamTypeEnum.TOKEN);
+				fooSp.setXpath("Patient.gender");
+		fooSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		fooSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		try {
+			mySearchParameterDao.create(fooSp, mySrd);
+			fail();
+		} catch (UnprocessableEntityException e) {
+			assertEquals("SearchParameter.base is missing", e.getMessage());
+		}
+	}
+
+	@Test
+	public void testCreateInvalidParamMismatchedResourceName() {
+		SearchParameter fooSp = new SearchParameter();
+		fooSp.setBase(ResourceTypeEnum.PATIENT);
+		fooSp.setCode("foo");
+		fooSp.setType(SearchParamTypeEnum.TOKEN);
+				fooSp.setXpath("Patient.gender or Observation.code");
+		fooSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		fooSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		try {
+			mySearchParameterDao.create(fooSp, mySrd);
+			fail();
+		} catch (UnprocessableEntityException e) {
+			assertEquals("Invalid SearchParameter.expression value \"Observation.code\". All paths in a single SearchParameter must match the same resource type", e.getMessage());
+		}
+	}
+
+	@Test
+	public void testCreateInvalidParamNoPath() {
+		SearchParameter fooSp = new SearchParameter();
+		fooSp.setBase(ResourceTypeEnum.PATIENT);
+		fooSp.setCode("foo");
+		fooSp.setType(SearchParamTypeEnum.TOKEN);
+				fooSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		fooSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		try {
+			mySearchParameterDao.create(fooSp, mySrd);
+			fail();
+		} catch (UnprocessableEntityException e) {
+			assertEquals("SearchParameter.expression is missing", e.getMessage());
+		}
+	}
+
+	@Test
+	public void testCreateInvalidParamNoResourceName() {
+		SearchParameter fooSp = new SearchParameter();
+		fooSp.setBase(ResourceTypeEnum.PATIENT);
+		fooSp.setCode("foo");
+		fooSp.setType(SearchParamTypeEnum.TOKEN);
+				fooSp.setXpath("gender");
+		fooSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		fooSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		try {
+			mySearchParameterDao.create(fooSp, mySrd);
+			fail();
+		} catch (UnprocessableEntityException e) {
+			assertEquals("Invalid SearchParameter.expression value \"gender\". Must start with a resource name", e.getMessage());
+		}
+	}
+
+	@Test
+	public void testCreateInvalidParamParamNullStatus() {
+
+		SearchParameter fooSp = new SearchParameter();
+		fooSp.setBase(ResourceTypeEnum.PATIENT);
+		fooSp.setCode("foo");
+		fooSp.setType(SearchParamTypeEnum.TOKEN);
+				fooSp.setXpath("Patient.gender");
+		fooSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		fooSp.setStatus((ConformanceResourceStatusEnum) null);
+		try {
+			mySearchParameterDao.create(fooSp, mySrd);
+			fail();
+		} catch (UnprocessableEntityException e) {
+			assertEquals("SearchParameter.status is missing or invalid", e.getMessage());
+		}
+
+	}
+
+	@Test
+	public void testExtensionWithNoValueIndexesWithoutFailure() {
+		SearchParameter eyeColourSp = new SearchParameter();
+		eyeColourSp.setBase(ResourceTypeEnum.PATIENT);
+		eyeColourSp.setCode("eyecolour");
+		eyeColourSp.setType(SearchParamTypeEnum.TOKEN);
+		eyeColourSp.setXpath("Patient.extension('http://acme.org/eyecolour')");
+		eyeColourSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		eyeColourSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		mySearchParameterDao.create(eyeColourSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient p1 = new Patient();
+		p1.setActive(true);
+		p1.addUndeclaredExtension(false, "http://acme.org/eyecolour").addUndeclaredExtension(false, "http://foo").setValue(new StringDt("VAL"));
+		IIdType p1id = myPatientDao.create(p1).getId().toUnqualifiedVersionless();
+
+	}
+
+	@Test
+	public void testSearchForExtensionToken() {
+		SearchParameter eyeColourSp = new SearchParameter();
+		eyeColourSp.setBase(ResourceTypeEnum.PATIENT);
+		eyeColourSp.setCode("eyecolour");
+		eyeColourSp.setType(SearchParamTypeEnum.TOKEN);
+		eyeColourSp.setXpath("Patient.extension('http://acme.org/eyecolour')");
+		eyeColourSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		eyeColourSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		mySearchParameterDao.create(eyeColourSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient p1 = new Patient();
+		p1.setActive(true);
+		p1.addUndeclaredExtension(false, "http://acme.org/eyecolour").setValue(new CodeDt("blue"));
+		IIdType p1id = myPatientDao.create(p1).getId().toUnqualifiedVersionless();
+
+		Patient p2 = new Patient();
+		p2.setActive(true);
+		p2.addUndeclaredExtension(false, "http://acme.org/eyecolour").setValue(new CodeDt("green"));
+		IIdType p2id = myPatientDao.create(p2).getId().toUnqualifiedVersionless();
+
+		// Try with custom gender SP
+		SearchParameterMap map = new SearchParameterMap();
+		map.add("eyecolour", new TokenParam(null, "blue"));
+		IBundleProvider results = myPatientDao.search(map);
+		List<String> foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p1id.getValue()));
+
+	}
+
+	@Test
+	public void testSearchForExtensionTwoDeepCoding() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("foobar");
+		siblingSp.setType(SearchParamTypeEnum.TOKEN);
+		siblingSp.setXpath("Patient.extension('http://acme.org/foo').extension('http://acme.org/bar')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		siblingSp.addTarget(ResourceTypeEnum.ORGANIZATION);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient patient = new Patient();
+		patient.addName().addFamily("P2");
+		ExtensionDt extParent = patient
+			.addUndeclaredExtension(false,
+			"http://acme.org/foo");
+		extParent
+				.addUndeclaredExtension(false,
+				"http://acme.org/bar")
+				.setValue(new CodingDt().setSystem("foo").setCode("bar"));
+
+		IIdType p2id = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.add("foobar", new TokenParam("foo", "bar"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+	}
+
+	@Test
+	public void testSearchForExtensionTwoDeepCodeableConcept() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("foobar");
+		siblingSp.setType(SearchParamTypeEnum.TOKEN);
+		siblingSp.setXpath("Patient.extension('http://acme.org/foo').extension('http://acme.org/bar')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		siblingSp.addTarget(ResourceTypeEnum.ORGANIZATION);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient patient = new Patient();
+		patient.addName().addFamily("P2");
+		ExtensionDt extParent = patient
+			.addUndeclaredExtension(false,"http://acme.org/foo");
+		extParent
+				.addUndeclaredExtension(false,
+				"http://acme.org/bar")
+				.setValue(new CodeableConceptDt().addCoding(new CodingDt().setSystem("foo").setCode("bar")));
+
+		IIdType p2id = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.add("foobar", new TokenParam("foo", "bar"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+	}
+
+	@Test
+	public void testSearchForExtensionTwoDeepReferenceWrongType() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("foobar");
+		siblingSp.setType(SearchParamTypeEnum.REFERENCE);
+		siblingSp.setXpath("Patient.extension('http://acme.org/foo').extension('http://acme.org/bar')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		siblingSp.addTarget(ResourceTypeEnum.OBSERVATION);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Appointment apt = new Appointment();
+		apt.setStatus(AppointmentStatusEnum.ARRIVED);
+		IIdType aptId = myAppointmentDao.create(apt).getId().toUnqualifiedVersionless();
+
+		Patient patient = new Patient();
+		patient.addName().addFamily("P2");
+		ExtensionDt extParent = patient
+			.addUndeclaredExtension(false,
+			"http://acme.org/foo");
+
+		extParent
+				.addUndeclaredExtension(false,
+				"http://acme.org/bar")
+				.setValue(new ResourceReferenceDt(aptId.getValue()));
+
+		IIdType p2id = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.add("foobar", new ReferenceParam(aptId.getValue()));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, empty());
+	}
+
+	@Test
+	public void testSearchForExtensionTwoDeepReferenceWithoutType() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("foobar");
+		siblingSp.setType(SearchParamTypeEnum.REFERENCE);
+		siblingSp.setXpath("Patient.extension('http://acme.org/foo').extension('http://acme.org/bar')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Appointment apt = new Appointment();
+		apt.setStatus(AppointmentStatusEnum.ARRIVED);
+		IIdType aptId = myAppointmentDao.create(apt).getId().toUnqualifiedVersionless();
+
+		Patient patient = new Patient();
+		patient.addName().addFamily("P2");
+		ExtensionDt extParent = patient
+			.addUndeclaredExtension(false,
+			"http://acme.org/foo");
+
+		extParent
+				.addUndeclaredExtension(false,
+				"http://acme.org/bar")
+				.setValue(new ResourceReferenceDt(aptId.getValue()));
+
+		IIdType p2id = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.add("foobar", new ReferenceParam(aptId.getValue()));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+	}
+
+	@Test
+	public void testSearchForExtensionTwoDeepReference() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("foobar");
+		siblingSp.setType(SearchParamTypeEnum.REFERENCE);
+		siblingSp.setXpath("Patient.extension('http://acme.org/foo').extension('http://acme.org/bar')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		siblingSp.addTarget(ResourceTypeEnum.APPOINTMENT);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Appointment apt = new Appointment();
+		apt.setStatus(AppointmentStatusEnum.ARRIVED);
+		IIdType aptId = myAppointmentDao.create(apt).getId().toUnqualifiedVersionless();
+
+		Patient patient = new Patient();
+		patient.addName().addFamily("P2");
+		ExtensionDt extParent = patient
+			.addUndeclaredExtension(false,			"http://acme.org/foo");
+
+		extParent
+				.addUndeclaredExtension(false,				"http://acme.org/bar")
+				.setValue(new ResourceReferenceDt(aptId.getValue()));
+
+		IIdType p2id = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.add("foobar", new ReferenceParam(aptId.getValue()));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+	}
+
+	@Test
+	public void testSearchForExtensionTwoDeepDate() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("foobar");
+		siblingSp.setType(SearchParamTypeEnum.DATE_DATETIME);
+		siblingSp.setXpath("Patient.extension('http://acme.org/foo').extension('http://acme.org/bar')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Appointment apt = new Appointment();
+		apt.setStatus(AppointmentStatusEnum.ARRIVED);
+		IIdType aptId = myAppointmentDao.create(apt).getId().toUnqualifiedVersionless();
+
+		Patient patient = new Patient();
+		patient.addName().addFamily("P2");
+		ExtensionDt extParent = patient
+			.addUndeclaredExtension(false,
+			"http://acme.org/foo");
+
+		extParent
+				.addUndeclaredExtension(false,
+				"http://acme.org/bar")
+				.setValue(new DateDt("2012-01-02"));
+
+		IIdType p2id = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.add("foobar", new DateParam("2012-01-02"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+	}
+
+	@Test
+	public void testSearchForExtensionTwoDeepNumber() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("foobar");
+		siblingSp.setType(SearchParamTypeEnum.NUMBER);
+		siblingSp.setXpath("Patient.extension('http://acme.org/foo').extension('http://acme.org/bar')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient patient = new Patient();
+		patient.addName().addFamily("P2");
+		ExtensionDt extParent = patient
+			.addUndeclaredExtension(false,
+			"http://acme.org/foo");
+		extParent
+				.addUndeclaredExtension(false,
+				"http://acme.org/bar")
+				.setValue(new IntegerDt(5));
+
+		IIdType p2id = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.add("foobar", new NumberParam("5"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+	}
+
+	@Test
+	public void testSearchForExtensionTwoDeepDecimal() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("foobar");
+		siblingSp.setType(SearchParamTypeEnum.NUMBER);
+		siblingSp.setXpath("Patient.extension('http://acme.org/foo').extension('http://acme.org/bar')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient patient = new Patient();
+		patient.addName().addFamily("P2");
+		ExtensionDt extParent = patient
+			.addUndeclaredExtension(false,
+			"http://acme.org/foo");
+		extParent
+				.addUndeclaredExtension(false,
+				"http://acme.org/bar")
+				.setValue(new DecimalDt("2.1"));
+
+		IIdType p2id = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.add("foobar", new NumberParam("2.1"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+	}
+
+	@Test
+	public void testSearchForExtensionTwoDeepString() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("foobar");
+		siblingSp.setType(SearchParamTypeEnum.STRING);
+		siblingSp.setXpath("Patient.extension('http://acme.org/foo').extension('http://acme.org/bar')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient patient = new Patient();
+		patient.addName().addFamily("P2");
+		ExtensionDt extParent = patient
+			.addUndeclaredExtension(false,
+			"http://acme.org/foo");
+		extParent
+				.addUndeclaredExtension(false,
+				"http://acme.org/bar")
+				.setValue(new StringDt("HELLOHELLO"));
+
+		IIdType p2id = myPatientDao.create(patient).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.add("foobar", new StringParam("hello"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+	}
+
+	@Test
+	public void testSearchForExtensionReferenceWithNonMatchingTarget() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("sibling");
+		siblingSp.setType(SearchParamTypeEnum.REFERENCE);
+		siblingSp.setXpath("Patient.extension('http://acme.org/sibling')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		siblingSp.addTarget(ResourceTypeEnum.ORGANIZATION);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient p1 = new Patient();
+		p1.addName().addFamily("P1");
+		IIdType p1id = myPatientDao.create(p1).getId().toUnqualifiedVersionless();
+
+		Patient p2 = new Patient();
+		p2.addName().addFamily("P2");
+		p2.addUndeclaredExtension(false, "http://acme.org/sibling").setValue(new ResourceReferenceDt(p1id));
+		IIdType p2id = myPatientDao.create(p2).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		// Search by ref
+		map = new SearchParameterMap();
+		map.add("sibling", new ReferenceParam(p1id.getValue()));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, empty());
+
+		// Search by chain
+		map = new SearchParameterMap();
+		map.add("sibling", new ReferenceParam("name", "P1"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, empty());
+
+	}
+
+
+
+	@Test
+	public void testSearchForExtensionReferenceWithoutTarget() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("sibling");
+		siblingSp.setType(SearchParamTypeEnum.REFERENCE);
+		siblingSp.setXpath("Patient.extension('http://acme.org/sibling')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient p1 = new Patient();
+		p1.addName().addFamily("P1");
+		IIdType p1id = myPatientDao.create(p1).getId().toUnqualifiedVersionless();
+
+		Patient p2 = new Patient();
+		p2.addName().addFamily("P2");
+		p2.addUndeclaredExtension(false, "http://acme.org/sibling").setValue(new ResourceReferenceDt(p1id));
+
+		IIdType p2id = myPatientDao.create(p2).getId().toUnqualifiedVersionless();
+		Appointment app = new Appointment();
+		app.addParticipant().getActor().setReference(p2id.getValue());
+		IIdType appid = myAppointmentDao.create(app).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		// Search by ref
+		map = new SearchParameterMap();
+		map.add("sibling", new ReferenceParam(p1id.getValue()));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+
+		// Search by chain
+		map = new SearchParameterMap();
+		map.add("sibling", new ReferenceParam("name", "P1"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+
+		// Search by two level chain
+		map = new SearchParameterMap();
+		map.add("patient", new ReferenceParam("sibling.name", "P1"));
+		results = myAppointmentDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, containsInAnyOrder(appid.getValue()));
+
+
+	}
+
+	@Test
+	public void testSearchForExtensionReferenceWithTarget() {
+		SearchParameter siblingSp = new SearchParameter();
+		siblingSp.setBase(ResourceTypeEnum.PATIENT);
+		siblingSp.setCode("sibling");
+		siblingSp.setType(SearchParamTypeEnum.REFERENCE);
+		siblingSp.setXpath("Patient.extension('http://acme.org/sibling')");
+		siblingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		siblingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		siblingSp.addTarget(ResourceTypeEnum.PATIENT);
+		mySearchParameterDao.create(siblingSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient p1 = new Patient();
+		p1.addName().addFamily("P1");
+		IIdType p1id = myPatientDao.create(p1).getId().toUnqualifiedVersionless();
+
+		Patient p2 = new Patient();
+		p2.addName().addFamily("P2");
+		p2.addUndeclaredExtension(false, "http://acme.org/sibling").setValue(new ResourceReferenceDt(p1id));
+		IIdType p2id = myPatientDao.create(p2).getId().toUnqualifiedVersionless();
+
+		Appointment app = new Appointment();
+		app.addParticipant().getActor().setReference(p2id.getValue());
+		IIdType appid = myAppointmentDao.create(app).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		// Search by ref
+		map = new SearchParameterMap();
+		map.add("sibling", new ReferenceParam(p1id.getValue()));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+
+		// Search by chain
+		map = new SearchParameterMap();
+		map.add("sibling", new ReferenceParam("name", "P1"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(p2id.getValue()));
+
+		// Search by two level chain
+		map = new SearchParameterMap();
+		map.add("patient", new ReferenceParam("sibling.name", "P1"));
+		results = myAppointmentDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, containsInAnyOrder(appid.getValue()));
+
+	}
+
+	@Test
+	public void testIncludeExtensionReferenceAsRecurse() {
+		SearchParameter attendingSp = new SearchParameter();
+		attendingSp.setBase(ResourceTypeEnum.PATIENT);
+		attendingSp.setCode("attending");
+		attendingSp.setType(SearchParamTypeEnum.REFERENCE);
+		attendingSp.setXpath("Patient.extension('http://acme.org/attending')");
+		attendingSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		attendingSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		attendingSp.addTarget(ResourceTypeEnum.PRACTITIONER);
+		IIdType spId = mySearchParameterDao.create(attendingSp, mySrd).getId().toUnqualifiedVersionless();
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Practitioner p1 = new Practitioner();
+		p1.getName().addFamily("P1");
+		IIdType p1id = myPractitionerDao.create(p1).getId().toUnqualifiedVersionless();
+
+		Patient p2 = new Patient();
+		p2.addName().addFamily("P2");
+		p2.addUndeclaredExtension(false, "http://acme.org/attending").setValue(new ResourceReferenceDt(p1id));
+		IIdType p2id = myPatientDao.create(p2).getId().toUnqualifiedVersionless();
+
+		Appointment app = new Appointment();
+		app.addParticipant().getActor().setReference(p2id.getValue());
+		IIdType appId = myAppointmentDao.create(app).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		map = new SearchParameterMap();
+		map.addInclude(new Include("Appointment:patient", true));
+		map.addInclude(new Include("Patient:attending", true));
+		results = myAppointmentDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(appId.getValue(), p2id.getValue(), p1id.getValue()));
+
+	}
+
+	@Test
+	public void testSearchWithCustomParam() {
+
+		SearchParameter fooSp = new SearchParameter();
+		fooSp.setBase(ResourceTypeEnum.PATIENT);
+		fooSp.setCode("foo");
+		fooSp.setType(SearchParamTypeEnum.TOKEN);
+				fooSp.setXpath("Patient.gender");
+		fooSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		fooSp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		IIdType spId = mySearchParameterDao.create(fooSp, mySrd).getId().toUnqualifiedVersionless();
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient pat = new Patient();
+		pat.setGender(AdministrativeGenderEnum.MALE);
+		IIdType patId = myPatientDao.create(pat, mySrd).getId().toUnqualifiedVersionless();
+
+		Patient pat2 = new Patient();
+		pat.setGender(AdministrativeGenderEnum.FEMALE);
+		IIdType patId2 = myPatientDao.create(pat2, mySrd).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		// Try with custom gender SP
+		map = new SearchParameterMap();
+		map.add("foo", new TokenParam(null, "male"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(patId.getValue()));
+
+		// Try with normal gender SP
+		map = new SearchParameterMap();
+		map.add("gender", new TokenParam(null, "male"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(patId.getValue()));
+
+		// Delete the param
+		mySearchParameterDao.delete(spId, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+		mySystemDao.performReindexingPass(100);
+
+		// Try with custom gender SP
+		map = new SearchParameterMap();
+		map.add("foo", new TokenParam(null, "male"));
+		try {
+			myPatientDao.search(map).size();
+			fail();
+		} catch (InvalidRequestException e) {
+			assertEquals("Unknown search parameter foo for resource type Patient", e.getMessage());
+		}
+	}
+
+	@Test
+	public void testSearchWithCustomParamDraft() {
+
+		SearchParameter fooSp = new SearchParameter();
+		fooSp.setBase(ResourceTypeEnum.PATIENT);
+		fooSp.setCode("foo");
+		fooSp.setType(SearchParamTypeEnum.TOKEN);
+				fooSp.setXpath("Patient.gender");
+		fooSp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		fooSp.setStatus(ConformanceResourceStatusEnum.DRAFT);
+		mySearchParameterDao.create(fooSp, mySrd);
+
+		mySearchParamRegsitry.forceRefresh();
+
+		Patient pat = new Patient();
+		pat.setGender(AdministrativeGenderEnum.MALE);
+		IIdType patId = myPatientDao.create(pat, mySrd).getId().toUnqualifiedVersionless();
+
+		Patient pat2 = new Patient();
+		pat.setGender(AdministrativeGenderEnum.FEMALE);
+		IIdType patId2 = myPatientDao.create(pat2, mySrd).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+		IBundleProvider results;
+		List<String> foundResources;
+
+		// Try with custom gender SP (should find nothing)
+		map = new SearchParameterMap();
+		map.add("foo", new TokenParam(null, "male"));
+		try {
+			myPatientDao.search(map).size();
+			fail();
+		} catch (InvalidRequestException e) {
+			assertEquals("Unknown search parameter foo for resource type Patient", e.getMessage());
+		}
+
+		// Try with normal gender SP
+		map = new SearchParameterMap();
+		map.add("gender", new TokenParam(null, "male"));
+		results = myPatientDao.search(map);
+		foundResources = toUnqualifiedVersionlessIdValues(results);
+		assertThat(foundResources, contains(patId.getValue()));
+
+	}
+
+	@Test
+	public void testCustomReferenceParameter() throws Exception {
+		SearchParameter sp = new SearchParameter();
+		sp.setBase(ResourceTypeEnum.PATIENT);
+		sp.setCode("myDoctor");
+		sp.setType(SearchParamTypeEnum.REFERENCE);
+		sp.setXpath("Patient.extension('http://fmcna.com/myDoctor')");
+		sp.setXpathUsage(XPathUsageTypeEnum.NORMAL);
+		sp.setStatus(ConformanceResourceStatusEnum.ACTIVE);
+		mySearchParameterDao.create(sp);
+
+		Practitioner pract = new Practitioner();
+		pract.setId("A");
+		pract.getName().addFamily("PRACT");
+		myPractitionerDao.update(pract);
+		
+		Patient pat = new Patient();
+		pat.addUndeclaredExtension(false, "http://fmcna.com/myDoctor").setValue(new ResourceReferenceDt("Practitioner/A"));
+
+		IIdType pid = myPatientDao.create(pat, mySrd).getId().toUnqualifiedVersionless();
+		
+		SearchParameterMap params = new SearchParameterMap();
+		params.add("myDoctor", new ReferenceParam("A"));
+		IBundleProvider outcome = myPatientDao.search(params);
+		List<String> ids = toUnqualifiedVersionlessIdValues(outcome);
+		ourLog.info("IDS: " + ids);
+		assertThat(ids, contains(pid.getValue()));
+	}
+
+	@AfterClass
+	public static void afterClassClearContext() {
+		TestUtil.clearAllStaticFieldsForUnitTest();
+	}
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchCustomSearchParamTest.java

@@ -130,7 +130,7 @@ public class FhirResourceDaoDstu3SearchCustomSearchParamTest extends BaseJpaDstu
 			mySearchParameterDao.create(fooSp, mySrd);
 			fail();
 		} catch (UnprocessableEntityException e) {
-			assertEquals("SearchParameter.status is missing or invalid: null", e.getMessage());
+			assertEquals("SearchParameter.status is missing or invalid", e.getMessage());
 		}
 
 	}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchCustomSearchParamTest.java

@@ -132,7 +132,7 @@ public class FhirResourceDaoR4SearchCustomSearchParamTest extends BaseJpaR4Test
 			mySearchParameterDao.create(fooSp, mySrd);
 			fail();
 		} catch (UnprocessableEntityException e) {
-			assertEquals("SearchParameter.status is missing or invalid: null", e.getMessage());
+			assertEquals("SearchParameter.status is missing or invalid", e.getMessage());
 		}
 
 	}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/email/EmailSubscriptionDstu3Test.java

@@ -176,6 +176,53 @@ public class EmailSubscriptionDstu3Test extends BaseResourceProviderDstu3Test {
 		String criteria1 = "Observation?code=SNOMED-CT|" + code + "&_format=xml";
 		Subscription sub1 = createSubscription(criteria1, payload);
 
+		Subscription subscriptionTemp = ourClient.read(Subscription.class, sub1.getId());
+		Assert.assertNotNull(subscriptionTemp);
+		subscriptionTemp.getChannel().addExtension()
+			.setUrl(JpaConstants.EXT_SUBSCRIPTION_EMAIL_FROM)
+			.setValue(new StringType("mailto:myfrom@from.com"));
+		subscriptionTemp.getChannel().addExtension()
+			.setUrl(JpaConstants.EXT_SUBSCRIPTION_SUBJECT_TEMPLATE)
+			.setValue(new StringType("This is a subject"));
+		subscriptionTemp.setIdElement(subscriptionTemp.getIdElement().toUnqualifiedVersionless());
+
+
+		ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(subscriptionTemp));
+
+		ourClient.update().resource(subscriptionTemp).withId(subscriptionTemp.getIdElement()).execute();
+		waitForQueueToDrain();
+
+
+		sendObservation(code, "SNOMED-CT");
+		waitForQueueToDrain();
+
+		waitForSize(1, 20000, new Callable<Number>() {
+			@Override
+			public Number call() throws Exception {
+				return ourTestSmtp.getReceivedMessages().length;
+			}
+		});
+
+		List<MimeMessage> received = Arrays.asList(ourTestSmtp.getReceivedMessages());
+		assertEquals(1, received.size());
+		assertEquals(1, received.get(0).getFrom().length);
+		assertEquals("myfrom@from.com", ((InternetAddress)received.get(0).getFrom()[0]).getAddress());
+		assertEquals(1, received.get(0).getAllRecipients().length);
+		assertEquals("foo@example.com", ((InternetAddress)received.get(0).getAllRecipients()[0]).getAddress());
+		assertEquals("text/plain; charset=us-ascii", received.get(0).getContentType());
+		assertEquals("This is a subject", received.get(0).getSubject().toString().trim());
+		assertEquals("This is the body", received.get(0).getContent().toString().trim());
+		assertEquals(mySubscriptionIds.get(0).toUnqualifiedVersionless().getValue(), received.get(0).getHeader("X-FHIR-Subscription")[0]);
+	}
+
+	@Test
+	public void testEmailSubscriptionWithCustomNoMailtoOnFrom() throws Exception {
+		String payload = "This is the body";
+
+		String code = "1000000050";
+		String criteria1 = "Observation?code=SNOMED-CT|" + code + "&_format=xml";
+		Subscription sub1 = createSubscription(criteria1, payload);
+
 		Subscription subscriptionTemp = ourClient.read(Subscription.class, sub1.getId());
 		Assert.assertNotNull(subscriptionTemp);
 		subscriptionTemp.getChannel().addExtension()