Merge pull request #2567 from hapifhir/2566_update_s13n_and_validation_handling

Update s13n and validation handling

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/TerserUtil.java

@@ -25,6 +25,7 @@ import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
 import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
+import org.apache.commons.lang3.Validate;
 import org.apache.commons.lang3.tuple.Triple;
 import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseResource;
@@ -172,9 +173,7 @@ public final class TerserUtil {
 
 		RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
 		BaseRuntimeChildDefinition childDefinition = definition.getChildByName(theField);
-		if (childDefinition == null) {
+		Validate.notNull(childDefinition);
-			throw new IllegalArgumentException(String.format("Unable to find child definition %s in %s", theField, theFrom));
-		}
 
 		List<IBase> theFromFieldValues = childDefinition.getAccessor().getValues(theFrom);
 		List<IBase> theToFieldValues = childDefinition.getAccessor().getValues(theTo);
@@ -226,9 +225,7 @@ public final class TerserUtil {
 		}
 
 		final Method method = getMethod(theItem1, EQUALS_DEEP);
-		if (method == null) {
+		Validate.notNull(method);
-			throw new IllegalArgumentException(String.format("Instance %s do not provide %s method", theItem1, EQUALS_DEEP));
-		}
 		return equals(theItem1, theItem2, method);
 	}
 
@@ -315,9 +312,7 @@ public final class TerserUtil {
 	 */
 	public static void replaceField(FhirContext theFhirContext, String theFieldName, IBaseResource theFrom, IBaseResource theTo) {
 		RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
-		if (definition == null) {
+		Validate.notNull(definition);
-			throw new IllegalArgumentException(String.format("Field %s does not exist in %s", theFieldName, theFrom));
-		}
 		replaceField(theFrom, theTo, theFhirContext.getResourceDefinition(theFrom).getChildByName(theFieldName));
 	}
 
@@ -333,6 +328,20 @@ public final class TerserUtil {
 		childDefinition.getAccessor().getValues(theResource).clear();
 	}
 
+	/**
+	 * Clears the specified field on the element provided
+	 *
+	 * @param theFhirContext Context holding resource definition
+	 * @param theFieldName   Name of the field to clear values for
+	 * @param theBase        The element definition to clear values on
+	 */
+	public static void clearField(FhirContext theFhirContext, String theFieldName, IBase theBase) {
+		BaseRuntimeElementDefinition definition = theFhirContext.getElementDefinition(theBase.getClass());
+		BaseRuntimeChildDefinition childDefinition = definition.getChildByName(theFieldName);
+		Validate.notNull(childDefinition);
+		childDefinition.getAccessor().getValues(theBase).clear();
+	}
+
 	/**
 	 * Sets the provided field with the given values. This method will add to the collection of existing field values
 	 * in case of multiple cardinality. Use {@link #clearField(FhirContext, String, IBaseResource)}
@@ -512,9 +521,7 @@ public final class TerserUtil {
 	private static BaseRuntimeChildDefinition getBaseRuntimeChildDefinition(FhirContext theFhirContext, String theFieldName, IBaseResource theFrom) {
 		RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
 		BaseRuntimeChildDefinition childDefinition = definition.getChildByName(theFieldName);
-		if (childDefinition == null) {
+		Validate.notNull(childDefinition);
-			throw new IllegalStateException(String.format("Field %s does not exist", theFieldName));
-		}
 		return childDefinition;
 	}
 
@@ -577,9 +584,7 @@ public final class TerserUtil {
 	 */
 	public static <T extends IBase> T newElement(FhirContext theFhirContext, String theElementType, Object theConstructorParam) {
 		BaseRuntimeElementDefinition def = theFhirContext.getElementDefinition(theElementType);
-		if (def == null) {
+		Validate.notNull(def);
-			throw new IllegalArgumentException(String.format("Unable to find element type definition for %s", theElementType));
-		}
 		return (T) def.newInstance(theConstructorParam);
 	}
 

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/interceptors/built_in_server_interceptors.md

@@ -299,12 +299,9 @@ The UserRequestRetryVersionConflictsInterceptor allows clients to request that t
 
 The RepositoryValidatingInterceptor can be used to enforce validation rules on data stored in a HAPI FHIR JPA Repository. See [Repository Validating Interceptor](/docs/validation/repository_validating_interceptor.html) for more information. 
 
-
-
-
 # Data Standardization
 
-`StandardizingInterceptor` handles data standardization (s13n) requirements. This interceptor applies standardization rules on all FHIR primitives as configured in the `s13n.json` file that should be made available on the classpath. This file contains FHIRPath definitions together with the standardizers that should be applied to that path. It comes with six pre-built standardizers: NAME_FAMILY, NAME_GIVEN, EMAIL, TITLE, PHONE and TEXT. Custom standardizers can be developed by implementing `ca.uhn.fhir.rest.server.interceptor.s13n.standardizers.IStandardizer` interface.
+`StandardizingInterceptor` handles data standardization (s13n) requirements. This interceptor applies standardization rules on all FHIR primitives as configured in the `s13n.json` file that should be made available on the classpath. This file contains FHIRPath definitions together with the standardizers that should be applied to that path. Currently, there are six pre-built standardizers: NAME_FAMILY, NAME_GIVEN, EMAIL, TITLE, PHONE and TEXT. Custom standardizers can be developed by implementing `ca.uhn.fhir.rest.server.interceptor.s13n.standardizers.IStandardizer` interface and providing class name in the configuration.
 
 A sample configuration file can be found below:
 
@@ -331,7 +328,7 @@ Standardization can be disabled for a given request by providing `HAPI-Standardi
 
 # Validation: Address Validation
 
-`AddressValidatingInterceptor` takes care of validation of addresses on all incoming resources through a 3rd party address validation service. Before a resource containing an Address field is stored, this interceptor invokes address validation service and then stores validation results as an extension on the address with `https://hapifhir.org/AddressValidation/` URL.
+`AddressValidatingInterceptor` validates addresses on all incoming resources through a 3rd party address validation service. This interceptor invokes address validation service, updates the address with the validated results and adds a validation extension with `http://hapifhir.org/StructureDefinition/ext-validation-address-has-error` URL. 
 
 This interceptor is configured in `address-validation.properties` file that should be made available on the classpath. This file must contain `validator.class` property, which defines a fully qualified class implementing `ca.uhn.fhir.rest.server.interceptor.validation.address.IAddressValidator` interface. The specified implementation must provide service-specific logic for validating an Address instance. An example implementation can be found in `ca.uhn.fhir.rest.server.interceptor.validation.address.impl.LoquateAddressValidator` class which validates addresses by using Loquate Data Cleanse service.
 
@@ -339,12 +336,12 @@ Address validation can be disabled for a given request by providing `HAPI-Addres
 
 # Validation: Field-Level Validation
 
-`FieldValidatingInterceptor` allows validating primitive fields on various FHIR resources. It expects validation rules to be provided via `field-validation-rules.json` file that should be available on the classpath. JSON in this file defines a mapping of FHIRPath expressions to validators that should be applied to those fields. Custom validators that implement `ca.uhn.fhir.rest.server.interceptor.validation.fields.IValidator` interface can be provided.
+`FieldValidatingInterceptor` enables validation of primitive values on various FHIR resources. It expects validation rules to be provided via `field-validation-rules.json` file that should be available on the classpath. JSON in this file defines a mapping of FHIRPath expressions to validators that should be applied to those fields. Custom validators that implement `ca.uhn.fhir.rest.server.interceptor.validation.fields.IValidator` interface can be provided.
 
 ```json
 {
-	"telecom.where(system='email').value" : "EMAIL",
+	"telecom.where(system='email')" : "EMAIL",
-   "telecom.where(system='phone').value" : "org.example.validation.MyCustomValidator"
+   "telecom.where(system='phone')" : "org.example.validation.MyCustomValidator"
 }
 ```
 

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/validation/address/AddressValidatingInterceptor.java

@@ -21,6 +21,8 @@ package ca.uhn.fhir.rest.server.interceptor.validation.address;
  */
 
 import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
+import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
+import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
 import ca.uhn.fhir.interceptor.api.Hook;
@@ -29,9 +31,14 @@ import ca.uhn.fhir.interceptor.api.Pointcut;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.server.interceptor.ConfigLoader;
 import ca.uhn.fhir.util.ExtensionUtil;
+import ca.uhn.fhir.util.FhirTerser;
+import ca.uhn.fhir.util.IModelVisitor2;
+import ca.uhn.fhir.util.TerserUtil;
 import org.apache.commons.lang3.Validate;
 import org.hl7.fhir.instance.model.api.IBase;
+import org.hl7.fhir.instance.model.api.IBaseExtension;
 import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.instance.model.api.IDomainResource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -47,6 +54,7 @@ public class AddressValidatingInterceptor {
 
 	public static final String ADDRESS_TYPE_NAME = "Address";
 	public static final String PROPERTY_VALIDATOR_CLASS = "validator.class";
+	public static final String PROPERTY_EXTENSION_URL = "extension.url";
 
 	public static final String ADDRESS_VALIDATION_DISABLED_HEADER = "HAPI-Address-Validation-Disabled";
 
@@ -54,7 +62,6 @@ public class AddressValidatingInterceptor {
 
 	private Properties myProperties;
 
-
 	public AddressValidatingInterceptor() {
 		super();
 
@@ -65,6 +72,7 @@ public class AddressValidatingInterceptor {
 
 	public AddressValidatingInterceptor(Properties theProperties) {
 		super();
+		myProperties = theProperties;
 		start(theProperties);
 	}
 
@@ -118,28 +126,76 @@ public class AddressValidatingInterceptor {
 
 		if (!theRequest.getHeaders(ADDRESS_VALIDATION_DISABLED_HEADER).isEmpty()) {
 			ourLog.debug("Address validation is disabled for this request via header");
+			return;
 		}
 
 		FhirContext ctx = theRequest.getFhirContext();
-		getAddresses(theResource, ctx)
+		List<IBase> addresses = getAddresses(theResource, ctx)
 			.stream()
-			.filter(a -> {
+			.filter(this::isValidating)
-				return !ExtensionUtil.hasExtension(a, IAddressValidator.ADDRESS_VALIDATION_EXTENSION_URL) ||
+			.collect(Collectors.toList());
-					ExtensionUtil.hasExtension(a, IAddressValidator.ADDRESS_VALIDATION_EXTENSION_URL, IAddressValidator.EXT_UNABLE_TO_VALIDATE);
+
-			})
+		if (!addresses.isEmpty()) {
-			.forEach(a -> validateAddress(a, ctx));
+			validateAddresses(theRequest, theResource, addresses);
+		}
 	}
 
-	protected void validateAddress(IBase theAddress, FhirContext theFhirContext) {
+	/**
+	 * Validates specified child addresses for the resource
+	 *
+	 * @return Returns true if all addresses are valid, or false if there is at least one invalid address
+	 */
+	protected boolean validateAddresses(RequestDetails theRequest, IBaseResource theResource, List<IBase> theAddresses) {
+		boolean retVal = true;
+		for (IBase address : theAddresses) {
+			retVal &= validateAddress(address, theRequest.getFhirContext());
+		}
+		return retVal;
+	}
+
+	private boolean isValidating(IBase theAddress) {
+		IBaseExtension ext = ExtensionUtil.getExtensionByUrl(theAddress, getExtensionUrl());
+		if (ext == null) {
+			return true;
+		}
+		if (ext.getValue() == null || ext.getValue().isEmpty()) {
+			return true;
+		}
+		return !"false".equals(ext.getValue().toString());
+	}
+
+	protected boolean validateAddress(IBase theAddress, FhirContext theFhirContext) {
 		try {
 			AddressValidationResult validationResult = getAddressValidator().isValid(theAddress, theFhirContext);
 			ourLog.debug("Validated address {}", validationResult);
 
-			ExtensionUtil.setExtensionAsString(theFhirContext, theAddress, IAddressValidator.ADDRESS_VALIDATION_EXTENSION_URL,
+			clearPossibleDuplicatesDueToTerserCloning(theAddress, theFhirContext);
-				validationResult.isValid() ? IAddressValidator.EXT_VALUE_VALID : IAddressValidator.EXT_VALUE_INVALID);
+			ExtensionUtil.setExtension(theFhirContext, theAddress, getExtensionUrl(), "boolean", !validationResult.isValid());
+			if (validationResult.getValidatedAddress() != null) {
+				theFhirContext.newTerser().cloneInto(validationResult.getValidatedAddress(), theAddress, true);
+			} else {
+				ourLog.info("Validated address is not provided - skipping update on the target address instance");
+			}
+			return validationResult.isValid();
 		} catch (Exception ex) {
 			ourLog.warn("Unable to validate address", ex);
-			ExtensionUtil.setExtensionAsString(theFhirContext, theAddress, IAddressValidator.ADDRESS_VALIDATION_EXTENSION_URL, IAddressValidator.EXT_UNABLE_TO_VALIDATE);
+			IBaseExtension extension = ExtensionUtil.getOrCreateExtension(theAddress, getExtensionUrl());
+			IBaseExtension errorValue = ExtensionUtil.getOrCreateExtension(extension, "error");
+			errorValue.setValue(TerserUtil.newElement(theFhirContext, "string", ex.getMessage()));
+			return false;
+		}
+	}
+
+	private void clearPossibleDuplicatesDueToTerserCloning(IBase theAddress, FhirContext theFhirContext) {
+		TerserUtil.clearField(theFhirContext, "line", theAddress);
+		ExtensionUtil.clearExtensionsByUrl(theAddress, getExtensionUrl());
+	}
+
+	protected String getExtensionUrl() {
+		if (getProperties().containsKey(PROPERTY_EXTENSION_URL)) {
+			return getProperties().getProperty(PROPERTY_EXTENSION_URL);
+		} else {
+			return IAddressValidator.ADDRESS_VALIDATION_EXTENSION_URL;
 		}
 	}
 

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/validation/address/IAddressValidator.java

@@ -29,24 +29,29 @@ import org.hl7.fhir.instance.model.api.IBase;
 public interface IAddressValidator {
 
 	/**
-	 * URL for validation results that should be placed on addresses
+	 * URL for validation results that should be placed on addresses. Extension with boolean value "true" indicates there there is an address validation error.
 	 */
-	public static final String ADDRESS_VALIDATION_EXTENSION_URL = "https://hapifhir.org/AddressValidation/";
+	public static final String ADDRESS_VALIDATION_EXTENSION_URL = "http://hapifhir.org/StructureDefinition/ext-validation-address-has-error";
 
 	/**
-	 * Extension value confirming that address can be considered valid (it exists and can be traced to the building)
+	 * URL for an optional address quality extensions that may be added to addresses.
 	 */
-	public static final String EXT_VALUE_VALID = "valid";
+	public static final String ADDRESS_QUALITY_EXTENSION_URL = "http://hapifhir.org/StructureDefinition/ext-validation-address-quality";
 
 	/**
-	 * Extension value confirming that address is invalid (doesn't exist)
+	 * URL for an optional geocoding accuracy extensions that may be added to addresses.
 	 */
-	public static final String EXT_VALUE_INVALID = "invalid";
+	public static final String ADDRESS_GEO_ACCURACY_EXTENSION_URL = "http://hapifhir.org/StructureDefinition/ext-validation-address-geo-accuracy";
 
 	/**
-	 * Extension value indicating that address validation was attempted but could not complete successfully
+	 * URL for an optional address verification extensions that may be added to addresses.
 	 */
-	public static final String EXT_UNABLE_TO_VALIDATE = "not-validated";
+	public static final String ADDRESS_VERIFICATION_CODE_EXTENSION_URL = "http://hapifhir.org/StructureDefinition/ext-validation-address-verification";
+
+	/**
+	 * URL for an optional FHIR geolocation extension.
+	 */
+	public static final String FHIR_GEOCODE_EXTENSION_URL = "http://hl7.org/fhir/StructureDefinition/geolocation";
 
 	/**
 	 * Validates address against a service

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/validation/address/impl/BaseRestfulValidator.java

@@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationException;
 import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationResult;
+import org.apache.commons.lang3.Validate;
 import org.hl7.fhir.instance.model.api.IBase;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -37,6 +38,7 @@ import java.util.Properties;
 public abstract class BaseRestfulValidator implements IAddressValidator {
 
 	public static final String PROPERTY_SERVICE_KEY = "service.key";
+	public static final String PROPERTY_SERVICE_ENDPOINT = "service.endpoint";
 
 	private static final Logger ourLog = LoggerFactory.getLogger(BaseRestfulValidator.class);
 
@@ -74,7 +76,7 @@ public abstract class BaseRestfulValidator implements IAddressValidator {
 		retVal.setRawResponse(responseBody);
 
 		try {
-			JsonNode response  = new ObjectMapper().readTree(responseBody);
+			JsonNode response = new ObjectMapper().readTree(responseBody);
 			ourLog.debug("Parsed address validator response {}", response);
 			return getValidationResult(retVal, response, theFhirContext);
 		} catch (Exception e) {
@@ -97,4 +99,8 @@ public abstract class BaseRestfulValidator implements IAddressValidator {
 	protected String getApiKey() {
 		return getProperties().getProperty(PROPERTY_SERVICE_KEY);
 	}
+
+	protected String getApiEndpoint() {
+		return getProperties().getProperty(PROPERTY_SERVICE_ENDPOINT);
+	}
 }

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/validation/address/impl/LoquateAddressValidator.java

@@ -21,16 +21,20 @@ package ca.uhn.fhir.rest.server.interceptor.validation.address.impl;
  */
 
 import ca.uhn.fhir.context.FhirContext;
-import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationException;
 import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationResult;
 import ca.uhn.fhir.rest.server.interceptor.validation.helpers.AddressHelper;
+import ca.uhn.fhir.util.ExtensionUtil;
+import ca.uhn.fhir.util.TerserUtil;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Validate;
 import org.apache.http.entity.ContentType;
 import org.hl7.fhir.instance.model.api.IBase;
+import org.hl7.fhir.instance.model.api.IBaseExtension;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpEntity;
@@ -38,7 +42,14 @@ import org.springframework.http.HttpHeaders;
 import org.springframework.http.ResponseEntity;
 
 import javax.annotation.Nullable;
+import java.math.BigDecimal;
+import java.util.Arrays;
 import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static ca.uhn.fhir.rest.server.interceptor.validation.address.IAddressValidator.ADDRESS_QUALITY_EXTENSION_URL;
+import static ca.uhn.fhir.rest.server.interceptor.validation.address.IAddressValidator.ADDRESS_VERIFICATION_CODE_EXTENSION_URL;
 
 /**
  * For more details regarind the API refer to
@@ -50,33 +61,32 @@ public class LoquateAddressValidator extends BaseRestfulValidator {
 
 	private static final Logger ourLog = LoggerFactory.getLogger(LoquateAddressValidator.class);
 
-	private static final String[] DUPLICATE_FIELDS_IN_ADDRESS_LINES = {"Locality", "AdministrativeArea", "PostalCode"};
+	public static final String PROPERTY_GEOCODE = "service.geocode";
+	public static final String LOQUATE_AQI = "AQI";
+	public static final String LOQUATE_AVC = "AVC";
+	public static final String LOQUATE_GEO_ACCURACY = "GeoAccuracy";
 
-	private static final String DATA_CLEANSE_ENDPOINT = "https://api.addressy.com/Cleansing/International/Batch/v1.00/json4.ws";
+	protected static final String[] DUPLICATE_FIELDS_IN_ADDRESS_LINES = {"Locality", "AdministrativeArea", "PostalCode"};
-	private static final int MAX_ADDRESS_LINES = 8;
+	protected static final String DEFAULT_DATA_CLEANSE_ENDPOINT = "https://api.addressy.com/Cleansing/International/Batch/v1.00/json4.ws";
+	protected static final int MAX_ADDRESS_LINES = 8;
+
+	private Pattern myCommaPattern = Pattern.compile("\\,(\\S)");
 
 	public LoquateAddressValidator(Properties theProperties) {
 		super(theProperties);
-		if (!theProperties.containsKey(PROPERTY_SERVICE_KEY)) {
+		Validate.isTrue(theProperties.containsKey(PROPERTY_SERVICE_KEY) || theProperties.containsKey(PROPERTY_SERVICE_ENDPOINT),
-			throw new IllegalArgumentException(String.format("Missing service key defined as %s", PROPERTY_SERVICE_KEY));
+			"Expected service key or custom service endpoint in the configuration, but got " + theProperties);
-		}
 	}
 
 	@Override
 	protected AddressValidationResult getValidationResult(AddressValidationResult theResult, JsonNode response, FhirContext theFhirContext) {
-		if (!response.isArray() || response.size() < 1) {
+		Validate.isTrue(response.isArray() && response.size() >= 1, "Invalid response - expected to get an array of validated addresses");
-			throw new AddressValidationException("Invalid response - expected to get an array of validated addresses");
-		}
 
 		JsonNode firstMatch = response.get(0);
-		if (!firstMatch.has("Matches")) {
+		Validate.isTrue(firstMatch.has("Matches"), "Invalid response - matches are unavailable");
-			throw new AddressValidationException("Invalid response - matches are unavailable");
-		}
 
 		JsonNode matches = firstMatch.get("Matches");
-		if (!matches.isArray()) {
+		Validate.isTrue(matches.isArray(), "Invalid response - expected to get a validated match in the response");
-			throw new AddressValidationException("Invalid response - expected to get a validated match in the response");
-		}
 
 		JsonNode match = matches.get(0);
 		return toAddressValidationResult(theResult, match, theFhirContext);
@@ -97,26 +107,34 @@ public class LoquateAddressValidator extends BaseRestfulValidator {
 	}
 
 	protected boolean isValid(JsonNode theMatch) {
-		String addressQualityIndex = null;
+		String addressQualityIndex = getField(theMatch, LOQUATE_AQI);
-		if (theMatch.has("AQI")) {
+		return "A".equals(addressQualityIndex) || "B".equals(addressQualityIndex) || "C".equals(addressQualityIndex);
-			addressQualityIndex = theMatch.get("AQI").asText();
+	}
-		}
 
-		ourLog.debug("Address quality index {}", addressQualityIndex);
+	private String getField(JsonNode theMatch, String theFieldName) {
-		return "A".equals(addressQualityIndex) || "B".equals(addressQualityIndex);
+		String field = null;
+		if (theMatch.has(theFieldName)) {
+			field = theMatch.get(theFieldName).asText();
+		}
+		ourLog.debug("Found {}={}", theFieldName, field);
+		return field;
 	}
 
 	protected IBase toAddress(JsonNode match, FhirContext theFhirContext) {
 		IBase addressBase = theFhirContext.getElementDefinition("Address").newInstance();
 
 		AddressHelper helper = new AddressHelper(theFhirContext, addressBase);
-		helper.setText(getString(match, "Address"));
+		helper.setText(standardize(getString(match, "Address")));
 
 		String str = getString(match, "Address1");
 		if (str != null) {
 			helper.addLine(str);
 		}
 
+		if (isGeocodeEnabled()) {
+			toGeolocation(match, helper, theFhirContext);
+		}
+
 		removeDuplicateAddressLines(match, helper);
 
 		helper.setCity(getString(match, "Locality"));
@@ -124,9 +142,46 @@ public class LoquateAddressValidator extends BaseRestfulValidator {
 		helper.setPostalCode(getString(match, "PostalCode"));
 		helper.setCountry(getString(match, "CountryName"));
 
+		addExtension(match, LOQUATE_AQI, ADDRESS_QUALITY_EXTENSION_URL, helper, theFhirContext);
+		addExtension(match, LOQUATE_AVC, ADDRESS_VERIFICATION_CODE_EXTENSION_URL, helper, theFhirContext);
+		addExtension(match, LOQUATE_GEO_ACCURACY, ADDRESS_GEO_ACCURACY_EXTENSION_URL, helper, theFhirContext);
+
 		return helper.getAddress();
 	}
 
+	private void addExtension(JsonNode theMatch, String theMatchField, String theExtUrl, AddressHelper theHelper, FhirContext theFhirContext) {
+		String addressQuality = getField(theMatch, theMatchField);
+		if (StringUtils.isEmpty(addressQuality)) {
+			ourLog.debug("{} is not found in {}", theMatchField, theMatch);
+			return;
+		}
+
+		IBase address = theHelper.getAddress();
+		ExtensionUtil.clearExtensionsByUrl(address, theExtUrl);
+
+		IBaseExtension addressQualityExt = ExtensionUtil.addExtension(address, theExtUrl);
+		addressQualityExt.setValue(TerserUtil.newElement(theFhirContext, "string", addressQuality));
+	}
+
+	private void toGeolocation(JsonNode theMatch, AddressHelper theHelper, FhirContext theFhirContext) {
+		if (!theMatch.has("Latitude") || !theMatch.has("Longitude")) {
+			ourLog.warn("Geocode is not provided in JSON {}", theMatch);
+			return;
+		}
+
+		IBase address = theHelper.getAddress();
+		ExtensionUtil.clearExtensionsByUrl(address, FHIR_GEOCODE_EXTENSION_URL);
+		IBaseExtension geolocation = ExtensionUtil.addExtension(address, FHIR_GEOCODE_EXTENSION_URL);
+
+		IBaseExtension latitude = ExtensionUtil.addExtension(geolocation, "latitude");
+		latitude.setValue(TerserUtil.newElement(theFhirContext, "decimal",
+			BigDecimal.valueOf(theMatch.get("Latitude").asDouble())));
+
+		IBaseExtension longitude = ExtensionUtil.addExtension(geolocation, "longitude");
+		longitude.setValue(TerserUtil.newElement(theFhirContext, "decimal",
+			BigDecimal.valueOf(theMatch.get("Longitude").asDouble())));
+	}
+
 	private void removeDuplicateAddressLines(JsonNode match, AddressHelper address) {
 		int lineCount = 1;
 		String addressLine = null;
@@ -150,7 +205,7 @@ public class LoquateAddressValidator extends BaseRestfulValidator {
 	}
 
 	@Nullable
-	private String getString(JsonNode theNode, String theField) {
+	protected String getString(JsonNode theNode, String theField) {
 		if (!theNode.has(theField)) {
 			return null;
 		}
@@ -159,7 +214,25 @@ public class LoquateAddressValidator extends BaseRestfulValidator {
 		if (field.asText().isEmpty()) {
 			return null;
 		}
-		return theNode.get(theField).asText();
+
+		String text = theNode.get(theField).asText();
+		if (StringUtils.isEmpty(text)) {
+			return "";
+		}
+		return text;
+	}
+
+	protected String standardize(String theText) {
+		if (StringUtils.isEmpty(theText)) {
+			return "";
+		}
+
+		theText = theText.replaceAll("\\s\\s", ", ");
+		Matcher m = myCommaPattern.matcher(theText);
+		if (m.find()) {
+			theText = m.replaceAll(", $1");
+		}
+		return theText.trim();
 	}
 
 	@Override
@@ -171,14 +244,22 @@ public class LoquateAddressValidator extends BaseRestfulValidator {
 
 		String requestBody = getRequestBody(theFhirContext, theAddress);
 		HttpEntity<String> request = new HttpEntity<>(requestBody, headers);
-		return newTemplate().postForEntity(DATA_CLEANSE_ENDPOINT, request, String.class);
+		return newTemplate().postForEntity(getApiEndpoint(), request, String.class);
+	}
+
+	@Override
+	protected String getApiEndpoint() {
+		String endpoint = super.getApiEndpoint();
+		return StringUtils.isEmpty(endpoint) ? DEFAULT_DATA_CLEANSE_ENDPOINT : endpoint;
 	}
 
 	protected String getRequestBody(FhirContext theFhirContext, IBase... theAddresses) throws JsonProcessingException {
 		ObjectMapper mapper = new ObjectMapper();
 		ObjectNode rootNode = mapper.createObjectNode();
-		rootNode.put("Key", getApiKey());
+		if (!StringUtils.isEmpty(getApiKey())) {
-		rootNode.put("Geocode", false);
+			rootNode.put("Key", getApiKey());
+		}
+		rootNode.put("Geocode", isGeocodeEnabled());
 
 		ArrayNode addressesArrayNode = mapper.createArrayNode();
 		int i = 0;
@@ -209,4 +290,11 @@ public class LoquateAddressValidator extends BaseRestfulValidator {
 		addressNode.put("Country", helper.getCountry());
 		return addressNode;
 	}
+
+	protected boolean isGeocodeEnabled() {
+		if (!getProperties().containsKey(PROPERTY_GEOCODE)) {
+			return false;
+		}
+		return Boolean.parseBoolean(getProperties().getProperty(PROPERTY_GEOCODE));
+	}
 }

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/validation/address/impl/MelissaAddressValidator.java

@@ -1,139 +0,0 @@
-package ca.uhn.fhir.rest.server.interceptor.validation.address.impl;
-
-/*-
- * #%L
- * HAPI FHIR - Server Framework
- * %%
- * Copyright (C) 2014 - 2021 Smile CDR, Inc.
- * %%
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * #L%
- */
-
-import ca.uhn.fhir.context.FhirContext;
-import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationException;
-import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationResult;
-import ca.uhn.fhir.rest.server.interceptor.validation.helpers.AddressHelper;
-import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.commons.lang3.StringUtils;
-import org.hl7.fhir.instance.model.api.IBase;
-import org.springframework.http.ResponseEntity;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-public class MelissaAddressValidator extends BaseRestfulValidator {
-
-	public static final String GLOBAL_ADDRESS_VALIDATION_ENDPOINT = "https://address.melissadata.net/v3/WEB/GlobalAddress/doGlobalAddress" +
-		"?id={id}&a1={a1}&a2={a2}&ctry={ctry}&format={format}";
-
-	public MelissaAddressValidator(Properties theProperties) {
-		super(theProperties);
-	}
-
-	@Override
-	protected AddressValidationResult getValidationResult(AddressValidationResult theResult, JsonNode theResponse, FhirContext theFhirContext) {
-		Response response = new Response(theResponse);
-		theResult.setValid(response.isValidAddress());
-		theResult.setValidatedAddressString(response.getAddress());
-		return theResult;
-	}
-
-	@Override
-	protected ResponseEntity<String> getResponseEntity(IBase theAddress, FhirContext theFhirContext) throws Exception {
-		Map<String, String> requestParams = getRequestParams(theAddress);
-		return newTemplate().getForEntity(GLOBAL_ADDRESS_VALIDATION_ENDPOINT, String.class, requestParams);
-	}
-
-	protected Map<String, String> getRequestParams(IBase theAddress) {
-		AddressHelper helper = new AddressHelper(null, theAddress);
-
-		Map<String, String> requestParams = new HashMap<>();
-		requestParams.put("t", UUID.randomUUID().toString());
-		requestParams.put("id", getApiKey());
-		requestParams.put("a1", helper.getLine());
-		requestParams.put("a2", helper.getParts());
-		requestParams.put("ctry", helper.getCountry());
-		requestParams.put("format", "json");
-		return requestParams;
-	}
-
-	private static class Response {
-		private JsonNode root;
-		private JsonNode records;
-		private JsonNode results;
-
-		private List<String> addressErrors = new ArrayList<>();
-		private List<String> addressChange = new ArrayList<>();
-		private List<String> geocodeStatus = new ArrayList<>();
-		private List<String> geocodeError = new ArrayList<>();
-		private List<String> addressVerification = new ArrayList<>();
-
-		public Response(JsonNode theRoot) {
-			root = theRoot;
-
-			// see codes here - http://wiki.melissadata.com/index.php?title=Result_Codes
-			String transmissionResults = root.get("TransmissionResults").asText();
-			if (!StringUtils.isEmpty(transmissionResults)) {
-				geocodeError.add(transmissionResults);
-				throw new AddressValidationException(String.format("Transmission result %s indicate an error with the request - please check API_KEY", transmissionResults));
-			}
-
-			int recordCount = root.get("TotalRecords").asInt();
-			if (recordCount < 1) {
-				throw new AddressValidationException("Expected at least one record in the address validation response");
-			}
-
-			// get first match
-			records = root.get("Records").get(0);
-			results = records.get("Results");
-
-			// full list of response codes is available here
-			// http://wiki.melissadata.com/index.php?title=Result_Code_Details#Global_Address_Verification
-			for (String s : results.asText().split(",")) {
-				if (s.startsWith("AE")) {
-					addressErrors.add(s);
-				} else if (s.startsWith("AC")) {
-					addressChange.add(s);
-				} else if (s.startsWith("GS")) {
-					geocodeStatus.add(s);
-				} else if (s.startsWith("GE")) {
-					geocodeError.add(s);
-				} else if (s.startsWith("AV")) {
-					addressVerification.add(s);
-				}
-			}
-		}
-
-		public boolean isValidAddress() {
-			if (!geocodeError.isEmpty()) {
-				return false;
-			}
-			return addressErrors.isEmpty() && (geocodeStatus.contains("GS05") || geocodeStatus.contains("GS06"));
-		}
-
-		public String getAddress() {
-			if (records == null) {
-				return "";
-			}
-			if (!records.has("FormattedAddress")) {
-				return "";
-			}
-			return records.get("FormattedAddress").asText("");
-		}
-	}
-}

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/validation/fields/FieldValidatingInterceptor.java

@@ -27,7 +27,9 @@ import ca.uhn.fhir.interceptor.api.Interceptor;
 import ca.uhn.fhir.interceptor.api.Pointcut;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.server.interceptor.ConfigLoader;
-import ca.uhn.fhir.rest.server.interceptor.validation.address.IAddressValidator;
+import ca.uhn.fhir.util.ExtensionUtil;
+import org.hl7.fhir.instance.model.api.IBase;
+import org.hl7.fhir.instance.model.api.IBaseExtension;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
 import org.slf4j.Logger;
@@ -36,9 +38,13 @@ import org.slf4j.LoggerFactory;
 import java.util.List;
 import java.util.Map;
 
+import static ca.uhn.fhir.rest.server.interceptor.validation.fields.IValidator.VALIDATION_EXTENSION_URL;
+
 @Interceptor
 public class FieldValidatingInterceptor {
 
+	public static final String FHIR_PATH_VALUE = "value";
+
 	public enum ValidatorType {
 		EMAIL;
 	}
@@ -46,12 +52,10 @@ public class FieldValidatingInterceptor {
 	private static final Logger ourLog = LoggerFactory.getLogger(FieldValidatingInterceptor.class);
 
 	public static final String VALIDATION_DISABLED_HEADER = "HAPI-Field-Validation-Disabled";
-
+	public static final String PROPERTY_EXTENSION_URL = "validation.extension.url";
-	private IAddressValidator myAddressValidator;
 
 	private Map<String, String> myConfig;
 
-
 	public FieldValidatingInterceptor() {
 		super();
 
@@ -84,20 +88,48 @@ public class FieldValidatingInterceptor {
 
 		FhirContext ctx = theRequest.getFhirContext();
 		IFhirPath fhirPath = ctx.newFhirPath();
+
 		for (Map.Entry<String, String> e : myConfig.entrySet()) {
 			IValidator validator = getValidator(e.getValue());
+			if (validator == null) {
+				continue;
+			}
 
-			List<IPrimitiveType> values = fhirPath.evaluate(theResource, e.getKey(), IPrimitiveType.class);
+			List<IBase> fields = fhirPath.evaluate(theResource, e.getKey(), IBase.class);
-			for (IPrimitiveType value : values) {
+			for (IBase field : fields) {
-				String valueAsString = value.getValueAsString();
+
-				if (!validator.isValid(valueAsString)) {
+				List<IPrimitiveType> values = fhirPath.evaluate(field, FHIR_PATH_VALUE, IPrimitiveType.class);
-					throw new IllegalArgumentException(String.format("Invalid resource %s", valueAsString));
+				boolean isValid = true;
+				for (IPrimitiveType value : values) {
+					String valueAsString = value.getValueAsString();
+					isValid = validator.isValid(valueAsString);
+					ourLog.debug("Field {} at path {} validated {}", value, e.getKey(), isValid);
+					if (!isValid) {
+						break;
+					}
 				}
+				setValidationStatus(ctx, field, isValid);
 			}
 		}
 	}
 
+	private void setValidationStatus(FhirContext ctx, IBase theBase, boolean isValid) {
+		ExtensionUtil.clearExtensionsByUrl(theBase, getValidationExtensionUrl());
+		ExtensionUtil.setExtension(ctx, theBase, getValidationExtensionUrl(), "boolean", !isValid);
+	}
+
+	private String getValidationExtensionUrl() {
+		if (myConfig.containsKey(PROPERTY_EXTENSION_URL)) {
+			return myConfig.get(PROPERTY_EXTENSION_URL);
+		}
+		return VALIDATION_EXTENSION_URL;
+	}
+
 	private IValidator getValidator(String theValue) {
+		if (PROPERTY_EXTENSION_URL.equals(theValue)) {
+			return null;
+		}
+
 		if (ValidatorType.EMAIL.name().equals(theValue)) {
 			return new EmailValidator();
 		}

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/interceptor/validation/fields/IValidator.java

@@ -22,6 +22,8 @@ package ca.uhn.fhir.rest.server.interceptor.validation.fields;
 
 public interface IValidator {
 
+	public static final String VALIDATION_EXTENSION_URL = "https://hapifhir.org/StructureDefinition/ext-validation-field-has-error";
+
 	public boolean isValid(String theString);
 
 }

hapi-fhir-server/src/main/resources/field-validation-rules.json

@@ -1,3 +1,3 @@
 {
-	"telecom.where(system='email').value" : "EMAIL"
+	"telecom.where(system='email')" : "EMAIL"
 }

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/interceptor/validation/address/AddressValidatingInterceptorTest.java

@@ -2,24 +2,30 @@ package ca.uhn.fhir.rest.server.interceptor.validation.address;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
+import ca.uhn.fhir.rest.server.interceptor.validation.address.impl.LoquateAddressValidator;
 import org.checkerframework.checker.units.qual.A;
 import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.r4.model.Address;
+import org.hl7.fhir.r4.model.Extension;
 import org.hl7.fhir.r4.model.Person;
 import org.hl7.fhir.r4.model.StringType;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
-import org.mockito.Mock;
 import org.mockito.Mockito;
 
 import javax.annotation.Nonnull;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Properties;
 
-import static ca.uhn.fhir.rest.server.interceptor.s13n.StandardizingInterceptor.STANDARDIZATION_DISABLED_HEADER;
 import static ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidatingInterceptor.ADDRESS_VALIDATION_DISABLED_HEADER;
+import static ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidatingInterceptor.PROPERTY_EXTENSION_URL;
 import static ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidatingInterceptor.PROPERTY_VALIDATOR_CLASS;
+import static ca.uhn.fhir.rest.server.interceptor.validation.address.IAddressValidator.ADDRESS_VALIDATION_EXTENSION_URL;
+import static ca.uhn.fhir.rest.server.interceptor.validation.address.impl.BaseRestfulValidator.PROPERTY_SERVICE_KEY;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -41,6 +47,35 @@ class AddressValidatingInterceptorTest {
 
 	private RequestDetails myRequestDetails;
 
+	@Test
+	@Disabled
+	public void testValidationCallAgainstLiveLoquateEndpoint() {
+		Properties config = new Properties();
+		config.setProperty(PROPERTY_VALIDATOR_CLASS, LoquateAddressValidator.class.getCanonicalName());
+		config.setProperty(PROPERTY_SERVICE_KEY, "KR26-JA29-HB16-PA11"); // Replace with a real key when testing
+		AddressValidatingInterceptor interceptor = new AddressValidatingInterceptor(config);
+
+		Address address = new Address();
+		address.setUse(Address.AddressUse.WORK);
+		address.addLine("100 Somewhere");
+		address.setCity("Burloak");
+		address.setPostalCode("A0A0A0");
+		address.setCountry("Canada");
+		interceptor.validateAddress(address, ourCtx);
+
+		assertTrue(address.hasExtension());
+		assertEquals("true", address.getExtensionFirstRep().getValueAsPrimitive().getValueAsString());
+		assertEquals("E",
+			address.getExtensionByUrl(IAddressValidator.ADDRESS_QUALITY_EXTENSION_URL).getValueAsPrimitive().getValueAsString());
+
+		assertEquals("100 Somewhere, Burloak", address.getText());
+		assertEquals(1, address.getLine().size());
+		assertEquals("100 Somewhere", address.getLine().get(0).getValueAsString());
+		assertEquals("Burloak", address.getCity());
+		assertEquals("A0A0A0", address.getPostalCode());
+		assertEquals("Canada", address.getCountry());
+	}
+
 	@Test
 	void start() throws Exception {
 		AddressValidatingInterceptor interceptor = new AddressValidatingInterceptor(new Properties());
@@ -60,6 +95,22 @@ class AddressValidatingInterceptorTest {
 		assertNotNull(interceptor.getAddressValidator());
 	}
 
+	@Test
+	public void testEmptyRequest() {
+		try {
+			myInterceptor.handleRequest(null, null);
+		} catch (Exception ex) {
+			fail();
+		}
+
+		try {
+			myInterceptor.setAddressValidator(null);
+			myInterceptor.handleRequest(null, null);
+		} catch (Exception ex) {
+			fail();
+		}
+	}
+
 	@BeforeEach
 	void setup() {
 		myValidator = mock(IAddressValidator.class);
@@ -99,7 +150,31 @@ class AddressValidatingInterceptorTest {
 
 		Address address = new Address();
 		myInterceptor.validateAddress(address, ourCtx);
-		assertValidated(address, "not-validated");
+		Extension ext = assertValidationErrorExtension(address);
+		assertTrue(ext.hasExtension());
+		assertEquals("error", ext.getExtensionFirstRep().getUrl());
+	}
+
+	@Test
+	public void testValidationWithCustomUrl() {
+		myInterceptor.getProperties().setProperty(PROPERTY_EXTENSION_URL, "MY_URL");
+		Address address = new Address();
+		address.setCity("City");
+		address.addLine("Line");
+		AddressValidationResult res = new AddressValidationResult();
+		res.setValidatedAddressString("City, Line");
+		res.setValidatedAddress(address);
+		when(myValidator.isValid(any(), any())).thenReturn(res);
+
+		Address addressToValidate = new Address();
+		myInterceptor.validateAddress(addressToValidate, ourCtx);
+
+		assertNotNull(res.toString());
+		assertTrue(addressToValidate.hasExtension());
+		assertNotNull(addressToValidate.getExtensionByUrl("MY_URL"));
+		assertFalse(address.hasExtension());
+		assertEquals(address.getCity(), addressToValidate.getCity());
+		assertTrue(address.getLine().get(0).equalsDeep(addressToValidate.getLine().get(0)));
 	}
 
 	@Test
@@ -109,14 +184,19 @@ class AddressValidatingInterceptorTest {
 		address.setCity("City");
 
 		myInterceptor.validateAddress(address, ourCtx);
-		assertValidated(address, "invalid");
+		assertValidationErrorValue(address, "true");
 	}
 
-	private void assertValidated(Address theAddress, String theValidationResult) {
+	private Extension assertValidationErrorExtension(Address theAddress) {
 		assertTrue(theAddress.hasExtension());
 		assertEquals(1, theAddress.getExtension().size());
 		assertEquals(IAddressValidator.ADDRESS_VALIDATION_EXTENSION_URL, theAddress.getExtensionFirstRep().getUrl());
-		assertEquals(theValidationResult, theAddress.getExtensionFirstRep().getValueAsPrimitive().toString());
+		return theAddress.getExtensionFirstRep();
+	}
+
+	private void assertValidationErrorValue(Address theAddress, String theValidationResult) {
+		Extension ext = assertValidationErrorExtension(theAddress);
+		assertEquals(theValidationResult, ext.getValueAsPrimitive().getValueAsString());
 	}
 
 	@Test
@@ -130,29 +210,29 @@ class AddressValidatingInterceptorTest {
 
 		myInterceptor.resourcePreCreate(myRequestDetails, person);
 
-		assertValidated(person.getAddressFirstRep(), "invalid");
+		assertValidationErrorValue(person.getAddressFirstRep(), "true");
 	}
 
 	@Test
 	void validateOnUpdate() {
-		Address address = new Address();
+		Address validAddress = new Address();
-		address.addLine("Line");
+		validAddress.addLine("Line");
-		address.setCity("City");
+		validAddress.setCity("City");
-		address.addExtension(IAddressValidator.ADDRESS_VALIDATION_EXTENSION_URL, new StringType("..."));
+		validAddress.addExtension(IAddressValidator.ADDRESS_VALIDATION_EXTENSION_URL, new StringType("false"));
 
-		Address address2 = new Address();
+		Address notValidatedAddress = new Address();
-		address2.addLine("Line 2");
+		notValidatedAddress.addLine("Line 2");
-		address2.setCity("City 2");
+		notValidatedAddress.setCity("City 2");
 
 		Person person = new Person();
-		person.addAddress(address);
+		person.addAddress(validAddress);
-		person.addAddress(address2);
+		person.addAddress(notValidatedAddress);
 
 		myInterceptor.resourcePreUpdate(myRequestDetails, null, person);
 
 		verify(myValidator, times(1)).isValid(any(), any());
-		assertValidated(person.getAddress().get(0), "...");
+		assertValidationErrorValue(person.getAddress().get(0), "false");
-		assertValidated(person.getAddress().get(1), "invalid");
+		assertValidationErrorValue(person.getAddress().get(1), "true");
 	}
 
 	public static class TestAddressValidator implements IAddressValidator {

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/interceptor/validation/address/impl/LoquateAddressValidatorTest.java

@@ -3,18 +3,28 @@ package ca.uhn.fhir.rest.server.interceptor.validation.address.impl;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationException;
 import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationResult;
+import ca.uhn.fhir.rest.server.interceptor.validation.address.IAddressValidator;
+import ca.uhn.fhir.util.ExtensionUtil;
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import org.hl7.fhir.instance.model.api.IBase;
+import org.hl7.fhir.instance.model.api.IBaseExtension;
 import org.hl7.fhir.r4.model.Address;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.springframework.http.HttpEntity;
 import org.springframework.web.client.RestTemplate;
 
+import java.util.HashMap;
 import java.util.Properties;
 
+import static ca.uhn.fhir.rest.server.interceptor.validation.address.impl.LoquateAddressValidator.PROPERTY_GEOCODE;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
@@ -53,7 +63,7 @@ class LoquateAddressValidatorTest {
 		"    },\n" +
 		"    \"Matches\": [\n" +
 		"      {\n" +
-		"        \"AQI\": \"C\",\n" +
+		"        \"AQI\": \"D\",\n" +
 		"        \"Address\": \"\"\n" +
 		"      }\n" +
 		"    ]\n" +
@@ -74,6 +84,24 @@ class LoquateAddressValidatorTest {
 		"  }\n" +
 		"]";
 
+	private static final String RESPONSE_VALID_ADDRESS_W_GEO = "[\n" +
+		"  {\n" +
+		"    \"Input\": {\n" +
+		"      \"Address\": \"\"\n" +
+		"    },\n" +
+		"    \"Matches\": [\n" +
+		"      {\n" +
+		"        \"AQI\": \"A\",\n" +
+		"        \"AVC\": \"V44-I44-P6-100\",\n" +
+		"        \"GeoAccuracy\": \"Z1\",\n" +
+		"        \"Address\": \"My Valid Address\",\n" +
+		"        \"Latitude\": \"-32.94217742803439\",\n" +
+		"        \"Longitude\": \"-60.640132034941836\"\n" +
+		"      }\n" +
+		"    ]\n" +
+		"  }\n" +
+		"]";
+
 	private static final String RESPONSE_INVALID_KEY = "{\n" +
 		"  \"Number\": 2,\n" +
 		"  \"Description\": \"Unknown key\",\n" +
@@ -94,6 +122,30 @@ class LoquateAddressValidatorTest {
 		myValidator = new LoquateAddressValidator(myProperties);
 	}
 
+	@Test
+	public void testGetText() {
+		ObjectNode node = new ObjectNode(null, new HashMap<>());
+		node.set("text1", new TextNode("This,Is,Text"));
+		node.set("text2", new TextNode("This Is-Text,"));
+		node.set("text3", new TextNode("This Is-Text  with Invalid Formatting"));
+
+		assertEquals("This, Is, Text", myValidator.standardize(myValidator.getString(node, "text1")));
+		assertEquals("This Is-Text,", myValidator.standardize(myValidator.getString(node, "text2")));
+		assertEquals("This Is-Text, with Invalid Formatting", myValidator.standardize(myValidator.getString(node, "text3")));
+	}
+
+	@Test
+	public void testEndpointOverride() {
+		assertEquals(LoquateAddressValidator.DEFAULT_DATA_CLEANSE_ENDPOINT, myValidator.getApiEndpoint());
+
+		myProperties = new Properties();
+		myProperties.setProperty(LoquateAddressValidator.PROPERTY_SERVICE_KEY, "MY_KEY");
+		myProperties.setProperty(LoquateAddressValidator.PROPERTY_SERVICE_ENDPOINT, "HTTP://MY_ENDPOINT/LOQUATE");
+		myValidator = new LoquateAddressValidator(myProperties);
+
+		assertEquals("HTTP://MY_ENDPOINT/LOQUATE", myValidator.getApiEndpoint());
+	}
+
 	@Test
 	public void testInvalidInit() {
 		try {
@@ -109,7 +161,7 @@ class LoquateAddressValidatorTest {
 			AddressValidationResult res = myValidator.getValidationResult(new AddressValidationResult(),
 				new ObjectMapper().readTree(RESPONSE_INVALID), ourCtx);
 			fail();
-		} catch (AddressValidationException e) {
+		} catch (Exception e) {
 		}
 	}
 
@@ -168,9 +220,36 @@ class LoquateAddressValidatorTest {
 		assertEquals("My Valid Address", res.getValidatedAddressString());
 	}
 
+	@Test
+	public void testSuccessfulResponsesWithGeocodeAndQuality() throws Exception {
+		myValidator.getProperties().setProperty(PROPERTY_GEOCODE, "true");
+		AddressValidationResult res = myValidator.getValidationResult(new AddressValidationResult(),
+			new ObjectMapper().readTree(RESPONSE_VALID_ADDRESS_W_GEO), ourCtx);
+		assertTrue(res.isValid());
+
+		IBase address = res.getValidatedAddress();
+		IBaseExtension geocode = ExtensionUtil.getExtensionByUrl(address, IAddressValidator.FHIR_GEOCODE_EXTENSION_URL);
+		assertNotNull(geocode);
+		assertEquals(2, geocode.getExtension().size());
+		assertEquals("latitude", ((IBaseExtension)geocode.getExtension().get(0)).getUrl());
+		assertEquals("longitude", ((IBaseExtension)geocode.getExtension().get(1)).getUrl());
+
+		IBaseExtension quality = ExtensionUtil.getExtensionByUrl(address, IAddressValidator.ADDRESS_QUALITY_EXTENSION_URL);
+		assertNotNull(quality);
+		assertEquals("A", quality.getValue().toString());
+
+		IBaseExtension verificationCode = ExtensionUtil.getExtensionByUrl(address, IAddressValidator.ADDRESS_VERIFICATION_CODE_EXTENSION_URL);
+		assertNotNull(verificationCode);
+		assertEquals("V44-I44-P6-100", verificationCode.getValue().toString());
+
+		IBaseExtension geoAccuracy = ExtensionUtil.getExtensionByUrl(address, IAddressValidator.ADDRESS_GEO_ACCURACY_EXTENSION_URL);
+		assertNotNull(geoAccuracy);
+		assertEquals("Z1", geoAccuracy.getValue().toString());
+	}
+
 	@Test
 	public void testErrorResponses() throws Exception {
-		assertThrows(AddressValidationException.class, () -> {
+		assertThrows(IllegalArgumentException.class, () -> {
 			myValidator.getValidationResult(new AddressValidationResult(),
 				new ObjectMapper().readTree(RESPONSE_INVALID_KEY), ourCtx);
 		});

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/interceptor/validation/address/impl/MelissaAddressValidatorTest.java

@@ -1,139 +0,0 @@
-package ca.uhn.fhir.rest.server.interceptor.validation.address.impl;
-
-import ca.uhn.fhir.context.FhirContext;
-import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationException;
-import ca.uhn.fhir.rest.server.interceptor.validation.address.AddressValidationResult;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.hl7.fhir.r4.model.Address;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.springframework.web.client.RestTemplate;
-
-import java.util.Map;
-import java.util.Properties;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-class MelissaAddressValidatorTest {
-
-	private static final String RESPONSE_INVALID_ADDRESS = "{\n" +
-		"  \"Version\": \"3.0.1.160\",\n" +
-		"  \"TransmissionReference\": \"1\",\n" +
-		"  \"TransmissionResults\": \"\",\n" +
-		"  \"TotalRecords\": \"1\",\n" +
-		"  \"Records\": [\n" +
-		"    {\n" +
-		"      \"RecordID\": \"1\",\n" +
-		"      \"Results\": \"AC01,AC12,AE02,AV12,GE02\",\n" +
-		"      \"FormattedAddress\": \"100 Main Street\",\n" +
-		"      \"Organization\": \"\",\n" +
-		"      \"AddressLine1\": \"100 Main Street\"\n" +
-		"    }\n" +
-		"  ]\n" +
-		"}";
-
-	private static final String RESPONSE_VALID_ADDRESS = "{\n" +
-		"  \"Version\": \"3.0.1.160\",\n" +
-		"  \"TransmissionReference\": \"1\",\n" +
-		"  \"TransmissionResults\": \"\",\n" +
-		"  \"TotalRecords\": \"1\",\n" +
-		"  \"Records\": [\n" +
-		"    {\n" +
-		"      \"RecordID\": \"1\",\n" +
-		"      \"Results\": \"AC01,AV24,GS05\",\n" +
-		"      \"FormattedAddress\": \"100 Main St W;Hamilton ON  L8P 1H6\"\n" +
-		"   }\n" +
-		"  ]\n" +
-		"}";
-
-	private static final String RESPONSE_INVALID_KEY = "{\n" +
-		"  \"Version\": \"3.0.1.160\",\n" +
-		"  \"TransmissionReference\": \"1\",\n" +
-		"  \"TransmissionResults\": \"GE05\",\n" +
-		"  \"TotalRecords\": \"0\"\n" +
-		"}";
-
-	private static FhirContext ourContext = FhirContext.forR4();
-
-	private MelissaAddressValidator myValidator;
-
-	@BeforeEach
-	public void init() {
-		Properties props = new Properties();
-		props.setProperty(MelissaAddressValidator.PROPERTY_SERVICE_KEY, "MY_KEY");
-		myValidator = new MelissaAddressValidator(props);
-
-	}
-
-	@Test
-	public void testRequestBody() {
-		Map<String, String> params = myValidator.getRequestParams(getAddress());
-
-		assertEquals("Line 1, Line 2", params.get("a1"));
-		assertEquals("City, POSTAL", params.get("a2"));
-		assertEquals("Country", params.get("ctry"));
-		assertEquals("MY_KEY", params.get("id"));
-		assertEquals("json", params.get("format"));
-		assertTrue(params.containsKey("t"));
-	}
-
-	@Test
-	public void testServiceCalled() {
-		Address address = getAddress();
-
-		final RestTemplate template = mock(RestTemplate.class);
-
-		Properties props = new Properties();
-		props.setProperty(BaseRestfulValidator.PROPERTY_SERVICE_KEY, "MY_KEY");
-		MelissaAddressValidator val = new MelissaAddressValidator(props) {
-			@Override
-			protected RestTemplate newTemplate() {
-				return template;
-			}
-		};
-
-		try {
-			val.getResponseEntity(address, ourContext);
-		} catch (Exception e) {
-			fail();
-		}
-
-		verify(template, times(1)).getForEntity(any(String.class), eq(String.class), any(Map.class));
-	}
-
-	private Address getAddress() {
-		Address address = new Address();
-		address.addLine("Line 1").addLine("Line 2").setCity("City").setPostalCode("POSTAL").setCountry("Country");
-		return address;
-	}
-
-	@Test
-	public void testSuccessfulResponses() throws Exception {
-		AddressValidationResult res = myValidator.getValidationResult(new AddressValidationResult(),
-			new ObjectMapper().readTree(RESPONSE_INVALID_ADDRESS), ourContext);
-		assertFalse(res.isValid());
-
-		res = myValidator.getValidationResult(new AddressValidationResult(),
-			new ObjectMapper().readTree(RESPONSE_VALID_ADDRESS), ourContext);
-		assertTrue(res.isValid());
-		assertEquals("100 Main St W;Hamilton ON  L8P 1H6", res.getValidatedAddressString());
-	}
-
-	@Test
-	public void testErrorResponses() throws Exception {
-		assertThrows(AddressValidationException.class, () -> {
-			myValidator.getValidationResult(new AddressValidationResult(),
-				new ObjectMapper().readTree(RESPONSE_INVALID_KEY), ourContext);
-		});
-	}
-
-}

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/interceptor/validation/fields/FieldValidatingInterceptorTest.java

@@ -4,11 +4,15 @@ import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import org.apache.commons.lang3.StringUtils;
 import org.hl7.fhir.r4.model.ContactPoint;
+import org.hl7.fhir.r4.model.Extension;
 import org.hl7.fhir.r4.model.Person;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.Arrays;
+import java.util.HashMap;
 
 import static ca.uhn.fhir.rest.server.interceptor.s13n.StandardizingInterceptor.STANDARDIZATION_DISABLED_HEADER;
 import static ca.uhn.fhir.rest.server.interceptor.validation.fields.FieldValidatingInterceptor.VALIDATION_DISABLED_HEADER;
@@ -19,6 +23,8 @@ import static org.mockito.Mockito.when;
 
 class FieldValidatingInterceptorTest {
 
+	private static final Logger ourLog = LoggerFactory.getLogger(FieldValidatingInterceptorTest.class);
+
 	private FhirContext myFhirContext = FhirContext.forR4();
 	private FieldValidatingInterceptor myInterceptor = new FieldValidatingInterceptor();
 
@@ -33,6 +39,17 @@ class FieldValidatingInterceptorTest {
 		myInterceptor = new FieldValidatingInterceptor();
 	}
 
+	@Test
+	public void testEmptyRequests() {
+		try {
+			myInterceptor.setConfig(new HashMap<>());
+			myInterceptor.resourcePreCreate(null, null);
+			myInterceptor.resourcePreUpdate(null, null, null);
+		} catch (Exception ex) {
+			fail();
+		}
+	}
+
 	@Test
 	public void testDisablingValidationViaHeader() {
 		RequestDetails request = newRequestDetails();
@@ -61,17 +78,28 @@ class FieldValidatingInterceptorTest {
 	public void testInvalidEmailValidation() {
 		Person person = new Person();
 		person.addTelecom().setSystem(ContactPoint.ContactPointSystem.EMAIL).setValue("@garbage");
+		person.addTelecom().setSystem(ContactPoint.ContactPointSystem.EMAIL).setValue("my@email.com");
 
 		try {
 			myInterceptor.handleRequest(newRequestDetails(), person);
-			fail();
 		} catch (Exception e) {
+			fail();
 		}
+
+		ourLog.info("Resource looks like {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(person));
+
+		ContactPoint invalidEmail = person.getTelecomFirstRep();
+		assertTrue(invalidEmail.hasExtension());
+		assertEquals("true", invalidEmail.getExtensionString(IValidator.VALIDATION_EXTENSION_URL));
+
+		ContactPoint validEmail = person.getTelecom().get(1);
+		assertTrue(validEmail.hasExtension());
+		assertEquals("false", validEmail.getExtensionString(IValidator.VALIDATION_EXTENSION_URL));
 	}
 
 	@Test
 	public void testCustomInvalidValidation() {
-		myInterceptor.getConfig().put("telecom.where(system='phone').value", "ClassThatDoesntExist");
+		myInterceptor.getConfig().put("telecom.where(system='phone')", "ClassThatDoesntExist");
 		try {
 			myInterceptor.handleRequest(newRequestDetails(), new Person());
 			fail();
@@ -81,7 +109,7 @@ class FieldValidatingInterceptorTest {
 
 	@Test
 	public void testCustomValidation() {
-		myInterceptor.getConfig().put("telecom.where(system='phone').value", EmptyValidator.class.getName());
+		myInterceptor.getConfig().put("telecom.where(system='phone')", EmptyValidator.class.getName());
 
 		Person person = new Person();
 		person.addTelecom().setSystem(ContactPoint.ContactPointSystem.EMAIL).setValue("email@email.com");
@@ -103,8 +131,8 @@ class FieldValidatingInterceptorTest {
 		person.addTelecom().setSystem(ContactPoint.ContactPointSystem.PHONE).setValue(" ");
 		try {
 			myInterceptor.handleRequest(newRequestDetails(), person);
-			fail();
 		} catch (Exception e) {
+			fail();
 		}
 	}
 

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/TerserUtilTest.java

@@ -2,6 +2,7 @@ package ca.uhn.fhir.util;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
+import org.checkerframework.checker.units.qual.A;
 import org.hl7.fhir.r4.model.Address;
 import org.hl7.fhir.r4.model.DateTimeType;
 import org.hl7.fhir.r4.model.DateType;
@@ -319,12 +320,25 @@ class TerserUtilTest {
 
 	@Test
 	public void testClearFields() {
-		Patient p1 = new Patient();
+		{
-		p1.addName().setFamily("Doe");
+			Patient p1 = new Patient();
+			p1.addName().setFamily("Doe");
 
-		TerserUtil.clearField(ourFhirContext, "name", p1);
+			TerserUtil.clearField(ourFhirContext, "name", p1);
 
-		assertEquals(0, p1.getName().size());
+			assertEquals(0, p1.getName().size());
+		}
+
+		{
+			Address a1 = new Address();
+			a1.addLine("Line 1");
+			a1.addLine("Line 2");
+			a1.setCity("Test");
+			TerserUtil.clearField(ourFhirContext, "line", a1);
+
+			assertEquals(0, a1.getLine().size());
+			assertEquals("Test", a1.getCity());
+		}
 	}
 
 	@Test