Batch validation of codes in value sets

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR2.java

@@ -35,8 +35,10 @@ import java.net.URISyntaxException;
 import java.util.Map;
 
 import org.hl7.fhir.convertors.VersionConvertor_10_50;
+import org.hl7.fhir.convertors.VersionConvertor_40_50;
 import org.hl7.fhir.dstu2.utils.client.FHIRToolingClient;
 import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.r5.model.Bundle;
 import org.hl7.fhir.r5.model.CapabilityStatement;
 import org.hl7.fhir.r5.model.Parameters;
 import org.hl7.fhir.r5.model.TerminologyCapabilities;
@@ -116,6 +118,11 @@ public class TerminologyClientR2 implements TerminologyClient {
   public int getRetryCount() throws FHIRException {
     return client.getRetryCount();
   }
-  
+   
-  
+  @Override
+  public Bundle validateBatch(Bundle batch) {
+    return (Bundle) VersionConvertor_10_50.convertResource(client.transaction((org.hl7.fhir.dstu2.model.Bundle) VersionConvertor_10_50.convertResource(batch)));
+  }
+
+
 }

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR3.java

@@ -35,8 +35,10 @@ import java.net.URISyntaxException;
 import java.util.Map;
 
 import org.hl7.fhir.convertors.VersionConvertor_30_50;
+import org.hl7.fhir.convertors.VersionConvertor_40_50;
 import org.hl7.fhir.dstu3.utils.client.FHIRToolingClient;
 import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.r5.model.Bundle;
 import org.hl7.fhir.r5.model.CapabilityStatement;
 import org.hl7.fhir.r5.model.Parameters;
 import org.hl7.fhir.r5.model.TerminologyCapabilities;
@@ -116,5 +118,11 @@ public class TerminologyClientR3 implements TerminologyClient {
   public int getRetryCount() throws FHIRException {
     return client.getRetryCount();
   }
-  
+ 
+  @Override
+  public Bundle validateBatch(Bundle batch) {
+    return (Bundle) VersionConvertor_30_50.convertResource(client.transaction((org.hl7.fhir.dstu3.model.Bundle) VersionConvertor_30_50.convertResource(batch, false)), false);
+  }
+
+
 }

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR4.java

@@ -38,6 +38,7 @@ import org.hl7.fhir.convertors.VersionConvertor_40_50;
 import org.hl7.fhir.convertors.conv40_50.TerminologyCapabilities40_50;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.r4.utils.client.FHIRToolingClient;
+import org.hl7.fhir.r5.model.Bundle;
 import org.hl7.fhir.r5.model.CapabilityStatement;
 import org.hl7.fhir.r5.model.Parameters;
 import org.hl7.fhir.r5.model.TerminologyCapabilities;
@@ -117,5 +118,10 @@ public class TerminologyClientR4 implements TerminologyClient {
   public int getRetryCount() throws FHIRException {
     return client.getRetryCount();
   }
-  
+ 
+  @Override
+  public Bundle validateBatch(Bundle batch) {
+    return (Bundle) VersionConvertor_40_50.convertResource(client.transaction((org.hl7.fhir.r4.model.Bundle) VersionConvertor_40_50.convertResource(batch)));
+  }
+
 }

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR5.java

@@ -35,6 +35,7 @@ import java.net.URISyntaxException;
 import java.util.Map;
 
 import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.r5.model.Bundle;
 import org.hl7.fhir.r5.model.CapabilityStatement;
 import org.hl7.fhir.r5.model.CodeSystem;
 import org.hl7.fhir.r5.model.Parameters;
@@ -110,4 +111,9 @@ public class TerminologyClientR5 implements TerminologyClient {
     return client.getRetryCount();
   }
 
+  @Override
+  public Bundle validateBatch(Bundle batch) {
+    return client.transaction(batch);
+  }
+
 }

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java

@@ -55,6 +55,7 @@ import org.hl7.fhir.r5.context.CanonicalResourceManager.CanonicalResourceProxy;
 import org.hl7.fhir.r5.context.IWorkerContext.ILoggingService.LogCategory;
 import org.hl7.fhir.r5.context.TerminologyCache.CacheToken;
 import org.hl7.fhir.r5.model.BooleanType;
+import org.hl7.fhir.r5.model.Bundle;
 import org.hl7.fhir.r5.model.CanonicalResource;
 import org.hl7.fhir.r5.model.CapabilityStatement;
 import org.hl7.fhir.r5.model.CodeSystem;
@@ -88,6 +89,8 @@ import org.hl7.fhir.r5.model.StructureMap;
 import org.hl7.fhir.r5.model.TerminologyCapabilities;
 import org.hl7.fhir.r5.model.TerminologyCapabilities.TerminologyCapabilitiesCodeSystemComponent;
 import org.hl7.fhir.r5.model.ValueSet;
+import org.hl7.fhir.r5.model.Bundle.BundleEntryComponent;
+import org.hl7.fhir.r5.model.Bundle.BundleType;
 import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
 import org.hl7.fhir.r5.model.ValueSet.ValueSetComposeComponent;
 import org.hl7.fhir.r5.terminologies.CodeSystemUtilities;
@@ -111,6 +114,8 @@ import org.hl7.fhir.utilities.validation.ValidationOptions.ValueSetMode;
 
 import com.google.gson.JsonObject;
 
+import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum;
+
 public abstract class BaseWorkerContext extends I18nBase implements IWorkerContext{
 
   public class MetadataResourceVersionComparator<T extends CanonicalResource> implements Comparator<T> {
@@ -685,10 +690,99 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
     return validateCode(options.guessSystem(), c, vs);
   }
 
+
+  @Override
+  public void validateCodeBatch(ValidationOptions options, List<? extends CodingValidationRequest> codes, ValueSet vs) {
+    if (options == null) {
+      options = ValidationOptions.defaults();
+    }
+    // 1st pass: what is in the cache? 
+    // 2nd pass: What can we do internally 
+    // 3rd pass: hit the server
+    for (CodingValidationRequest t : codes) {
+      t.setCacheToken(txCache != null ? txCache.generateValidationToken(options, t.getCoding(), vs) : null);
+      if (txCache != null) { 
+        t.setResult(txCache.getValidation(t.getCacheToken()));
+      }
+    }
+    if (options.isUseClient()) {
+      for (CodingValidationRequest t : codes) {
+        if (!t.hasResult()) {
+          try {
+            ValueSetCheckerSimple vsc = new ValueSetCheckerSimple(options, vs, this); 
+            ValidationResult res = vsc.validateCode(t.getCoding());
+            if (txCache != null)
+              txCache.cacheValidation(t.getCacheToken(), res, TerminologyCache.TRANSIENT);
+            t.setResult(res);
+          } catch (Exception e) {
+          }
+        }
+      }      
+    }  
+
+    for (CodingValidationRequest t : codes) {
+      if (!t.hasResult()) {
+        if (!options.isUseServer()) {
+         t.setResult(new ValidationResult(IssueSeverity.WARNING,formatMessage(I18nConstants.UNABLE_TO_VALIDATE_CODE_WITHOUT_USING_SERVER), TerminologyServiceErrorClass.BLOCKED_BY_OPTIONS));
+        } else if (unsupportedCodeSystems.contains(t.getCoding().getSystem())) {
+          t.setResult(new ValidationResult(IssueSeverity.ERROR,formatMessage(I18nConstants.TERMINOLOGY_TX_SYSTEM_NOTKNOWN, t.getCoding().getSystem()), TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED));      
+        } else if (noTerminologyServer) {
+          t.setResult(new ValidationResult(IssueSeverity.ERROR,formatMessage(I18nConstants.ERROR_VALIDATING_CODE_RUNNING_WITHOUT_TERMINOLOGY_SERVICES), TerminologyServiceErrorClass.NOSERVICE));
+        }
+      }
+    }
+    
+    if (expParameters == null)
+      throw new Error(formatMessage(I18nConstants.NO_EXPANSIONPROFILE_PROVIDED));
+    // for those that that failed, we try to validate on the server
+    Bundle batch = new Bundle();
+    batch.setType(BundleType.BATCH);
+    Set<String> systems = new HashSet<>();
+    for (CodingValidationRequest t : codes) {
+      if (!t.hasResult()) {
+        Parameters pIn = new Parameters();
+        pIn.addParameter().setName("coding").setValue(t.getCoding());
+        if (options.isGuessSystem())
+          pIn.addParameter().setName("implySystem").setValue(new BooleanType(true));
+        if (vs != null) {
+          pIn.addParameter().setName("valueSet").setResource(vs);
+        }
+        pIn.addParameter().setName("profile").setResource(expParameters);
+        setTerminologyOptions(options, pIn);
+        BundleEntryComponent be = batch.addEntry();
+        be.setResource(pIn);
+        be.getRequest().setUrl("ValueSet/$validate-code");
+        be.setUserData("source", t);
+        systems.add(t.getCoding().getSystem());
+      }
+    }
+    if (batch.getEntry().size() > 0) {
+      tlog("$batch validate for "+batch.getEntry().size()+" codes on systems "+systems.toString());
+      if (txClient == null) {
+        throw new FHIRException(formatMessage(I18nConstants.ATTEMPT_TO_USE_TERMINOLOGY_SERVER_WHEN_NO_TERMINOLOGY_SERVER_IS_AVAILABLE));
+      }
+      if (txLog != null) {
+        txLog.clearLastId();
+      }
+      Bundle resp = txClient.validateBatch(batch);
+      for (int i = 0; i < batch.getEntry().size(); i++) {
+        CodingValidationRequest t = (CodingValidationRequest) batch.getEntry().get(i).getUserData("source");
+        BundleEntryComponent r = resp.getEntry().get(i);
+        if (r.getResponse().getStatus().startsWith("2")) {
+          t.setResult(new ValidationResult(IssueSeverity.ERROR, getResponseText(r.getResource())).setTxLink(txLog == null ? null : txLog.getLastId()));
+        } else {
+          t.setResult(processValidationResult((Parameters) r.getResource()));
+        }
+      }
+    }    
+  }
+  
+  private String getResponseText(Resource resource) {
+    return "Todo";
+  }
+
   @Override
   public ValidationResult validateCode(ValidationOptions options, Coding code, ValueSet vs) {
-    assert options != null;
-    
     if (options == null) {
       options = ValidationOptions.defaults();
     }
@@ -813,6 +907,10 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
       pOut = txClient.validateCS(pin);
     else
       pOut = txClient.validateVS(pin);
+    return processValidationResult(pOut);
+  }
+
+  public ValidationResult processValidationResult(Parameters pOut) {
     boolean ok = false;
     String message = "No Message returned";
     String display = null;
@@ -1622,4 +1720,5 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
     return txClient;
   }
 
+
 }

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/IWorkerContext.java

@@ -44,6 +44,8 @@ import org.fhir.ucum.UcumService;
 import org.hl7.fhir.exceptions.DefinitionException;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.exceptions.TerminologyServiceException;
+import org.hl7.fhir.r5.context.IWorkerContext.CodingValidationRequest;
+import org.hl7.fhir.r5.context.TerminologyCache.CacheToken;
 import org.hl7.fhir.r5.formats.IParser;
 import org.hl7.fhir.r5.formats.ParserType;
 import org.hl7.fhir.r5.model.Bundle;
@@ -93,6 +95,53 @@ import com.google.gson.JsonSyntaxException;
  */
 public interface IWorkerContext {
 
+  public class CodingValidationRequest {
+    private Coding coding;
+    private ValidationResult result;
+    private CacheToken cacheToken;
+    
+    public CodingValidationRequest(Coding coding) {
+      super();
+      this.coding = coding;
+    }
+
+    public ValidationResult getResult() {
+      return result;
+    }
+
+    public void setResult(ValidationResult result) {
+      this.result = result;
+    }
+
+    public Coding getCoding() {
+      return coding;
+    }
+
+    public boolean hasResult() {
+      return result != null;
+    }
+
+    /**
+     * internal logic; external users of batch validation should ignore this property
+     * 
+     * @return
+     */
+    public CacheToken getCacheToken() {
+      return cacheToken;
+    }
+
+    /**
+     * internal logic; external users of batch validation should ignore this property
+     * 
+     * @param cacheToken
+     */
+    public void setCacheToken(CacheToken cacheToken) {
+      this.cacheToken = cacheToken;
+    }
+    
+    
+  }
+
   public class PackageVersion {
     private String id;
     private String version;
@@ -619,6 +668,8 @@ public interface IWorkerContext {
    * @return
    */
   public ValidationResult validateCode(ValidationOptions options, Coding code, ValueSet vs);
+
+  public void validateCodeBatch(ValidationOptions options, List<? extends CodingValidationRequest> codes, ValueSet vs);
   
   /**
    * returns the recommended tla for the type  (from the structure definitions)

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/type/ValueSetValidator.java

@@ -1,9 +1,11 @@
 package org.hl7.fhir.validation.instance.type;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import org.hl7.fhir.r5.model.ValueSet;
 import org.hl7.fhir.r5.context.IWorkerContext;
+import org.hl7.fhir.r5.context.IWorkerContext.CodingValidationRequest;
 import org.hl7.fhir.r5.context.IWorkerContext.ValidationResult;
 import org.hl7.fhir.r5.elementmodel.Element;
 import org.hl7.fhir.r5.model.CodeSystem;
@@ -19,10 +21,26 @@ import org.hl7.fhir.utilities.validation.ValidationMessage.Source;
 import org.hl7.fhir.utilities.validation.ValidationOptions;
 import org.hl7.fhir.validation.BaseValidator;
 import org.hl7.fhir.validation.TimeTracker;
+import org.hl7.fhir.validation.instance.type.ValueSetValidator.VSCodingValidationRequest;
 import org.hl7.fhir.validation.instance.utils.NodeStack;
 
 public class ValueSetValidator extends BaseValidator {
 
+  public class VSCodingValidationRequest extends CodingValidationRequest {
+
+    private NodeStack stack;
+
+    public VSCodingValidationRequest(NodeStack stack, Coding code) {
+      super(code);
+      this.stack = stack;
+    }
+
+    public NodeStack getStack() {
+      return stack;
+    }
+    
+  }
+
   public ValueSetValidator(IWorkerContext context, TimeTracker timeTracker) {
     super(context);
     source = Source.InstanceValidator;
@@ -58,7 +76,6 @@ public class ValueSetValidator extends BaseValidator {
   private void validateValueSetInclude(List<ValidationMessage> errors, Element include, NodeStack stack) {
     String system = include.getChildValue("system");
     String version = include.getChildValue("version");
-    boolean systemOk = true;
     List<Element> valuesets = include.getChildrenByName("valueSet");
     int i = 0;
     for (Element ve : valuesets) {
@@ -79,13 +96,31 @@ public class ValueSetValidator extends BaseValidator {
     List<Element> concepts = include.getChildrenByName("concept");
     List<Element> filters = include.getChildrenByName("filter");
     if (!Utilities.noString(system)) {
+      boolean systemOk = true;
       int cc = 0;
+      List<VSCodingValidationRequest> batch = new ArrayList<>();
+      boolean first = true;
       for (Element concept : concepts) {
-        if (systemOk && !validateValueSetIncludeConcept(errors, concept, stack.push(concept, cc, null, null), system, version)) {
+        // we treat the first differently because we want to know if tbe system is worth validating. if it is, then we batch the rest
-          systemOk = false;
+        if (first) {
+          systemOk = validateValueSetIncludeConcept(errors, concept, stack.push(concept, cc, null, null), system, version);
+          first = false;
+        } else if (systemOk) {
+          batch.add(prepareValidateValueSetIncludeConcept(errors, concept, stack.push(concept, cc, null, null), system, version));
         }
         cc++;
       }    
+      if (batch.size() > 0) {
+        context.validateCodeBatch(ValidationOptions.defaults(), batch, null);
+        for (VSCodingValidationRequest cv : batch) {
+          if (version == null) {
+            warning(errors, IssueType.BUSINESSRULE, cv.getStack().getLiteralPath(), cv.getResult().isOk(), I18nConstants.VALUESET_INCLUDE_INVALID_CONCEPT_CODE, system, cv.getCoding().getCode());
+          } else {
+            warning(errors, IssueType.BUSINESSRULE, cv.getStack().getLiteralPath(), cv.getResult().isOk(), I18nConstants.VALUESET_INCLUDE_INVALID_CONCEPT_CODE_VER, system, version, cv.getCoding().getCode());
+          }
+        }
+      }
+      
       int cf = 0;
       for (Element filter : filters) {
         if (systemOk && !validateValueSetIncludeFilter(errors, include, stack.push(filter, cf, null, null), system, version)) {
@@ -121,6 +156,15 @@ public class ValueSetValidator extends BaseValidator {
     return true;
   }
 
+  private VSCodingValidationRequest prepareValidateValueSetIncludeConcept(List<ValidationMessage> errors, Element concept, NodeStack stack, String system, String version) {
+    String code = concept.getChildValue("code");
+    Coding c = new Coding(system, code, null);
+    if (version != null) {
+       c.setVersion(version);
+    }
+    return new VSCodingValidationRequest(stack, c);
+  }
+
   private boolean validateValueSetIncludeFilter(List<ValidationMessage> errors, Element include, NodeStack push, String system, String version) {
     return true;
   }