Significant improvement to performance rendering value sets, and only use server side caching if the server declares it supports it

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/VersionConvertor_10_50.java

@@ -3439,6 +3439,8 @@ public class VersionConvertor_10_50 {
         for (ParametersParameterComponent p : src.getParameter()) {
             if (p.getName().equals("system"))
                 res.addCodeSystem().setUri(p.getValue().primitiveValue());
+            if (p.getName().equals("expansion.parameter"))
+              res.getExpansion().addParameter().setName(p.getValue().primitiveValue());
         }
         return res;
     }

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/VersionConvertor_30_50.java

@@ -5539,6 +5539,8 @@ public class VersionConvertor_30_50 {
         for (ParametersParameterComponent p : src.getParameter()) {
             if (p.getName().equals("system"))
                 res.addCodeSystem().setUri(p.getValue().primitiveValue());
+            if (p.getName().equals("expansion.parameter"))
+              res.getExpansion().addParameter().setName(p.getValue().primitiveValue());
         }
         return res;
     }

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java

@@ -90,6 +90,7 @@ import org.hl7.fhir.r5.model.StructureDefinition.TypeDerivationRule;
 import org.hl7.fhir.r5.model.StructureMap;
 import org.hl7.fhir.r5.model.TerminologyCapabilities;
 import org.hl7.fhir.r5.model.TerminologyCapabilities.TerminologyCapabilitiesCodeSystemComponent;
+import org.hl7.fhir.r5.model.TerminologyCapabilities.TerminologyCapabilitiesExpansionParameterComponent;
 import org.hl7.fhir.r5.model.UriType;
 import org.hl7.fhir.r5.model.ValueSet;
 import org.hl7.fhir.r5.model.Bundle.BundleEntryComponent;
@@ -156,6 +157,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
   private Object lock = new Object(); // used as a lock for the data that follows
   protected String version;
   private String cacheId;
+  private boolean isTxCaching;
   private Set<String> cached = new HashSet<>();
   
   private Map<String, Map<String, Resource>> allResourcesById = new HashMap<String, Map<String, Resource>>();
@@ -504,7 +506,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
         if (txcaps == null) {
           try {
             log("Terminology server: Check for supported code systems for "+system);
-            txcaps = txClient.getTerminologyCapabilities();
+            setTxCaps(txClient.getTerminologyCapabilities());
           } catch (Exception e) {
             if (canRunWithoutTerminology) {
               noTerminologyServer = true;
@@ -520,11 +522,6 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
               throw new TerminologyServiceException(e);
             }
           }
-          if (txcaps != null) {
-            for (TerminologyCapabilitiesCodeSystemComponent tccs : txcaps.getCodeSystem()) {
-              supportedCodeSystems.add(tccs.getUri());
-            }
-          }
           if (supportedCodeSystems.contains(system)) {
             return true;
           }
@@ -588,7 +585,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
     Parameters p = expParameters.copy(); 
     p.setParameter("includeDefinition", false);
     p.setParameter("excludeNested", !hierarchical);
-    if (cacheId != null) {
+    if (isTxCaching && cacheId != null) {
       p.addParameter().setName("cache-id").setValue(new StringType(cacheId));              
     }
     addDependentResources(p, vs);
@@ -663,7 +660,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
     }
     
     // if that failed, we try to expand on the server
-    if (cacheId != null) {
+    if (isTxCaching && cacheId != null) {
       p.addParameter().setName("cache-id").setValue(new StringType(cacheId));              
     }
     addDependentResources(p, vs);
@@ -927,11 +924,11 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
   }
 
   private ValidationResult validateOnServer(ValueSet vs, Parameters pin) throws FHIRException {
-    if (cacheId != null) {
+    if (isTxCaching && cacheId != null) {
       pin.addParameter().setName("cache-id").setValue(new StringType(cacheId));              
     }
     if (vs != null) {
-      if (cacheId != null && cached.contains(vs.getUrl()+"|"+vs.getVersion())) {
+      if (isTxCaching && cacheId != null && cached.contains(vs.getUrl()+"|"+vs.getVersion())) {
         pin.addParameter().setName("url").setValue(new UriType(vs.getUrl()+"|"+vs.getVersion()));        
       } else {
         pin.addParameter().setName("valueSet").setResource(vs);
@@ -976,7 +973,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
     for (CanonicalType c : inc.getValueSet()) {
       ValueSet vs = fetchResource(ValueSet.class, c.getValue());
       if (vs != null) {
-        if (cacheId == null || !cached.contains(vs.getVUrl())) {
+        if (isTxCaching && cacheId == null || !cached.contains(vs.getVUrl())) {
           pin.addParameter().setName("tx-resource").setResource(vs);
           cached.add(vs.getVUrl());
         }
@@ -985,7 +982,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
     }
     CodeSystem cs = fetchResource(CodeSystem.class, inc.getSystem());
     if (cs != null) {
-      if (cacheId == null || !cached.contains(cs.getVUrl())) {
+      if (isTxCaching && cacheId == null || !cached.contains(cs.getVUrl())) {
         pin.addParameter().setName("tx-resource").setResource(cs);
         cached.add(cs.getVUrl());
       }
@@ -1859,4 +1856,23 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
     this.cacheId = cacheId;
   }
 
+  public TerminologyCapabilities getTxCaps() {
+    return txcaps;
+  }
+
+  public void setTxCaps(TerminologyCapabilities txCaps) {
+    this.txcaps = txCaps;
+    if (txCaps != null) {
+      for (TerminologyCapabilitiesExpansionParameterComponent t : txcaps.getExpansion().getParameter()) {
+        if ("cache-id".equals(t.getName())) {
+          isTxCaching = true;
+        }
+      }
+      for (TerminologyCapabilitiesCodeSystemComponent tccs : txcaps.getCodeSystem()) {
+        supportedCodeSystems.add(tccs.getUri());
+      }
+    }
+  }
+
+  
 }

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java

@@ -63,6 +63,7 @@ import org.hl7.fhir.r5.formats.XmlParser;
 import org.hl7.fhir.r5.model.Bundle;
 import org.hl7.fhir.r5.model.Bundle.BundleEntryComponent;
 import org.hl7.fhir.r5.model.CanonicalResource;
+import org.hl7.fhir.r5.model.CapabilityStatement;
 import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionBindingComponent;
 import org.hl7.fhir.r5.model.Questionnaire;
 import org.hl7.fhir.r5.model.Resource;
@@ -293,7 +294,9 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
         txLog = new HTMLClientLogger(log);
       }
       txClient.setLogger(txLog);
-      return txClient.getCapabilitiesStatementQuick().getSoftware().getVersion();
+      CapabilityStatement cps = txClient.getCapabilitiesStatementQuick();
+      setTxCaps(txClient.getTerminologyCapabilities());
+      return cps.getSoftware().getVersion();
     } catch (Exception e) {
       throw new FHIRException(formatMessage(I18nConstants.UNABLE_TO_CONNECT_TO_TERMINOLOGY_SERVER_USE_PARAMETER_TX_NA_TUN_RUN_WITHOUT_USING_TERMINOLOGY_SERVICES_TO_VALIDATE_LOINC_SNOMED_ICDX_ETC_ERROR__, e.getMessage()), e);
     }

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/renderers/ValueSetRenderer.java

@@ -6,17 +6,23 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import org.hl7.fhir.exceptions.DefinitionException;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.exceptions.FHIRFormatError;
 import org.hl7.fhir.exceptions.TerminologyServiceException;
+import org.hl7.fhir.r5.context.IWorkerContext.CodingValidationRequest;
 import org.hl7.fhir.r5.context.IWorkerContext.ValidationResult;
 import org.hl7.fhir.r5.model.BooleanType;
 import org.hl7.fhir.r5.model.CanonicalResource;
 import org.hl7.fhir.r5.model.CodeSystem;
 import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
+import org.hl7.fhir.r5.model.Coding;
 import org.hl7.fhir.r5.model.ConceptMap;
 import org.hl7.fhir.r5.model.DataType;
 import org.hl7.fhir.r5.model.DomainResource;
@@ -182,7 +188,7 @@ public class ValueSetRenderer extends TerminologyRenderer {
         p.addText(" and may be missing codes, or include codes that are not valid");
       }
     }
-
+    
     generateVersionNotice(x, vs.getExpansion());
 
     CodeSystem allCS = null;
@@ -737,6 +743,9 @@ public class ValueSetRenderer extends TerminologyRenderer {
             li.code(inc.getVersion()); 
           }
 
+          // for performance reasons, we do all the fetching in one batch
+          Map<String, ConceptDefinitionComponent> definitions = getConceptsForCodes(e, inc);
+          
           XhtmlNode t = li.table("none");
           boolean hasComments = false;
           boolean hasDefinition = false;
@@ -750,7 +759,7 @@ public class ValueSetRenderer extends TerminologyRenderer {
           for (ConceptReferenceComponent c : inc.getConcept()) {
             XhtmlNode tr = t.tr();
             XhtmlNode td = tr.td();
-            ConceptDefinitionComponent cc = getConceptForCode(e, c.getCode(), inc);
+            ConceptDefinitionComponent cc = definitions.get(c.getCode()); 
             addCodeToTable(false, inc.getSystem(), c.getCode(), c.hasDisplay()? c.getDisplay() : cc != null ? cc.getDisplay() : "", td);
 
             td = tr.td();
@@ -844,24 +853,13 @@ public class ValueSetRenderer extends TerminologyRenderer {
   }
 
 
-  private ConceptDefinitionComponent getConceptForCode(CodeSystem e, String code, ConceptSetComponent inc) {
+  private Map<String, ConceptDefinitionComponent> getConceptsForCodes(CodeSystem e, ConceptSetComponent inc) {
-    if (code == null) {
+    if (e == null) {
-      return null;
+      e = getContext().getWorker().fetchCodeSystem(inc.getSystem());
     }
-    // first, look in the code systems
-    if (e == null)
-    e = getContext().getWorker().fetchCodeSystem(inc.getSystem());
-    if (e != null) {
-      ConceptDefinitionComponent v = getConceptForCode(e.getConcept(), code);
-      if (v != null)
-        return v;
-    }
-
-    if (context.isNoSlowLookup())
-      return null;
     
-    if (!getContext().getWorker().hasCache()) {
+    ValueSetExpansionComponent vse = null;
-      ValueSetExpansionComponent vse;
+    if (!context.isNoSlowLookup() && !getContext().getWorker().hasCache()) {
       try {
         ValueSetExpansionOutcome vso = getContext().getWorker().expandVS(inc, false);   
         ValueSet valueset = vso.getValueset();
@@ -872,16 +870,39 @@ public class ValueSetRenderer extends TerminologyRenderer {
       } catch (TerminologyServiceException e1) {
         return null;
       }
-      if (vse != null) {
-        ConceptDefinitionComponent v = getConceptForCodeFromExpansion(vse.getContains(), code);
-      if (v != null)
-        return v;
     }
+    
+    Map<String, ConceptDefinitionComponent> results = new HashMap<>();
+    List<CodingValidationRequest> serverList = new ArrayList<>();
+    
+    // 1st pass, anything we can resolve internally
+    for (ConceptReferenceComponent cc : inc.getConcept()) {
+      String code = cc.getCode();
+      ConceptDefinitionComponent v = null;
+      if (e != null) {
+        v = getConceptForCode(e.getConcept(), code);
+      }
+      if (v == null && vse != null) {
+        v = getConceptForCodeFromExpansion(vse.getContains(), code);
+      }
+      if (v != null) {
+        results.put(code, v);
+      } else {
+        serverList.add(new CodingValidationRequest(new Coding(inc.getSystem(), code, null)));
+      }
     }
-
+    if (!context.isNoSlowLookup() && !serverList.isEmpty()) {
-    return getContext().getWorker().validateCode(getContext().getTerminologyServiceOptions(), inc.getSystem(), code, null).asConceptDefinition();
+      getContext().getWorker().validateCodeBatch(getContext().getTerminologyServiceOptions(), serverList, null);
+      for (CodingValidationRequest vr : serverList) {
+        ConceptDefinitionComponent v = vr.getResult().asConceptDefinition();
+        if (v != null) {
+          results.put(vr.getCoding().getCode(), v);
+        }
+      }
+    }
+    return results;
   }
-
+  
   private ConceptDefinitionComponent getConceptForCode(List<ConceptDefinitionComponent> list, String code) {
     for (ConceptDefinitionComponent c : list) {
     if (code.equals(c.getCode()))