From 62a082dbf358100b1f88c13109497b3cc5d03121 Mon Sep 17 00:00:00 2001
From: Grahame Grieve <grahameg@gmail.ccom>
Date: Mon, 13 Jun 2022 21:50:11 +0300
Subject: [PATCH] Add value set expand by URL and use it in vsac

---
 .../fhir/convertors/misc/VSACImporter.java    | 27 +++++++++++++++++--
 .../r4/utils/client/FHIRToolingClient.java    | 23 +++++++++++++++-
 2 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/VSACImporter.java b/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/VSACImporter.java
index 854340d9b..c87c25e0c 100644
--- a/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/VSACImporter.java
+++ b/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/VSACImporter.java
@@ -1,7 +1,11 @@
 package org.hl7.fhir.convertors.misc;
 
 import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.r4.formats.IParser.OutputStyle;
 import org.hl7.fhir.r4.formats.JsonParser;
+import org.hl7.fhir.r4.model.OperationOutcome;
+import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity;
+import org.hl7.fhir.r4.model.OperationOutcome.IssueType;
 import org.hl7.fhir.r4.model.ValueSet;
 import org.hl7.fhir.r4.utils.client.FHIRToolingClient;
 import org.hl7.fhir.utilities.CSVReader;
@@ -12,6 +16,8 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.text.ParseException;
+import java.util.HashMap;
+import java.util.Map;
 
 public class VSACImporter extends OIDBasedValueSetImporter {
 
@@ -28,25 +34,42 @@ public class VSACImporter extends OIDBasedValueSetImporter {
   private void process(String source, String dest, String apiKey) throws FHIRException, IOException, URISyntaxException {
     CSVReader csv = new CSVReader(new FileInputStream(source));
     csv.readHeaders();
+    Map<String, String> errs = new HashMap<>();
 
     FHIRToolingClient fhirToolingClient = new FHIRToolingClient("https://cts.nlm.nih.gov/fhir", "fhir/vsac");
     fhirToolingClient.setUsername("apikey");
     fhirToolingClient.setPassword(apiKey);
+    fhirToolingClient.setTimeout(30000);
 
     int i = 0;
+    int j = 0;
     while (csv.line()) {
       String oid = csv.cell("OID");
       try {
         ValueSet vs = fhirToolingClient.read(ValueSet.class, oid);
-        new JsonParser().compose(new FileOutputStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs);
+        try {
+          ValueSet vse = fhirToolingClient.expandValueset(vs.getUrl(), null);
+          vs.setExpansion(vse.getExpansion());
+          j++;
+        } catch (Exception e) {
+          errs.put(oid, "Expansion: " +e.getMessage());
+          System.out.println(e.getMessage());
+        }
+        new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs);
         i++;
         if (i % 100 == 0) {
-          System.out.println(i);
+          System.out.println(":"+i+" ("+j+")");
         }
       } catch (Exception e) {
         System.out.println("Unable to fetch OID " + oid + ": " + e.getMessage());
+        errs.put(oid, e.getMessage());
       }
     }
+    OperationOutcome oo = new OperationOutcome();
+    for (String oid : errs.keySet()) {
+      oo.addIssue().setSeverity(IssueSeverity.ERROR).setCode(IssueType.EXCEPTION).setDiagnostics(errs.get(oid)).addLocation(oid);
+    }
+    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "other", "OperationOutcome-vsac-errors.json")), oo);
     System.out.println("Done. " + i + " ValueSets");
   }
 }
diff --git a/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/utils/client/FHIRToolingClient.java b/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/utils/client/FHIRToolingClient.java
index 0df3ea66f..8894b0b28 100644
--- a/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/utils/client/FHIRToolingClient.java
+++ b/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/utils/client/FHIRToolingClient.java
@@ -1,6 +1,7 @@
 package org.hl7.fhir.r4.utils.client;
 
 import okhttp3.Headers;
+import okhttp3.Request;
 import okhttp3.internal.http2.Header;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.r4.model.*;
@@ -363,6 +364,27 @@ public class FHIRToolingClient {
     return feed;
   }
 
+  public ValueSet expandValueset(String vsUrl, Parameters expParams) {
+    Map<String,String> parameters = new HashMap<>();
+    parameters.put("url", vsUrl);
+  
+    org.hl7.fhir.r4.utils.client.network.ResourceRequest<Resource> result = null;
+    try {
+      result = client.issueGetResourceRequest(resourceAddress.resolveOperationUri(ValueSet.class, "expand", parameters),
+        getPreferredResourceFormat(),
+        generateHeaders(),
+        "ValueSet/$expand?url=" + vsUrl,
+        TIMEOUT_OPERATION_EXPAND);
+      if (result.isUnsuccessfulRequest()) {
+        throw new EFhirClientException("Server returned error code " + result.getHttpStatus(), (OperationOutcome) result.getPayload());
+      }
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+    return result == null ? null : (ValueSet) result.getPayload();
+  }
+
+
   public ValueSet expandValueset(ValueSet source, Parameters expParams) {
     Parameters p = expParams == null ? new Parameters() : expParams.copy();
     p.addParameter().setName("valueSet").setResource(source);
@@ -383,7 +405,6 @@ public class FHIRToolingClient {
     return result == null ? null : (ValueSet) result.getPayload();
   }
 
-
   public Parameters lookupCode(Map<String, String> params) {
     org.hl7.fhir.r4.utils.client.network.ResourceRequest<Resource> result = null;
     try {