NPM rework

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java

@@ -1,14 +1,19 @@
 package org.hl7.fhir.utilities.cache;
 
 import org.apache.commons.lang3.Validate;
+import org.hl7.fhir.utilities.Utilities;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 
 public abstract class BasePackageCacheManager implements IPackageCacheManager {
-
+  private static final Logger ourLog = LoggerFactory.getLogger(BasePackageCacheManager.class);
   private List<String> myPackageServers = new ArrayList<>();
 
   /**
@@ -35,16 +40,37 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager {
 
   /**
    * Load the latest version of the identified package from the cache - it it exists
-   *
-   * @param id
-   * @return
-   * @throws IOException
    */
   public NpmPackage loadPackageFromCacheOnly(String id) throws IOException {
     return loadPackageFromCacheOnly(id, null);
   }
 
-  protected abstract NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException;
+  /**
+   * Try to load a package from all registered package servers, and return <code>null</code>
+   * if it can not be found at any of them.
+   */
+  @Nullable
+  protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
+
+    for (String nextPackageServer : getPackageServers()) {
+      CachingPackageClient packageClient = new CachingPackageClient(nextPackageServer);
+      try {
+        if (Utilities.noString(version)) {
+          version = packageClient.getLatestVersion(id);
+        }
+        InputStream stream = packageClient.fetch(id, version);
+        String url = packageClient.url(id, version);
+        return new InputStreamWithSrc(stream, packageClient.url(id, version), version);
+      } catch (IOException e) {
+        ourLog.info("Failed to resolve package {}#{} from server: {}", id, version, nextPackageServer);
+      }
+
+    }
+
+    return null;
+  }
+
+  protected abstract NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException;
 
   @Override
   public String getPackageUrl(String packageId) throws IOException {
@@ -66,8 +92,8 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager {
 
 
   private String getPackageUrl(String packageId, String server) throws IOException {
-    PackageClient pc = new PackageClient(server);
+    CachingPackageClient pc = new CachingPackageClient(server);
-    List<PackageClient.PackageInfo> res = pc.search(packageId, null, null, false);
+    List<CachingPackageClient.PackageInfo> res = pc.search(packageId, null, null, false);
     if (res.size() == 0) {
       return null;
     } else {
@@ -91,13 +117,13 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager {
   }
 
   private String getPackageId(String canonical, String server) throws IOException {
-    PackageClient pc = new PackageClient(server);
+    CachingPackageClient pc = new CachingPackageClient(server);
-    List<PackageClient.PackageInfo> res = pc.search(null, canonical, null, false);
+    List<CachingPackageClient.PackageInfo> res = pc.search(null, canonical, null, false);
     if (res.size() == 0) {
       return null;
     } else {
       // this is driven by HL7 Australia (http://hl7.org.au/fhir/ is the canonical url for the base package, and the root for all the others)
-      for (PackageClient.PackageInfo pi : res) {
+      for (CachingPackageClient.PackageInfo pi : res) {
         if (canonical.equals(pi.getCanonical())) {
           return pi.getId();
         }
@@ -106,5 +132,17 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager {
     }
   }
 
+  public class InputStreamWithSrc {
+
+    public InputStream stream;
+    public String url;
+    public String version;
+
+    public InputStreamWithSrc(InputStream stream, String url, String version) {
+      this.stream = stream;
+      this.url = url;
+      this.version = version;
+    }
+  }
 
 }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageClient.java

@@ -1,5 +1,14 @@
 package org.hl7.fhir.utilities.cache;
 
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
+import org.hl7.fhir.utilities.TextFile;
+import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.VersionUtilities;
+import org.hl7.fhir.utilities.json.JSONUtil;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -13,19 +22,7 @@ import java.util.Date;
 import java.util.List;
 import java.util.Set;
 
-import org.hl7.fhir.exceptions.FHIRException;
+public abstract class BasePackageClient {
-import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
-import org.hl7.fhir.utilities.TextFile;
-import org.hl7.fhir.utilities.Utilities;
-import org.hl7.fhir.utilities.VersionUtilities;
-import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
-import org.hl7.fhir.utilities.json.JSONUtil;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-
-public class PackageClient {
 
   public class PackageInfo {
     private String id;
@@ -34,7 +31,7 @@ public class PackageClient {
     private String description;
     private String url;
     private String canonical;
-    
+
     public PackageInfo(String id, String version, String fhirVersion, String description, String url, String canonical) {
       super();
       this.id = id;
@@ -62,21 +59,21 @@ public class PackageClient {
     public String getUrl() {
       return url;
     }
-    
+
     public String getCanonical() {
       return canonical;
     }
     @Override
     public String toString() {
       return id+"#"+(version == null ? "?pc-pi?" : version)+(fhirVersion == null ? "": " ("+canonical+") for FHIR "+fhirVersion)+(url == null ? "" : " @"+url)+(description == null ? "" : " '"+description+"'");
-    }    
+    }
   }
- 
+
   private String address;
   private String cacheFolder;
 
 
-  public PackageClient(String address) {
+  public BasePackageClient(String address) {
     super();
     this.address = address;
     try {
@@ -109,15 +106,10 @@ public class PackageClient {
   }
 
   public InputStream fetchCached(String url) throws IOException, FileNotFoundException {
-    File cacheFile = new File(Utilities.path(cacheFolder, fn(url)));
+    return fetchUrl(url, null);
-    if (cacheFile.exists()) {
-      return new FileInputStream(cacheFile);
-    }
-    TextFile.bytesToFile(TextFile.streamToBytes(fetchUrl(url, null)), cacheFile);
-    return new FileInputStream(cacheFile);
   }
 
-  private String fn(String url) {
+  protected String fn(String url) {
     String[] p = url.split("\\/");
     return p[2]+"-"+p[p.length-2]+"-"+p[p.length-1]+".tgz";
   }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java

@@ -0,0 +1,71 @@
+package org.hl7.fhir.utilities.cache;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+
+import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
+import org.hl7.fhir.utilities.TextFile;
+import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.VersionUtilities;
+import org.hl7.fhir.utilities.json.JSONUtil;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+/**
+ * Implementation of a package client that keeps a local disk cache of downloaded artifacts
+ * in order to avoid re-downloading things
+ */
+public class CachingPackageClient extends BasePackageClient {
+
+  private String cacheFolder;
+
+
+  public CachingPackageClient(String address) {
+    super(address);
+    try {
+      cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "package-client");
+      Utilities.createDirectory(cacheFolder);
+    } catch (IOException e) {
+    }
+  }
+
+  public boolean exists(String id, String ver) throws IOException {
+    List<PackageInfo> vl = getVersions(id);
+    for (PackageInfo pi : vl) {
+      if (ver.equals(pi.getVersion())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public InputStream fetchCached(String url) throws IOException {
+    File cacheFile = new File(Utilities.path(cacheFolder, fn(url)));
+    if (cacheFile.exists()) {
+      return new FileInputStream(cacheFile);
+    }
+    InputStream fetchedPackage = super.fetchCached(url);
+    TextFile.bytesToFile(TextFile.streamToBytes(fetchedPackage), cacheFile);
+    return new FileInputStream(cacheFile);
+  }
+
+
+  public Date getNewPackages(Date lastCalled, List<PackageInfo> updates) {
+    return null;
+  }
+
+
+
+}

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java

@@ -40,7 +40,6 @@ import org.hl7.fhir.utilities.IniFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder;
-import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
 import org.hl7.fhir.utilities.json.JSONUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -67,7 +66,8 @@ import java.util.Map;
 import java.util.Map.Entry;
 
 /**
- * Package cache manager
+ * This is a package cache manager implementation that uses a local disk cache
+ *
  * <p>
  * API:
  * <p>
@@ -200,30 +200,19 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
 
 
   private void listSpecs(Map<String, String> specList, String server) throws IOException {
-    PackageClient pc = new PackageClient(server);
+    CachingPackageClient pc = new CachingPackageClient(server);
-    List<PackageInfo> matches = pc.search(null, null, null, false);
+    List<BasePackageClient.PackageInfo> matches = pc.search(null, null, null, false);
-    for (PackageInfo m : matches) {
+    for (BasePackageClient.PackageInfo m : matches) {
       if (!specList.containsKey(m.getId())) {
         specList.put(m.getId(), m.getUrl());
       }
     }
   }
 
-  private InputStreamWithSrc loadFromPackageServer(String id, String version) {
+  protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
-
+    InputStreamWithSrc retVal = super.loadFromPackageServer(id, version);
-    for (String nextPackageServer : getPackageServers()) {
+    if (retVal != null) {
-      PackageClient packageClient = new PackageClient(nextPackageServer);
+      return retVal;
-      try {
-        if (Utilities.noString(version)) {
-          version = packageClient.getLatestVersion(id);
-        }
-        InputStream stream = packageClient.fetch(id, version);
-        String url = packageClient.url(id, version);
-        return new InputStreamWithSrc(stream, packageClient.url(id, version), version);
-      } catch (IOException e) {
-        ourLog.info("Failed to resolve package {}#{} from server: {}", id, version, nextPackageServer);
-      }
-
     }
 
     // ok, well, we'll try the old way
@@ -232,7 +221,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
 
   public String getLatestVersion(String id) throws IOException {
     for (String nextPackageServer : getPackageServers()) {
-      PackageClient pc = new PackageClient(nextPackageServer);
+      CachingPackageClient pc = new CachingPackageClient(nextPackageServer);
       try {
         return pc.getLatestVersion(id);
       } catch (IOException e) {
@@ -443,6 +432,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
     if (!Utilities.noString(version) && version.startsWith("file:")) {
       return loadPackageFromFile(id, version.substring(5));
     }
+
     NpmPackage p = loadPackageFromCacheOnly(id, version);
     if (p != null) {
       if ("current".equals(version)) {
@@ -461,7 +451,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
     }
 
     // nup, don't have it locally (or it's expired)
-    InputStreamWithSrc source;
+    FilesystemPackageCacheManager.InputStreamWithSrc source;
     if ("current".equals(version)) {
       // special case - fetch from ci-build server
       source = loadFromCIBuild(id);
@@ -471,6 +461,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
     return addPackageToCache(id, version == null ? source.version : version, source.stream, source.url);
   }
 
+
   private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
     try {
       URL url = new URL(source);
@@ -732,19 +723,6 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
     InputStream resolvePackage(String packageId, String version);
   }
 
-  public class InputStreamWithSrc {
-
-    public InputStream stream;
-    public String url;
-    public String version;
-
-    public InputStreamWithSrc(InputStream stream, String url, String version) {
-      this.stream = stream;
-      this.url = url;
-      this.version = version;
-    }
-  }
-
   public class BuildRecordSorter implements Comparator<BuildRecord> {
 
     @Override

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java

@@ -7,7 +7,6 @@ import java.io.InputStream;
 
 public interface IPackageCacheManager {
 
-
   String getPackageId(String canonicalUrl) throws IOException;
 
   NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException;

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java

@@ -102,7 +102,7 @@ public class NpmPackage {
   public class NpmPackageFolder {
     private String name;
     private Map<String, List<String>> types = new HashMap<>();
-    private Map<String, byte[]> content = new HashMap<String, byte[]>(); 
+    private Map<String, byte[]> content = new HashMap<>();
     private JsonObject index;
     private File folder;
 
@@ -111,6 +111,10 @@ public class NpmPackage {
       this.name = name;
     }
 
+    public Map<String, List<String>> getTypes() {
+      return types;
+    }
+
     public String getName() {
       return name;
     }
@@ -192,15 +196,12 @@ public class NpmPackage {
   private Map<String, NpmPackageFolder> folders = new HashMap<>();
   private boolean changedByLoader; // internal qa only!
 
+  /**
+   * Constructor
+   */
   private NpmPackage() {
-
+    super();
   }
-  //  public NpmPackage(JsonObject npm, Map<String, byte[]> content, List<String> folders) {
-  //    this.path = null;
-  //    this.content = content;
-  //    this.npm = npm;
-  //    this.folders = folders;
-  //  }
 
   public static NpmPackage fromFolder(String path) throws IOException {
     NpmPackage res = new NpmPackage();
@@ -839,7 +840,9 @@ public class NpmPackage {
     stream.write(b);
   }
 
-
+  /**
+   * Keys are resource type names, values are filenames
+   */
   public Map<String, List<String>> getTypes() {
     return folders.get("package").types;
   }

org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java

@@ -1,18 +1,18 @@
 package org.hl7.fhir.utilities.tests;
 
-import org.hl7.fhir.utilities.cache.PackageClient;
+import org.hl7.fhir.utilities.cache.CachingPackageClient;
-import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
+import org.hl7.fhir.utilities.cache.CachingPackageClient.PackageInfo;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
 import java.util.List;
 
-public class PackageClientTests {
+public class CachingPackageClientTests {
 
   @Test
   public void testExists() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     Assertions.assertTrue(client.exists("hl7.fhir.r4.core", "4.0.1"));
     Assertions.assertTrue(!client.exists("hl7.fhir.r4.core", "1.0.2"));
     Assertions.assertTrue(!client.exists("hl7.fhir.nothing", "1.0.1"));
@@ -20,7 +20,7 @@ public class PackageClientTests {
 
   @Test
   public void testSearch() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     List<PackageInfo> matches = client.search("core", null, null, false);
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -30,14 +30,14 @@ public class PackageClientTests {
 
   @Test
   public void testSearchNoMatches() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     List<PackageInfo> matches = client.search("corezxxx", null, null, false);
     Assertions.assertTrue(matches.size() == 0);
   }
 
   @Test
   public void testVersions() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3");
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -47,14 +47,14 @@ public class PackageClientTests {
 
   @Test
   public void testVersionsNone() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3X");
     Assertions.assertTrue(matches.size() == 0);
   }
 
   @Test
   public void testExists2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     Assertions.assertTrue(client.exists("hl7.fhir.r4.core", "4.0.1"));
     Assertions.assertTrue(!client.exists("hl7.fhir.r4.core", "1.0.2"));
     Assertions.assertTrue(!client.exists("hl7.fhir.nothing", "1.0.1"));
@@ -62,7 +62,7 @@ public class PackageClientTests {
 
   @Test
   public void testSearch2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.search("core", null, null, false);
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -72,14 +72,14 @@ public class PackageClientTests {
 
   @Test
   public void testSearchNoMatches2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.search("corezxxx", null, null, false);
     Assertions.assertTrue(matches.size() == 0);
   }
 
   @Test
   public void testVersions2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3");
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -89,7 +89,7 @@ public class PackageClientTests {
 
   @Test
   public void testVersions2A() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.getVersions("hl7.fhir.us.core");
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -99,7 +99,7 @@ public class PackageClientTests {
 
   @Test
   public void testVersionsNone2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3X");
     Assertions.assertTrue(matches.size() == 0);
   }

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java

@@ -9,8 +9,8 @@ import org.hl7.fhir.exceptions.FHIRFormatError;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.cache.NpmPackage;
 import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
-import org.hl7.fhir.utilities.cache.PackageClient;
+import org.hl7.fhir.utilities.cache.CachingPackageClient;
-import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
+import org.hl7.fhir.utilities.cache.CachingPackageClient.PackageInfo;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 
 public class PackageValidator {
@@ -22,7 +22,7 @@ public class PackageValidator {
   private void execute() throws IOException {
     FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     
-    PackageClient pc = new PackageClient(FilesystemPackageCacheManager.PRIMARY_SERVER);
+    CachingPackageClient pc = new CachingPackageClient(FilesystemPackageCacheManager.PRIMARY_SERVER);
     for (PackageInfo t : pc.search(null, null, null, false)) {
       System.out.println("Check Package "+t.getId());
       List<PackageInfo> vl = pc.getVersions(t.getId());