Merge remote-tracking branch 'upstream/master'

module-snapshot-publishing.yml

@@ -21,7 +21,9 @@ strategy:
       module: "org.hl7.fhir.r5"
     validator:
       module: "org.hl7.fhir.validation"
-  maxParallel: 3
+    validator_cli:
+      module: "org.hl7.fhir.validation.cli"
+  maxParallel: 8
 
 pool:
   vmImage: "ubuntu-16.04"
@@ -105,6 +107,15 @@ steps:
     displayName: 'Deploy $(module) to Sonatype staging'
     inputs:
       mavenPomFile: '$(System.DefaultWorkingDirectory)/$(module)/pom.xml'
-      goals: deploy
+      goals: clean deploy
-      options: '--settings $(System.DefaultWorkingDirectory)/settings.xml '
+      options: '--settings $(System.DefaultWorkingDirectory)/settings.xml'
+      publishJUnitResults: false
+
+  # Clean any existing deployment artifacts
+  - task: Maven@3
+    displayName: 'Clean any existing deploy artifacts.'
+    inputs:
+      mavenPomFile: '$(System.DefaultWorkingDirectory)/$(module)/pom.xml'
+      goals: clean
+      options: '--settings $(System.DefaultWorkingDirectory)/settings.xml'
       publishJUnitResults: false

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/ExtensionDefinitionGenerator.java

@@ -62,7 +62,7 @@ import org.hl7.fhir.r4.utils.NPMPackageGenerator;
 import org.hl7.fhir.r4.utils.NPMPackageGenerator.Category;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.cache.NpmPackage;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.PackageGenerator.PackageType;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 
@@ -392,7 +392,7 @@ public class ExtensionDefinitionGenerator {
 
   private List<StructureDefinition> loadSource() throws IOException, FHIRException {
     List<StructureDefinition> list = new ArrayList<>();
-    PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     NpmPackage npm = pcm.loadPackage("hl7.fhir.core", sourceVersion.toCode());
     if (sourceVersion == FHIRVersion._4_0_0)
       context = SimpleWorkerContext.fromPackage(npm);

org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/test/utils/TestingUtilities.java

@@ -44,15 +44,13 @@ import javax.xml.parsers.DocumentBuilderFactory;
 
 import org.apache.commons.codec.binary.Base64;
 import org.fhir.ucum.UcumEssenceService;
-import org.fhir.ucum.UcumException;
-import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.r4.context.IWorkerContext;
 import org.hl7.fhir.r4.context.SimpleWorkerContext;
 import org.hl7.fhir.r4.model.Parameters;
 import org.hl7.fhir.utilities.CSFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -73,9 +71,9 @@ public class TestingUtilities {
 	
 	public static IWorkerContext context() {
 	  if (fcontext == null) {
-	    PackageCacheManager pcm;
+	    FilesystemPackageCacheManager pcm;
 	    try {
-	      pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+	      pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
 	      fcontext = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.core", "4.0.0"));
 	      fcontext.setUcumService(new UcumEssenceService(TestingUtilities.resourceNameToFile("ucum", "ucum-essence.xml")));
 	      fcontext.setExpansionProfile(new Parameters());

org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/CDARoundTripTests.java

@@ -8,7 +8,7 @@ import org.hl7.fhir.r4.elementmodel.Element;
 import org.hl7.fhir.r4.elementmodel.Manager;
 import org.hl7.fhir.r4.elementmodel.Manager.FhirFormat;
 import org.hl7.fhir.r4.formats.IParser.OutputStyle;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
@@ -29,7 +29,7 @@ public class CDARoundTripTests {
   @BeforeAll
   public void setUp() throws Exception {
     context = new SimpleWorkerContext();
-    PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     context.loadFromPackage(pcm.loadPackage("hl7.fhir.core", "current"), null, "StructureDefinition");
     context.loadFromPackage(pcm.loadPackage("hl7.fhir.cda", "current"), null, "StructureDefinition");
   }

org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/FHIRMappingLanguageTests.java

@@ -14,7 +14,7 @@ import org.hl7.fhir.r4.utils.StructureMapUtilities;
 import org.hl7.fhir.r4.utils.StructureMapUtilities.ITransformerServices;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.hl7.fhir.utilities.xml.XMLUtil;
 import org.junit.jupiter.api.BeforeAll;
@@ -61,7 +61,7 @@ public class FHIRMappingLanguageTests implements ITransformerServices {
   @BeforeAll
   static public void setUp() throws Exception {
     if (context == null) {
-      PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+      FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
       context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.core", "4.0.0"));
       jsonParser = new JsonParser();
       jsonParser.setOutputStyle(OutputStyle.PRETTY);

org.hl7.fhir.r5/pom.xml

@@ -13,12 +13,13 @@
 	<packaging>bundle</packaging>
 
 	<dependencies>
-
 		<!-- HAPI Dependencies -->
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>org.hl7.fhir.utilities</artifactId>
+			<version>${project.version}</version>
 		</dependency>
+
 		<dependency>
 		 <groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-base</artifactId>

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/test/utils/TestingUtilities.java

@@ -18,12 +18,11 @@ import org.fhir.ucum.UcumEssenceService;
 import org.hl7.fhir.r5.context.IWorkerContext;
 import org.hl7.fhir.r5.context.SimpleWorkerContext;
 import org.hl7.fhir.r5.model.Parameters;
-import org.hl7.fhir.r5.terminologies.TerminologyClientR5;
 import org.hl7.fhir.utilities.CSFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.hl7.fhir.utilities.tests.BaseTestingUtilities;
 import org.w3c.dom.Document;
@@ -83,9 +82,9 @@ public class TestingUtilities extends BaseTestingUtilities {
       fcontexts = new HashMap<>();
     }
     if (!fcontexts.containsKey(v)) {
-      PackageCacheManager pcm;
+      FilesystemPackageCacheManager pcm;
       try {
-        pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+        pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
         IWorkerContext fcontext = SimpleWorkerContext.fromPackage(pcm.loadPackage(VersionUtilities.packageForVersion(version), version));
         fcontext.setUcumService(new UcumEssenceService(TestingUtilities.loadTestResourceStream("ucum", "ucum-essence.xml")));
         fcontext.setExpansionProfile(new Parameters());

org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/CDARoundTripTests.java

@@ -9,7 +9,7 @@ import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat;
 import org.hl7.fhir.r5.model.StructureDefinition;
 import org.hl7.fhir.r5.test.utils.TestingUtilities;
 import org.hl7.fhir.r5.utils.FHIRPathEngine;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
@@ -175,7 +175,7 @@ public class CDARoundTripTests {
 
   @Test
   public void testSimple() throws IOException {
-    PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     SimpleWorkerContext context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.r4.core", "4.0.1"));
     context.loadFromFile(TestingUtilities.loadTestResourceStream("validator", "cda", "any.xml"), "any.xml", null);
     context.loadFromFile(TestingUtilities.loadTestResourceStream("validator", "cda", "ii.xml"), "ii.xml", null);

org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/FHIRMappingLanguageTests.java

@@ -33,7 +33,7 @@ import org.hl7.fhir.r5.utils.StructureMapUtilities;
 import org.hl7.fhir.r5.utils.StructureMapUtilities.ITransformerServices;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.hl7.fhir.utilities.xml.XMLUtil;
 import org.junit.jupiter.api.BeforeAll;
@@ -67,7 +67,7 @@ public class FHIRMappingLanguageTests implements ITransformerServices {
 
   @BeforeAll
   public static void setUp() throws Exception {
-    PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.core", "4.0.1"));
     jsonParser = new JsonParser();
     jsonParser.setOutputStyle(OutputStyle.PRETTY);

org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/SnapShotGenerationTests.java

@@ -43,7 +43,7 @@ import org.hl7.fhir.r5.utils.IResourceValidator;
 import org.hl7.fhir.r5.utils.XVerExtensionManager;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.cache.NpmPackage;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.hl7.fhir.utilities.validation.ValidationMessage;
 import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
@@ -522,7 +522,7 @@ public class SnapShotGenerationTests {
     pu.setDebug(test.isDebug());
     pu.setIds(test.getSource(), false);
     if (!TestingUtilities.context().hasPackage("hl7.fhir.xver-extensions", "0.0.4")) {
-      NpmPackage npm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION).loadPackage("hl7.fhir.xver-extensions", "0.0.4");
+      NpmPackage npm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION).loadPackage("hl7.fhir.xver-extensions", "0.0.4");
       TestingUtilities.context().loadFromPackage(npm, new TestLoader(new String[]{"StructureDefinition"}), new String[]{"StructureDefinition"});
     }
     pu.setXver(new XVerExtensionManager(TestingUtilities.context()));

org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/StructureMapUtilitiesTest.java

@@ -13,7 +13,7 @@ import org.hl7.fhir.r5.model.StructureMap;
 import org.hl7.fhir.r5.test.utils.TestingUtilities;
 import org.hl7.fhir.r5.utils.StructureMapUtilities;
 import org.hl7.fhir.r5.utils.StructureMapUtilities.ITransformerServices;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -25,7 +25,7 @@ public class StructureMapUtilitiesTest implements ITransformerServices {
 
   //  @BeforeAll
   static public void setUp() throws Exception {
-    PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.core", "4.0.0"));
   }
 

org.hl7.fhir.report/pom.xml

@@ -12,7 +12,6 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>org.hl7.fhir.report</artifactId>
-
     <packaging>bundle</packaging>
 
     <properties>

org.hl7.fhir.utilities/pom.xml

@@ -13,7 +13,6 @@
     <packaging>bundle</packaging>
 
     <dependencies>
-
         <dependency>
             <groupId>ca.uhn.hapi.fhir</groupId>
             <artifactId>hapi-fhir-base</artifactId>

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java

@@ -0,0 +1,158 @@
+package org.hl7.fhir.utilities.cache;
+
+import org.apache.commons.lang3.Validate;
+import org.hl7.fhir.utilities.Utilities;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+
+public abstract class BasePackageCacheManager implements IPackageCacheManager {
+  private static final Logger ourLog = LoggerFactory.getLogger(BasePackageCacheManager.class);
+  private List<String> myPackageServers = new ArrayList<>();
+  private Function<String, PackageClient> myClientFactory = address -> new CachingPackageClient(address);
+
+  /**
+   * Constructor
+   */
+  public BasePackageCacheManager() {
+    super();
+  }
+
+  /**
+   * Provide a new client factory implementation
+   */
+  public void setClientFactory(Function<String, PackageClient> theClientFactory) {
+    Validate.notNull(theClientFactory, "theClientFactory must not be null");
+    myClientFactory = theClientFactory;
+  }
+
+  public List<String> getPackageServers() {
+    return myPackageServers;
+  }
+
+  /**
+   * Add a package server that can be used to fetch remote packages
+   */
+  public void addPackageServer(@Nonnull String thePackageServer) {
+    Validate.notBlank(thePackageServer, "thePackageServer must not be null or empty");
+    if (!myPackageServers.contains(thePackageServer)) {
+      myPackageServers.add(thePackageServer);
+    }
+  }
+
+
+  /**
+   * Load the latest version of the identified package from the cache - it it exists
+   */
+  public NpmPackage loadPackageFromCacheOnly(String id) throws IOException {
+    return loadPackageFromCacheOnly(id, null);
+  }
+
+  /**
+   * Try to load a package from all registered package servers, and return <code>null</code>
+   * if it can not be found at any of them.
+   */
+  @Nullable
+  protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
+
+    for (String nextPackageServer : getPackageServers()) {
+      PackageClient packageClient = myClientFactory.apply(nextPackageServer);
+      try {
+        if (Utilities.noString(version)) {
+          version = packageClient.getLatestVersion(id);
+        }
+        InputStream stream = packageClient.fetch(id, version);
+        String url = packageClient.url(id, version);
+        return new InputStreamWithSrc(stream, url, version);
+      } catch (IOException e) {
+        ourLog.info("Failed to resolve package {}#{} from server: {}", id, version, nextPackageServer);
+      }
+
+    }
+
+    return null;
+  }
+
+  public abstract NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException;
+
+  @Override
+  public String getPackageUrl(String packageId) throws IOException {
+    String result = null;
+    NpmPackage npm = loadPackageFromCacheOnly(packageId);
+    if (npm != null) {
+      return npm.canonical();
+    }
+
+    for (String nextPackageServer : getPackageServers()) {
+      result = getPackageUrl(packageId, nextPackageServer);
+      if (result != null) {
+        return result;
+      }
+    }
+
+    return result;
+  }
+
+
+  private String getPackageUrl(String packageId, String server) throws IOException {
+    PackageClient pc = myClientFactory.apply(server);
+    List<PackageClient.PackageInfo> res = pc.search(packageId, null, null, false);
+    if (res.size() == 0) {
+      return null;
+    } else {
+      return res.get(0).getUrl();
+    }
+  }
+
+
+  @Override
+  public String getPackageId(String canonicalUrl) throws IOException {
+    String result = null;
+
+    for (String nextPackageServer : getPackageServers()) {
+      result = getPackageId(canonicalUrl, nextPackageServer);
+      if (result != null) {
+        break;
+      }
+    }
+
+    return result;
+  }
+
+  private String getPackageId(String canonical, String server) throws IOException {
+    PackageClient pc = myClientFactory.apply(server);
+    List<PackageClient.PackageInfo> res = pc.search(null, canonical, null, false);
+    if (res.size() == 0) {
+      return null;
+    } else {
+      // this is driven by HL7 Australia (http://hl7.org.au/fhir/ is the canonical url for the base package, and the root for all the others)
+      for (PackageClient.PackageInfo pi : res) {
+        if (canonical.equals(pi.getCanonical())) {
+          return pi.getId();
+        }
+      }
+      return res.get(0).getId();
+    }
+  }
+
+  public class InputStreamWithSrc {
+
+    public InputStream stream;
+    public String url;
+    public String version;
+
+    public InputStreamWithSrc(InputStream stream, String url, String version) {
+      this.stream = stream;
+      this.url = url;
+      this.version = version;
+    }
+  }
+
+}

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java

@@ -0,0 +1,58 @@
+package org.hl7.fhir.utilities.cache;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Date;
+import java.util.List;
+
+import org.hl7.fhir.utilities.TextFile;
+import org.hl7.fhir.utilities.Utilities;
+
+/**
+ * Implementation of a package client that keeps a local disk cache of downloaded artifacts
+ * in order to avoid re-downloading things
+ */
+public class CachingPackageClient extends PackageClient {
+
+  private String cacheFolder;
+
+
+  public CachingPackageClient(String address) {
+    super(address);
+    try {
+      cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "package-client");
+      Utilities.createDirectory(cacheFolder);
+    } catch (IOException e) {
+    }
+  }
+
+  public boolean exists(String id, String ver) throws IOException {
+    List<PackageInfo> vl = getVersions(id);
+    for (PackageInfo pi : vl) {
+      if (ver.equals(pi.getVersion())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public InputStream fetchCached(String url) throws IOException {
+    File cacheFile = new File(Utilities.path(cacheFolder, fn(url)));
+    if (cacheFile.exists()) {
+      return new FileInputStream(cacheFile);
+    }
+    InputStream fetchedPackage = super.fetchCached(url);
+    TextFile.bytesToFile(TextFile.streamToBytes(fetchedPackage), cacheFile);
+    return new FileInputStream(cacheFile);
+  }
+
+
+  public Date getNewPackages(Date lastCalled, List<PackageInfo> updates) {
+    return null;
+  }
+
+
+
+}

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java

@@ -30,9 +30,20 @@ package org.hl7.fhir.utilities.cache;
  */
 
 
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import org.apache.commons.io.FileUtils;
+import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.utilities.IniFile;
+import org.hl7.fhir.utilities.TextFile;
+import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder;
+import org.hl7.fhir.utilities.json.JSONUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -58,31 +69,12 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-import org.apache.commons.io.FileUtils;
-import org.hl7.fhir.exceptions.FHIRException;
-import org.hl7.fhir.utilities.IniFile;
-import org.hl7.fhir.utilities.TextFile;
-import org.hl7.fhir.utilities.Utilities;
-import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder;
-import org.hl7.fhir.utilities.cache.PackageCacheManager.BuildRecord;
-import org.hl7.fhir.utilities.cache.PackageCacheManager.BuildRecordSorter;
-import org.hl7.fhir.utilities.cache.PackageCacheManager.InputStreamWithSrc;
-import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
-import org.hl7.fhir.utilities.json.JSONUtil;
-
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-
 /**
- * Package cache manager
+ * This is a package cache manager implementation that uses a local disk cache
  *
+ * <p>
  * API:
- * 
+ * <p>
  * constructor
  * getPackageUrl
  * getPackageId
@@ -90,153 +82,30 @@ import com.google.gson.JsonObject;
  * addPackageToCache
  *
  * @author Grahame Grieve
- *
  */
-public class PackageCacheManager {
+public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager {
-
-  public class InputStreamWithSrc {
-
-    public InputStream stream;
-    public String url;
-    public String version;
-    
-    public InputStreamWithSrc(InputStream stream, String url, String version) {
-      this.stream = stream;
-      this.url = url;
-      this.version = version;
-    }
-  }
-
-  public class BuildRecordSorter implements Comparator<BuildRecord> {
-
-    @Override
-    public int compare(BuildRecord arg0, BuildRecord arg1) {
-      return arg1.date.compareTo(arg0.date);
-    }
-  }
-
-  public class BuildRecord {
-
-    private String url;
-    private String packageId;
-    private String repo;
-    private Date date;
-    public BuildRecord(String url, String packageId, String repo, Date date) {
-      super();
-      this.url = url;
-      this.packageId = packageId;
-      this.repo = repo;
-      this.date = date;
-    }
-    public String getUrl() {
-      return url;
-    }
-    public String getPackageId() {
-      return packageId;
-    }
-    public String getRepo() {
-      return repo;
-    }
-    public Date getDate() {
-      return date;
-    }
-    
-
-  }
-
-  /** if you don't provide and implementation of this interface, the PackageCacheManager will use the web directly. 
-   * 
-   * You can use this interface to 
-   * @author graha
-   *
-   */
-  public interface INetworkServices {
-    
-    InputStream resolvePackage(String packageId, String version);
-  }
-  
-  public class VersionHistory {
-    private String id;
-    private String canonical;
-    private String current;
-    private Map<String, String> versions = new HashMap<>();
-    public String getCanonical() {
-      return canonical;
-    }
-    public String getCurrent() {
-      return current;
-    }
-    public Map<String, String> getVersions() {
-      return versions;
-    }
-    public String getId() {
-      return id;
-    } 
-  }
-
-
-  public class PackageEntry {
-
-    private byte[] bytes;
-    private String name;
-
-    public PackageEntry(String name) {
-      this.name = name;
-    }
-
-    public PackageEntry(String name, byte[] bytes) {
-      this.name = name;
-      this.bytes = bytes;
-    }
-  }
-
-  public interface CacheLockFunction<T> {
-    T get() throws IOException;
-  }
-  public class CacheLock {
-
-    private final File lockFile;
-
-    public CacheLock(String name) throws IOException {
-      this.lockFile = new File(cacheFolder, name + ".lock");
-      if (!lockFile.isFile()) {
-        TextFile.stringToFile("", lockFile);
-      }
-    }
-
-    public <T> T doWithLock(CacheLockFunction<T> f) throws FileNotFoundException, IOException {
-      try (FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel()) {
-        final FileLock fileLock = channel.lock();
-        T result = null;
-        try {
-          result = f.get();
-        } finally {
-          fileLock.release();
-        }
-        if (!lockFile.delete()) {
-          lockFile.deleteOnExit();
-        }
-        return result;
-      } 
-    }
-  }
 
   public static final String PRIMARY_SERVER = "http://packages.fhir.org";
   public static final String SECONDARY_SERVER = "http://packages2.fhir.org/packages";
-//  private static final String SECONDARY_SERVER = "http://local.fhir.org:960/packages";
+  //  private static final String SECONDARY_SERVER = "http://local.fhir.org:960/packages";
   public static final String PACKAGE_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+$";
   public static final String PACKAGE_VERSION_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+\\#[a-z0-9\\-\\_]+(\\.[a-z0-9\\-\\_]+)*$";
+  private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class);
   private static final String CACHE_VERSION = "3"; // second version - see wiki page
   private String cacheFolder;
   private boolean progress = true;
-  private List<NpmPackage> temporaryPackages = new ArrayList<NpmPackage>();
+  private List<NpmPackage> temporaryPackages = new ArrayList<>();
+  private boolean buildLoaded = false;
+  private Map<String, String> ciList = new HashMap<String, String>();
+  private JsonArray buildInfo;
 
-//  private List<String> allUrls;
+  /**
-//  private Map<String, VersionHistory> historyCache = new HashMap<>();
+   * Constructor
+   */
+  public FilesystemPackageCacheManager(boolean userMode, int toolsVersion) throws IOException {
+    addPackageServer(PRIMARY_SERVER);
+    addPackageServer(SECONDARY_SERVER);
 
-  // ========================= Initialization ============================================================================
-
-  public PackageCacheManager(boolean userMode, int toolsVersion) throws IOException {
     if (userMode)
       cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
     else
@@ -244,7 +113,7 @@ public class PackageCacheManager {
     if (!(new File(cacheFolder).exists()))
       Utilities.createDirectory(cacheFolder);
     if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
-      TextFile.stringToFile("[cache]\r\nversion="+CACHE_VERSION+"\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false);  
+      TextFile.stringToFile("[cache]\r\nversion=" + CACHE_VERSION + "\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false);
     createIniFile();
   }
 
@@ -259,16 +128,10 @@ public class PackageCacheManager {
     }
   }
 
-  // ========================= Utilities ============================================================================
-  
   public String getFolder() {
     return cacheFolder;
   }
 
-  private static String userDir() throws IOException {
-    return Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
-  }
-
   private List<String> sorted(String[] keys) {
     List<String> names = new ArrayList<String>();
     for (String s : keys)
@@ -304,8 +167,7 @@ public class PackageCacheManager {
           }
           return null; // must return something
         });
-      }
+      } else if (!f.getName().equals("packages.ini"))
-      else if (!f.getName().equals("packages.ini"))
         FileUtils.forceDelete(f);
     }
     IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
@@ -326,114 +188,66 @@ public class PackageCacheManager {
 
   private void checkValidVersionString(String version, String id) {
     if (Utilities.noString(version)) {
-      throw new FHIRException("Cannot add package "+id+" to the package cache - a version must be provided");
+      throw new FHIRException("Cannot add package " + id + " to the package cache - a version must be provided");
     }
     if (version.startsWith("file:")) {
-      throw new FHIRException("Cannot add package "+id+" to the package cache - the version '"+version+"' is illegal in this context");
+      throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal in this context");
     }
     for (char ch : version.toCharArray()) {
       if (!Character.isAlphabetic(ch) && !Character.isDigit(ch) && !Utilities.existsInList(ch, '.', '-')) {
-        throw new FHIRException("Cannot add package "+id+" to the package cache - the version '"+version+"' is illegal (ch '"+ch+"'");
+        throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal (ch '" + ch + "'");
       }
     }
   }
 
-  private String getPackageId(String canonical, String server) throws IOException {
-    PackageClient pc = new PackageClient(server);
-    List<PackageInfo> res = pc.search(null, canonical, null, false);
-    if (res.size() == 0) {
-      return null;
-    } else {
-      // this is driven by HL7 Australia (http://hl7.org.au/fhir/ is the canonical url for the base package, and the root for all the others)
-      for (PackageInfo pi : res) {
-        if (canonical.equals(pi.getCanonical())) {
-          return pi.getId();
-        }
-      }
-      return res.get(0).getId();
-    }
-  }
-
-  private String getPackageUrl(String packageId, String server) throws IOException {
-    PackageClient pc = new PackageClient(server);
-    List<PackageInfo> res = pc.search(packageId, null, null, false);
-    if (res.size() == 0) {
-      return null;
-    } else {
-      return res.get(0).getUrl();
-    }
-  }
-
   private void listSpecs(Map<String, String> specList, String server) throws IOException {
-    PackageClient pc = new PackageClient(server);
+    CachingPackageClient pc = new CachingPackageClient(server);
-    List<PackageInfo> matches = pc.search(null, null, null, false);
+    List<PackageClient.PackageInfo> matches = pc.search(null, null, null, false);
-    for (PackageInfo m : matches) {
+    for (PackageClient.PackageInfo m : matches) {
       if (!specList.containsKey(m.getId())) {
         specList.put(m.getId(), m.getUrl());
       }
     }
   }
 
-  private InputStreamWithSrc loadFromPackageServer(String id, String v) {
+  protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
-    PackageClient pc = new PackageClient(PRIMARY_SERVER);
+    InputStreamWithSrc retVal = super.loadFromPackageServer(id, version);
-    String u = null;
+    if (retVal != null) {
-    InputStream stream;
+      return retVal;
-    try {
-      if (Utilities.noString(v)) {
-        v = pc.getLatestVersion(id);
     }
-      stream = pc.fetch(id, v);
+
-      u = pc.url(id, v);
-    } catch (IOException e) {
-      pc = new PackageClient(SECONDARY_SERVER);
-      try {
-        if (Utilities.noString(v)) {
-          v = pc.getLatestVersion(id);
-        }
-        stream = pc.fetch(id, v);
-        u = pc.url(id, v);
-      } catch (IOException e1) {
     // ok, well, we'll try the old way
-        return fetchTheOldWay(id, v);
+    return fetchTheOldWay(id, version);
-      }
-    }
-    return new InputStreamWithSrc(stream, pc.url(id, v), v);
   }
 
   public String getLatestVersion(String id) throws IOException {
-    PackageClient pc = new PackageClient(PRIMARY_SERVER);
+    for (String nextPackageServer : getPackageServers()) {
+      CachingPackageClient pc = new CachingPackageClient(nextPackageServer);
       try {
         return pc.getLatestVersion(id);
       } catch (IOException e) {
-      pc = new PackageClient(SECONDARY_SERVER);
+        ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer);
-      try {
+      }
-        return pc.getLatestVersion(id);
+    }
-      } catch (IOException e1) {
+
     return fetchVersionTheOldWay(id);
   }
-    }
-  }
-
-
 
   private NpmPackage loadPackageFromFile(String id, String folder) throws IOException {
     File f = new File(Utilities.path(folder, id));
     if (!f.exists()) {
-      throw new FHIRException("Package '"+id+"  not found in folder "+folder);
+      throw new FHIRException("Package '" + id + "  not found in folder " + folder);
     }
     if (!f.isDirectory()) {
-      throw new FHIRException("File for '"+id+"  found in folder "+folder+", not a folder");
+      throw new FHIRException("File for '" + id + "  found in folder " + folder + ", not a folder");
     }
     File fp = new File(Utilities.path(folder, id, "package", "package.json"));
     if (!fp.exists()) {
-      throw new FHIRException("Package '"+id+"  found in folder "+folder+", but does not contain a package.json file in /package");
+      throw new FHIRException("Package '" + id + "  found in folder " + folder + ", but does not contain a package.json file in /package");
     }
     return NpmPackage.fromFolder(f.getAbsolutePath());
   }
 
-
-  // ========================= Package Mgmt API =======================================================================
-
   /**
    * Clear the cache
    *
@@ -443,6 +257,8 @@ public class PackageCacheManager {
     clearCache();
   }
 
+  // ========================= Utilities ============================================================================
+
   /**
    * Remove a particular package from the cache
    *
@@ -451,13 +267,13 @@ public class PackageCacheManager {
    * @throws IOException
    */
   public void removePackage(String id, String ver) throws IOException {
-    new CacheLock(id+"#"+ver).doWithLock(() -> {
+    new CacheLock(id + "#" + ver).doWithLock(() -> {
-      String f = Utilities.path(cacheFolder, id+"#"+ver);
+      String f = Utilities.path(cacheFolder, id + "#" + ver);
       File ff = new File(f);
       if (ff.exists()) {
         Utilities.clearDirectory(f);
         IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
-        ini.removeProperty("packages", id+"#"+ver);
+        ini.removeProperty("packages", id + "#" + ver);
         ini.save();
         ff.delete();
       }
@@ -465,20 +281,9 @@ public class PackageCacheManager {
     });
   }
 
-  /**
-   * Load the latest version of the identified package from the cache - it it exists
-   * 
-   * @param id
-   * @return
-   * @throws IOException
-   */
-  public NpmPackage loadPackageFromCacheOnly(String id) throws IOException {
-    return loadPackageFromCacheOnly(id, null);    
-  }
-  
   /**
    * Load the identified package from the cache - it it exists
-   * 
+   * <p>
    * This is for special purpose only (testing, control over speed of loading).
    * Generally, use the loadPackage method
    *
@@ -487,6 +292,7 @@ public class PackageCacheManager {
    * @return
    * @throws IOException
    */
+  @Override
   public NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException {
     if (!Utilities.noString(version) && version.startsWith("file:")) {
       return loadPackageFromFile(id, version.substring(5));
@@ -501,7 +307,7 @@ public class PackageCacheManager {
       }
     }
     for (String f : sorted(new File(cacheFolder).list())) {
-      if (f.equals(id+"#"+version) || (Utilities.noString(version) && f.startsWith(id+"#"))) {
+      if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) {
         return loadPackageInfo(Utilities.path(cacheFolder, f));
       }
     }
@@ -514,31 +320,32 @@ public class PackageCacheManager {
   /**
    * Add an already fetched package to the cache
    */
-  public NpmPackage addPackageToCache(String id, String version, InputStream tgz, String sourceDesc) throws IOException {
+  @Override
+  public NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException {
     checkValidVersionString(version, id);
-    if (progress ) {
+    if (progress) {
-      System.out.println("Installing "+id+"#"+(version == null ? "?" : version)+" to the package cache");
+      System.out.println("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache");
       System.out.print("  Fetching:");
     }
 
-    NpmPackage npm = NpmPackage.fromPackage(tgz, sourceDesc, true);
+    NpmPackage npm = NpmPackage.fromPackage(packageTgzInputStream, sourceDesc, true);
 
-    if (progress ) {
+    if (progress) {
       System.out.println();
       System.out.print("  Installing: ");
     }
     if (npm.name() == null || id == null || !id.equals(npm.name())) {
       if (!id.equals("hl7.fhir.r5.core")) {// temporary work around
-        throw new IOException("Attempt to import a mis-identified package. Expected "+id+", got "+npm.name());
+        throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name());
       }
     }
     if (version == null)
       version = npm.version();
 
     String v = version;
-    return new CacheLock(id+"#"+version).doWithLock(() -> {
+    return new CacheLock(id + "#" + version).doWithLock(() -> {
       NpmPackage pck = null;
-      String packRoot = Utilities.path(cacheFolder, id+"#"+v);
+      String packRoot = Utilities.path(cacheFolder, id + "#" + v);
       try {
         // ok, now we have a lock on it... check if something created it while we were waiting
         if (!new File(packRoot).exists()) {
@@ -549,7 +356,7 @@ public class PackageCacheManager {
           int c = 0;
           int size = 0;
           for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
-            String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package",  e.getKey());;
+            String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());
             if (!(new File(dir).exists()))
               Utilities.createDirectory(dir);
             for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
@@ -573,8 +380,8 @@ public class PackageCacheManager {
 
           IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
           ini.setTimeStampFormat("yyyyMMddhhmmss");
-          ini.setTimestampProperty("packages", id+"#"+v, Timestamp.from(Instant.now()), null);
+          ini.setTimestampProperty("packages", id + "#" + v, Timestamp.from(Instant.now()), null);
-          ini.setIntegerProperty("package-sizes", id+"#"+v, size, null);
+          ini.setIntegerProperty("package-sizes", id + "#" + v, size, null);
           ini.save();
           if (progress)
             System.out.println(" done.");
@@ -591,12 +398,12 @@ public class PackageCacheManager {
             npm.getNpm().remove("version");
             npm.getNpm().addProperty("version", v);
           }
-          TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+v, "package", "package.json"), false);
+          TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id + "#" + v, "package", "package.json"), false);
         }
       } catch (Exception e) {
         try {
           // don't leave a half extracted package behind
-          System.out.println("Clean up package "+packRoot+" because installation failed: "+e.getMessage());
+          System.out.println("Clean up package " + packRoot + " because installation failed: " + e.getMessage());
           e.printStackTrace();
           Utilities.clearDirectory(packRoot);
           new File(packRoot).delete();
@@ -609,35 +416,13 @@ public class PackageCacheManager {
     });
   }
 
-  public String getPackageId(String canonical) throws IOException {
+  @Override
-    String result = null;
-    if (result == null) {
-      result = getPackageId(canonical, PRIMARY_SERVER);
-    }
-    if (result == null) {
-      result = getPackageId(canonical, SECONDARY_SERVER);
-    }
-    if (result == null) {
-      result = getPackageIdFromBuildList(canonical);
-    }
-    return result;
-  }
-
   public String getPackageUrl(String packageId) throws IOException {
-    String result = null;
+    String result = super.getPackageUrl(packageId);
-    NpmPackage npm = loadPackageFromCacheOnly(packageId);
-    if (npm != null) {
-      return npm.canonical();
-    }      
-    if (result == null) {
-      getPackageUrl(packageId, PRIMARY_SERVER);
-    }
-    if (result == null) {
-      result = getPackageUrl(packageId, SECONDARY_SERVER);
-    }
     if (result == null) {
       result = getPackageUrlFromBuildList(packageId);
     }
+
     return result;
   }
 
@@ -645,51 +430,47 @@ public class PackageCacheManager {
     for (NpmPackage p : temporaryPackages) {
       specList.put(p.name(), p.canonical());
     }
-    listSpecs(specList, PRIMARY_SERVER);
+    for (String next : getPackageServers()) {
-    listSpecs(specList, SECONDARY_SERVER);
+      listSpecs(specList, next);
+    }
     addCIBuildSpecs(specList);
   }
 
-  public NpmPackage loadPackage(String id, String v) throws FHIRException, IOException {
+  @Override
+  public NpmPackage loadPackage(String id, String version) throws FHIRException, IOException {
     //ok, try to resolve locally
-    if (!Utilities.noString(v) && v.startsWith("file:")) {
+    if (!Utilities.noString(version) && version.startsWith("file:")) {
-      return loadPackageFromFile(id, v.substring(5));
+      return loadPackageFromFile(id, version.substring(5));
     }
-    NpmPackage p = loadPackageFromCacheOnly(id, v);
+
+    NpmPackage p = loadPackageFromCacheOnly(id, version);
     if (p != null) {
-      if ("current".equals(v)) {
+      if ("current".equals(version)) {
         p = checkCurrency(id, p);
       }
       if (p != null)
         return p;
     }
 
-    if ("dev".equals(v)) {
+    if ("dev".equals(version)) {
       p = loadPackageFromCacheOnly(id, "current");
       p = checkCurrency(id, p);
       if (p != null)
         return p;
-      v = "current";
+      version = "current";
     }
 
     // nup, don't have it locally (or it's expired)
-    InputStreamWithSrc source;
+    FilesystemPackageCacheManager.InputStreamWithSrc source;
-    if ("current".equals(v)) {
+    if ("current".equals(version)) {
       // special case - fetch from ci-build server
       source = loadFromCIBuild(id);
     } else {
-      source = loadFromPackageServer(id, v);
+      source = loadFromPackageServer(id, version);
     }
-    return addPackageToCache(id, v == null ? source.version : v, source.stream, source.url);
+    return addPackageToCache(id, version == null ? source.version : version, source.stream, source.url);
   }
 
-
-  // ---------- Current Build SubSystem --------------------------------------------------------------------------------------
-
-  private boolean buildLoaded = false;
-  private Map<String, String> ciList = new HashMap<String, String>();
-  private JsonArray buildInfo;
-
   private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
     try {
       URL url = new URL(source);
@@ -709,32 +490,13 @@ public class PackageCacheManager {
       InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false);
       return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current");
     } else if (id.startsWith("hl7.fhir.r5")) {
-      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", id+".tgz"), false);
+      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", id + ".tgz"), false);
-      return new InputStreamWithSrc(stream, Utilities.pathURL("http://build.fhir.org", id+".tgz"), "current");
+      return new InputStreamWithSrc(stream, Utilities.pathURL("http://build.fhir.org", id + ".tgz"), "current");
     } else {
-      throw new FHIRException("The package '"+id+"' has not entry on the current build server");
+      throw new FHIRException("The package '" + id + "' has not entry on the current build server");
     }
   }
 
-  private String getPackageIdFromBuildList(String canonical) throws IOException {
-    checkBuildLoaded();
-    if (buildInfo != null) {
-      for (JsonElement n : buildInfo) {
-        JsonObject o = (JsonObject) n;
-        if (canonical.equals(JSONUtil.str(o, "url"))) {
-          return JSONUtil.str(o, "package-id");
-        }
-      }
-      for (JsonElement n : buildInfo) {
-        JsonObject o = (JsonObject) n;
-        if (JSONUtil.str(o, "url").startsWith(canonical+"/ImplementationGuide/")) {
-          return JSONUtil.str(o, "package-id");
-        }
-      }
-    }
-    return null;
-  }
-  
   private String getPackageUrlFromBuildList(String packageId) throws IOException {
     checkBuildLoaded();
     for (JsonElement n : buildInfo) {
@@ -746,7 +508,6 @@ public class PackageCacheManager {
     return null;
   }
 
-
   private void addCIBuildSpecs(Map<String, String> specList) throws IOException {
     checkBuildLoaded();
     for (JsonElement n : buildInfo) {
@@ -757,6 +518,38 @@ public class PackageCacheManager {
     }
   }
 
+  @Override
+  public String getPackageId(String canonicalUrl) throws IOException {
+    String retVal = super.getPackageId(canonicalUrl);
+
+    if (retVal == null) {
+      retVal = getPackageIdFromBuildList(canonicalUrl);
+    }
+
+    return retVal;
+  }
+
+
+  // ========================= Package Mgmt API =======================================================================
+
+  private String getPackageIdFromBuildList(String canonical) throws IOException {
+    checkBuildLoaded();
+    if (buildInfo != null) {
+      for (JsonElement n : buildInfo) {
+        JsonObject o = (JsonObject) n;
+        if (canonical.equals(JSONUtil.str(o, "url"))) {
+          return JSONUtil.str(o, "package-id");
+        }
+      }
+      for (JsonElement n : buildInfo) {
+        JsonObject o = (JsonObject) n;
+        if (JSONUtil.str(o, "url").startsWith(canonical + "/ImplementationGuide/")) {
+          return JSONUtil.str(o, "package-id");
+        }
+      }
+    }
+    return null;
+  }
 
   private NpmPackage checkCurrency(String id, NpmPackage p) throws IOException {
     checkBuildLoaded();
@@ -780,12 +573,11 @@ public class PackageCacheManager {
     try {
       loadFromBuildServer();
     } catch (Exception e) {
-      System.out.println("Error connecting to build server - running without build ("+e.getMessage()+")");
+      System.out.println("Error connecting to build server - running without build (" + e.getMessage() + ")");
     }
     return false;
   }
 
-  
   private void loadFromBuildServer() throws IOException {
     URL url = new URL("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
     HttpURLConnection connection = (HttpURLConnection) url.openConnection();
@@ -807,13 +599,13 @@ public class PackageCacheManager {
     Collections.sort(builds, new BuildRecordSorter());
     for (BuildRecord bld : builds) {
       if (!ciList.containsKey(bld.getPackageId())) {
-        ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/"+bld.getRepo());        
+        ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo());
       }
     }
     buildLoaded = true; // whether it succeeds or not
   }
 
-//  private String buildPath(String url) {
+  //  private String buildPath(String url) {
 //    for (JsonElement e : buildInfo) {
 //      JsonObject j = (JsonObject) e;
 //      if (j.has("url") && (url.equals(j.get("url").getAsString()) || j.get("url").getAsString().startsWith(url+"/ImplementationGuide"))) {
@@ -825,7 +617,7 @@ public class PackageCacheManager {
 //
   private String getRepo(String path) {
     String[] p = path.split("\\/");
-    return p[0]+"/"+p[1];
+    return p[0] + "/" + p[1];
   }
 
   private Date readDate(String s) {
@@ -849,7 +641,7 @@ public class PackageCacheManager {
       }
     }
     if (url == null) {
-      throw new FHIRException("Unable to resolve package id "+id+"#"+v);
+      throw new FHIRException("Unable to resolve package id " + id + "#" + v);
     }
     if (url.contains("/ImplementationGuide/")) {
       url = url.substring(0, url.indexOf("/ImplementationGuide/"));
@@ -866,11 +658,11 @@ public class PackageCacheManager {
         InputStreamWithSrc src = new InputStreamWithSrc(fetchFromUrlSpecific(pv, true), pv, v);
         return src;
       } catch (Exception e1) {
-        throw new FHIRException("Error fetching package directly ("+pv+"), or fetching package list for "+id+" from "+pu+": "+e1.getMessage(), e1);
+        throw new FHIRException("Error fetching package directly (" + pv + "), or fetching package list for " + id + " from " + pu + ": " + e1.getMessage(), e1);
       }
     }
     if (!id.equals(JSONUtil.str(json, "package-id")))
-      throw new FHIRException("Package ids do not match in "+pu+": "+id+" vs "+JSONUtil.str(json, "package-id"));
+      throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + JSONUtil.str(json, "package-id"));
     for (JsonElement e : json.getAsJsonArray("list")) {
       JsonObject vo = (JsonObject) e;
       if (v.equals(JSONUtil.str(vo, "version"))) {
@@ -883,6 +675,9 @@ public class PackageCacheManager {
     return null;
   }
 
+
+  // ---------- Current Build SubSystem --------------------------------------------------------------------------------------
+
   private String fetchVersionTheOldWay(String id) throws IOException {
     String url = getUrlForPackage(id);
     if (url == null) {
@@ -893,12 +688,12 @@ public class PackageCacheManager {
       }
     }
     if (url == null) {
-      throw new FHIRException("Unable to resolve package id "+id);
+      throw new FHIRException("Unable to resolve package id " + id);
     }
     String pu = Utilities.pathURL(url, "package-list.json");
     JsonObject json = fetchJson(pu);
     if (!id.equals(JSONUtil.str(json, "package-id")))
-      throw new FHIRException("Package ids do not match in "+pu+": "+id+" vs "+JSONUtil.str(json, "package-id"));
+      throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + JSONUtil.str(json, "package-id"));
     for (JsonElement e : json.getAsJsonArray("list")) {
       JsonObject vo = (JsonObject) e;
       if (JSONUtil.bool(vo, "current")) {
@@ -926,6 +721,135 @@ public class PackageCacheManager {
     return res;
   }
 
+  /**
+   * if you don't provide and implementation of this interface, the PackageCacheManager will use the web directly.
+   * <p>
+   * You can use this interface to
+   *
+   * @author graha
+   */
+  public interface INetworkServices {
+
+    InputStream resolvePackage(String packageId, String version);
+  }
+
+  public interface CacheLockFunction<T> {
+    T get() throws IOException;
+  }
+
+  public class BuildRecordSorter implements Comparator<BuildRecord> {
+
+    @Override
+    public int compare(BuildRecord arg0, BuildRecord arg1) {
+      return arg1.date.compareTo(arg0.date);
+    }
+  }
+
+  public class BuildRecord {
+
+    private String url;
+    private String packageId;
+    private String repo;
+    private Date date;
+
+    public BuildRecord(String url, String packageId, String repo, Date date) {
+      super();
+      this.url = url;
+      this.packageId = packageId;
+      this.repo = repo;
+      this.date = date;
+    }
+
+    public String getUrl() {
+      return url;
+    }
+
+    public String getPackageId() {
+      return packageId;
+    }
+
+    public String getRepo() {
+      return repo;
+    }
+
+    public Date getDate() {
+      return date;
+    }
+
+
+  }
+
+
+
+  public class VersionHistory {
+    private String id;
+    private String canonical;
+    private String current;
+    private Map<String, String> versions = new HashMap<>();
+
+    public String getCanonical() {
+      return canonical;
+    }
+
+    public String getCurrent() {
+      return current;
+    }
+
+    public Map<String, String> getVersions() {
+      return versions;
+    }
+
+    public String getId() {
+      return id;
+    }
+  }
+
+  public class PackageEntry {
+
+    private byte[] bytes;
+    private String name;
+
+    public PackageEntry(String name) {
+      this.name = name;
+    }
+
+    public PackageEntry(String name, byte[] bytes) {
+      this.name = name;
+      this.bytes = bytes;
+    }
+  }
+
+  public class CacheLock {
+
+    private final File lockFile;
+
+    public CacheLock(String name) throws IOException {
+      this.lockFile = new File(cacheFolder, name + ".lock");
+      if (!lockFile.isFile()) {
+        TextFile.stringToFile("", lockFile);
+      }
+    }
+
+    public <T> T doWithLock(CacheLockFunction<T> f) throws FileNotFoundException, IOException {
+      try (FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel()) {
+        final FileLock fileLock = channel.lock();
+        T result = null;
+        try {
+          result = f.get();
+        } finally {
+          fileLock.release();
+        }
+        if (!lockFile.delete()) {
+          lockFile.deleteOnExit();
+        }
+        return result;
+      }
+    }
+  }
+
+
+
+
 
 
 //public List<String> getUrls() throws IOException {

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java

@@ -0,0 +1,17 @@
+package org.hl7.fhir.utilities.cache;
+
+import org.hl7.fhir.exceptions.FHIRException;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public interface IPackageCacheManager {
+
+  String getPackageId(String canonicalUrl) throws IOException;
+
+  NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException;
+
+  String getPackageUrl(String packageId) throws IOException;
+
+  NpmPackage loadPackage(String id, String version) throws FHIRException, IOException;
+}

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java

@@ -35,12 +35,12 @@ import java.io.BufferedOutputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -57,20 +57,14 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
-import org.hl7.fhir.utilities.IniFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
-import org.hl7.fhir.utilities.cache.NpmPackage.IndexVersionSorter;
-import org.hl7.fhir.utilities.cache.PackageCacheManager.PackageEntry;
 import org.hl7.fhir.utilities.cache.PackageGenerator.PackageType;
 import org.hl7.fhir.utilities.json.JSONUtil;
 import org.hl7.fhir.utilities.json.JsonTrackingParser;
 
-import com.google.common.base.Charsets;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
@@ -109,7 +103,7 @@ public class NpmPackage {
   public class NpmPackageFolder {
     private String name;
     private Map<String, List<String>> types = new HashMap<>();
-    private Map<String, byte[]> content = new HashMap<String, byte[]>(); 
+    private Map<String, byte[]> content = new HashMap<>();
     private JsonObject index;
     private File folder;
 
@@ -118,6 +112,10 @@ public class NpmPackage {
       this.name = name;
     }
 
+    public Map<String, List<String>> getTypes() {
+      return types;
+    }
+
     public String getName() {
       return name;
     }
@@ -198,17 +196,18 @@ public class NpmPackage {
   private JsonObject npm;
   private Map<String, NpmPackageFolder> folders = new HashMap<>();
   private boolean changedByLoader; // internal qa only!
+  private Map<String, Object> userData = new HashMap<>();
 
+  /**
+   * Constructor
+   */
   private NpmPackage() {
-
+    super();
   }
-  //  public NpmPackage(JsonObject npm, Map<String, byte[]> content, List<String> folders) {
-  //    this.path = null;
-  //    this.content = content;
-  //    this.npm = npm;
-  //    this.folders = folders;
-  //  }
 
+  /**
+   * Factory method that parses a package from an extracted folder
+   */
   public static NpmPackage fromFolder(String path) throws IOException {
     NpmPackage res = new NpmPackage();
     loadFiles(res, path, new File(path));
@@ -216,6 +215,19 @@ public class NpmPackage {
     return res;
   }
 
+  /**
+   * Factory method that starts a new empty package using the given PackageGenerator to create the manifest
+   */
+  public static NpmPackage empty(PackageGenerator thePackageGenerator) {
+    NpmPackage retVal = new NpmPackage();
+    retVal.npm = thePackageGenerator.getRootJsonObject();
+    return retVal;
+  }
+
+  public Map<String, Object> getUserData() {
+    return userData;
+  }
+
   public static void loadFiles(NpmPackage res, String path, File source, String... exemptions) throws FileNotFoundException, IOException {
     res.npm = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.fileToString(Utilities.path(path, "package", "package.json")));
     res.path = path;
@@ -788,7 +800,7 @@ public class NpmPackage {
           TextFile.bytesToFile(b, Utilities.path(dir.getAbsolutePath(), n, s));
         }
       }
-      byte[] cnt = indexer.build().getBytes(Charset.forName("UTF-8"));
+      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
       TextFile.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), n, ".index.json"));
     }
     byte[] cnt = TextFile.stringToBytes(new GsonBuilder().setPrettyPrinting().create().toJson(npm), false);
@@ -826,7 +838,7 @@ public class NpmPackage {
           tar.closeArchiveEntry();
         }
       }
-      byte[] cnt = indexer.build().getBytes(Charset.forName("UTF-8"));
+      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
       TarArchiveEntry entry = new TarArchiveEntry(n+"/.index.json");
       entry.setSize(cnt.length);
       tar.putArchiveEntry(entry);
@@ -849,7 +861,9 @@ public class NpmPackage {
     stream.write(b);
   }
 
-
+  /**
+   * Keys are resource type names, values are filenames
+   */
   public Map<String, List<String>> getTypes() {
     return folders.get("package").types;
   }
@@ -941,6 +955,9 @@ public class NpmPackage {
   }
 
   public void addFile(String folderName, String name, byte[] cnt, String type) {
+    if (!folders.containsKey(folderName)) {
+      folders.put(folderName, new NpmPackageFolder(folderName));
+    }
     NpmPackageFolder folder = folders.get(folderName);
     folder.content.put(name, cnt);
     if (!folder.types.containsKey(type))

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageClient.java

@@ -1,5 +1,14 @@
 package org.hl7.fhir.utilities.cache;
 
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
+import org.hl7.fhir.utilities.TextFile;
+import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.VersionUtilities;
+import org.hl7.fhir.utilities.json.JSONUtil;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -13,18 +22,6 @@ import java.util.Date;
 import java.util.List;
 import java.util.Set;
 
-import org.hl7.fhir.exceptions.FHIRException;
-import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
-import org.hl7.fhir.utilities.TextFile;
-import org.hl7.fhir.utilities.Utilities;
-import org.hl7.fhir.utilities.VersionUtilities;
-import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
-import org.hl7.fhir.utilities.json.JSONUtil;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-
 public class PackageClient {
 
   public class PackageInfo {
@@ -109,15 +106,10 @@ public class PackageClient {
   }
 
   public InputStream fetchCached(String url) throws IOException, FileNotFoundException {
-    File cacheFile = new File(Utilities.path(cacheFolder, fn(url)));
+    return fetchUrl(url, null);
-    if (cacheFile.exists()) {
-      return new FileInputStream(cacheFile);
-    }
-    TextFile.bytesToFile(TextFile.streamToBytes(fetchUrl(url, null)), cacheFile);
-    return new FileInputStream(cacheFile);
   }
 
-  private String fn(String url) {
+  protected String fn(String url) {
     String[] p = url.split("\\/");
     return p[2]+"-"+p[p.length-2]+"-"+p[p.length-1]+".tgz";
   }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageGenerator.java

@@ -63,9 +63,14 @@ public class PackageGenerator {
       throw new Error("Unknown Type");
     }
   }
+
   private OutputStream stream;
   private JsonObject object;
 
+  public PackageGenerator() {
+    object = new JsonObject();
+  }
+
   public PackageGenerator(OutputStream stream) {
     super();
     this.stream = stream;
@@ -80,6 +85,10 @@ public class PackageGenerator {
 
   }
 
+  public JsonObject getRootJsonObject() {
+    return object;
+  }
+
   public void commit() throws IOException {
     Gson gson = new GsonBuilder().setPrettyPrinting().create();
     String json = gson.toJson(object);
@@ -91,7 +100,8 @@ public class PackageGenerator {
   }
   
   public PackageGenerator name(String value) {
-    object.addProperty("name", "@fhir/"+value);
+    // NOTE: I removed a prefix of "@fhir/" here. What was this for? -JA
+    object.addProperty("name", value);
     return this;
   }
    

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/tests/BaseTestingUtilities.java

@@ -22,8 +22,9 @@ public class BaseTestingUtilities {
      * the name of the project directory to something other than 'fhir-test-cases', or move it to another location, not
      * at the same directory level as the core project.
      */
-    if (new File("../../fhir-test-cases").exists() && isTryToLoadFromFileSystem()) {
+    String dir = System.getenv("FHIR-TEST-CASES");
-      String n = Utilities.path(System.getProperty("user.dir"), "..", "..", "fhir-test-cases", Utilities.path(paths));
+    if (dir != null && new File(dir).exists()) {
+      String n = Utilities.path(dir, Utilities.path(paths));
       // ok, we'll resolve this locally
       return TextFile.fileToString(new File(n));
     } else {
@@ -41,8 +42,9 @@ public class BaseTestingUtilities {
   }
 
   public static InputStream loadTestResourceStream(String... paths) throws IOException {
-    if (new File("../../fhir-test-cases").exists() && isTryToLoadFromFileSystem()) {
+    String dir = System.getenv("FHIR-TEST-CASES");
-      String n = Utilities.path(System.getProperty("user.dir"), "..", "..", "fhir-test-cases", Utilities.path(paths));
+    if (dir != null && new File(dir).exists()) {
+      String n = Utilities.path(dir, Utilities.path(paths));
       return new FileInputStream(n);
     } else {
       String classpath = ("/org/hl7/fhir/testcases/" + Utilities.pathURL(paths));
@@ -55,8 +57,9 @@ public class BaseTestingUtilities {
   }
 
   public static byte[] loadTestResourceBytes(String... paths) throws IOException {
-    if (new File("../../fhir-test-cases").exists() && isTryToLoadFromFileSystem()) {
+    String dir = System.getenv("FHIR-TEST-CASES");
-      String n = Utilities.path(System.getProperty("user.dir"), "..", "..", "fhir-test-cases", Utilities.path(paths));
+    if (dir != null && new File(dir).exists()) {
+      String n = Utilities.path(dir, Utilities.path(paths));
       return TextFile.fileToBytes(n);
     } else {
       String classpath = ("/org/hl7/fhir/testcases/" + Utilities.pathURL(paths));
@@ -69,8 +72,9 @@ public class BaseTestingUtilities {
   }
 
   public static boolean findTestResource(String... paths) throws IOException {
-    if (new File("../../fhir-test-cases").exists() && isTryToLoadFromFileSystem()) {
+    String dir = System.getenv("FHIR-TEST-CASES");
-      String n = Utilities.path(System.getProperty("user.dir"), "..", "..", "fhir-test-cases", Utilities.path(paths));
+    if (dir != null && new File(dir).exists()) {
+      String n = Utilities.path(dir, Utilities.path(paths));
       return new File(n).exists();
     } else {
       String classpath = ("/org/hl7/fhir/testcases/" + Utilities.pathURL(paths));
@@ -83,10 +87,5 @@ public class BaseTestingUtilities {
     }
   }
 
-  // TODO: JA need to figure out how to detect that we're running in maven
-  public static boolean isTryToLoadFromFileSystem() {
-    return !"true".equals(System.getProperty("dont_load_from_filesystem"));
-  }
-
 
 }

org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java

@@ -1,6 +1,6 @@
 package org.hl7.fhir.utilities.tests;
 
-import org.hl7.fhir.utilities.cache.PackageClient;
+import org.hl7.fhir.utilities.cache.CachingPackageClient;
 import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -8,11 +8,11 @@ import org.junit.jupiter.api.Test;
 import java.io.IOException;
 import java.util.List;
 
-public class PackageClientTests {
+public class CachingPackageClientTests {
 
   @Test
   public void testExists() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     Assertions.assertTrue(client.exists("hl7.fhir.r4.core", "4.0.1"));
     Assertions.assertTrue(!client.exists("hl7.fhir.r4.core", "1.0.2"));
     Assertions.assertTrue(!client.exists("hl7.fhir.nothing", "1.0.1"));
@@ -20,7 +20,7 @@ public class PackageClientTests {
 
   @Test
   public void testSearch() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     List<PackageInfo> matches = client.search("core", null, null, false);
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -30,14 +30,14 @@ public class PackageClientTests {
 
   @Test
   public void testSearchNoMatches() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     List<PackageInfo> matches = client.search("corezxxx", null, null, false);
     Assertions.assertTrue(matches.size() == 0);
   }
 
   @Test
   public void testVersions() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3");
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -47,14 +47,14 @@ public class PackageClientTests {
 
   @Test
   public void testVersionsNone() throws IOException {
-    PackageClient client = new PackageClient("http://packages.fhir.org");
+    CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
     List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3X");
     Assertions.assertTrue(matches.size() == 0);
   }
 
   @Test
   public void testExists2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     Assertions.assertTrue(client.exists("hl7.fhir.r4.core", "4.0.1"));
     Assertions.assertTrue(!client.exists("hl7.fhir.r4.core", "1.0.2"));
     Assertions.assertTrue(!client.exists("hl7.fhir.nothing", "1.0.1"));
@@ -62,7 +62,7 @@ public class PackageClientTests {
 
   @Test
   public void testSearch2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.search("core", null, null, false);
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -72,14 +72,14 @@ public class PackageClientTests {
 
   @Test
   public void testSearchNoMatches2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.search("corezxxx", null, null, false);
     Assertions.assertTrue(matches.size() == 0);
   }
 
   @Test
   public void testVersions2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3");
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -89,7 +89,7 @@ public class PackageClientTests {
 
   @Test
   public void testVersions2A() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.getVersions("hl7.fhir.us.core");
     for (PackageInfo pi : matches) {
       System.out.println(pi.toString());
@@ -99,7 +99,7 @@ public class PackageClientTests {
 
   @Test
   public void testVersionsNone2() throws IOException {
-    PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
+    CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
     List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3X");
     Assertions.assertTrue(matches.size() == 0);
   }

org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java

@@ -2,10 +2,9 @@ package org.hl7.fhir.utilities.tests;
 
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.cache.NpmPackage;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import java.io.File;
@@ -16,7 +15,7 @@ public class PackageCacheTests {
 
   @Test
   public void testPath() throws IOException {
-    PackageCacheManager cache = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    FilesystemPackageCacheManager cache = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     cache.clear();
     List<String> list = cache.listPackages();
     if (!list.isEmpty()) {

org.hl7.fhir.validation/pom.xml

@@ -23,9 +23,11 @@
             <groupId>ca.uhn.hapi.fhir</groupId>
             <artifactId>hapi-fhir-base</artifactId>
         </dependency>
+
         <dependency>
             <groupId>ca.uhn.hapi.fhir</groupId>
             <artifactId>org.hl7.fhir.utilities</artifactId>
+            <version>${project.version}</version>
         </dependency>
 
         <!-- FHIR Versions -->

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java

@@ -4,7 +4,6 @@ import org.apache.commons.io.IOUtils;
 import org.hl7.fhir.convertors.*;
 import org.hl7.fhir.exceptions.DefinitionException;
 import org.hl7.fhir.exceptions.FHIRException;
-import org.hl7.fhir.exceptions.FHIRFormatError;
 import org.hl7.fhir.r5.conformance.ProfileUtilities;
 import org.hl7.fhir.r5.context.IWorkerContext.IContextResourceLoader;
 import org.hl7.fhir.r5.context.IWorkerContext.PackageVersion;
@@ -33,7 +32,7 @@ import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.cache.NpmPackage;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.hl7.fhir.utilities.validation.ValidationMessage;
 import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
@@ -42,11 +41,8 @@ import org.hl7.fhir.utilities.validation.ValidationMessage.Source;
 import org.hl7.fhir.utilities.xhtml.XhtmlComposer;
 import org.xml.sax.SAXException;
 
-import com.google.gson.JsonObject;
-
 import java.io.*;
 import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLConnection;
@@ -254,7 +250,7 @@ public class ValidationEngine implements IValidatorResourceFetcher {
   private boolean anyExtensionsAllowed = false;
   private String version;
   private String language;
-  private PackageCacheManager pcm;
+  private FilesystemPackageCacheManager pcm;
   private PrintWriter mapLog;
   private boolean debug;
   private Set<String> loadedIgs = new HashSet<>();
@@ -301,7 +297,7 @@ public class ValidationEngine implements IValidatorResourceFetcher {
   }
   
   public ValidationEngine() throws IOException {
-    pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);  
+    pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
   }
     
   public void setTerminologyServer(String src, String log, FhirPublication version) throws Exception {
@@ -325,7 +321,7 @@ public class ValidationEngine implements IValidatorResourceFetcher {
   }
 
   public ValidationEngine(String src, String txsrvr, String txLog, FhirPublication version, boolean canRunWithoutTerminologyServer, String vString) throws Exception {
-    pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     loadCoreDefinitions(src, false);
     context.setCanRunWithoutTerminology(canRunWithoutTerminologyServer);
     setTerminologyServer(txsrvr, txLog, version);    
@@ -333,7 +329,7 @@ public class ValidationEngine implements IValidatorResourceFetcher {
   }
   
   public ValidationEngine(String src, String txsrvr, String txLog, FhirPublication version, String vString) throws Exception {
-    pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     loadCoreDefinitions(src, false);
     setTerminologyServer(txsrvr, txLog, version);
     this.version = vString;
@@ -341,7 +337,7 @@ public class ValidationEngine implements IValidatorResourceFetcher {
   
   public ValidationEngine(String src) throws Exception {
     loadCoreDefinitions(src, false);
-    pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
   }
   
   public String getLanguage() {
@@ -473,7 +469,7 @@ public class ValidationEngine implements IValidatorResourceFetcher {
         res.put(Utilities.changeFileExt(src, "."+fmt.getExtension()), TextFile.fileToBytesNCS(src));
         return res;
       }
-    } else if ((src.matches(PackageCacheManager.PACKAGE_REGEX) || src.matches(PackageCacheManager.PACKAGE_VERSION_REGEX)) && !src.endsWith(".zip") && !src.endsWith(".tgz")) {
+    } else if ((src.matches(FilesystemPackageCacheManager.PACKAGE_REGEX) || src.matches(FilesystemPackageCacheManager.PACKAGE_VERSION_REGEX)) && !src.endsWith(".zip") && !src.endsWith(".tgz")) {
       return fetchByPackage(src);
     }
     throw new Exception("Unable to find/resolve/read -ig "+src);
@@ -627,7 +623,7 @@ public class ValidationEngine implements IValidatorResourceFetcher {
     }
     if (pcm == null) {
       log("Creating Package manager?");
-      pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+      pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     }
     if (version == null) {
       version = pcm.getLatestVersion(id);

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Display.java

@@ -2,7 +2,7 @@ package org.hl7.fhir.validation.cli.utils;
 
 import org.hl7.fhir.r5.model.Constants;
 import org.hl7.fhir.utilities.VersionUtilities;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 
 import java.io.IOException;
@@ -20,7 +20,7 @@ public class Display {
       System.out.print(s.contains(" ") ? " \"" + s + "\"" : " " + s);
     }
     System.out.println();
-    System.out.println("Directories: Current = " + System.getProperty("user.dir") + ", Package Cache = " + new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION).getFolder());
+    System.out.println("Directories: Current = " + System.getProperty("user.dir") + ", Package Cache = " + new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION).getFolder());
   }
 
   public static void displayHelpDetails() {

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java

@@ -6,14 +6,11 @@ import java.util.List;
 
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.exceptions.FHIRFormatError;
-import org.hl7.fhir.r5.context.IWorkerContext;
-import org.hl7.fhir.r5.context.SimpleWorkerContext;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.cache.NpmPackage;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
-import org.hl7.fhir.utilities.cache.PackageClient;
+import org.hl7.fhir.utilities.cache.CachingPackageClient;
 import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
-import org.xml.sax.InputSource;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 
 public class PackageValidator {
@@ -23,9 +20,9 @@ public class PackageValidator {
   }
 
   private void execute() throws IOException {
-    PackageCacheManager pcm = new  PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     
-    PackageClient pc = new PackageClient(PackageCacheManager.PRIMARY_SERVER);
+    CachingPackageClient pc = new CachingPackageClient(FilesystemPackageCacheManager.PRIMARY_SERVER);
     for (PackageInfo t : pc.search(null, null, null, false)) {
       System.out.println("Check Package "+t.getId());
       List<PackageInfo> vl = pc.getVersions(t.getId());

org.hl7.fhir.validation/src/test/java/org/hl7/fhir/comparison/tests/ComparisonTests.java

@@ -30,7 +30,7 @@ import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.cache.NpmPackage;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.hl7.fhir.utilities.validation.ValidationMessage;
 import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
@@ -102,7 +102,7 @@ public class ComparisonTests {
     if (!new File(Utilities.path("[tmp]", "comparison")).exists()) {
       System.out.println("---- Set up Output ----------------------------------------------------------");
       Utilities.createDirectory(Utilities.path("[tmp]", "comparison"));
-      PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+      FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
       NpmPackage npm = pcm.loadPackage("hl7.fhir.pubpack", "0.0.5");
       for (String f : npm.list("other")) {
         TextFile.streamToFile(npm.load("other", f), Utilities.path("[tmp]", "comparison", f));

org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/R3R4ConversionTests.java

@@ -21,7 +21,7 @@ import org.hl7.fhir.r4.utils.StructureMapUtilities.ITransformerServices;
 import org.hl7.fhir.utilities.IniFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 import org.hl7.fhir.utilities.validation.ValidationMessage;
 import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
@@ -42,7 +42,7 @@ import java.util.zip.ZipInputStream;
 public class R3R4ConversionTests implements ITransformerServices, IValidatorResourceFetcher {
 
   private static final boolean SAVING = true;
-  private PackageCacheManager pcm = null;
+  private FilesystemPackageCacheManager pcm = null;
 
   public static Stream<Arguments> data() throws ParserConfigurationException, SAXException, IOException {
     if (!(new File(Utilities.path(TestingUtilities.home(), "implementations", "r3maps", "outcomes.json")).exists()))
@@ -285,7 +285,7 @@ public class R3R4ConversionTests implements ITransformerServices, IValidatorReso
     if (contextR3 != null)
       return;
 
-    pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
     R3ToR4Loader ldr = new R3ToR4Loader().setPatchUrls(true).setKillPrimitives(true);
 
     System.out.println("loading R3");

org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/TestingUtilitiesX.java

@@ -58,7 +58,7 @@ import org.hl7.fhir.utilities.CSFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.ToolsVersion;
 
 import org.w3c.dom.Document;
@@ -83,9 +83,9 @@ public class TestingUtilitiesX {
       fcontexts = new HashMap<>();
     }
 	  if (!fcontexts.containsKey(version)) {
-	    PackageCacheManager pcm;
+	    FilesystemPackageCacheManager pcm;
 	    try {
-	      pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+	      pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
 	      IWorkerContext fcontext = SimpleWorkerContext.fromPackage(pcm.loadPackage(VersionUtilities.packageForVersion(version), version), loaderForVersion(version));
 	      fcontext.setUcumService(new UcumEssenceService(TestingUtilitiesX.loadTestResourceStream("ucum", "ucum-essence.xml")));
 	      fcontext.setExpansionProfile(new Parameters());

org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/tests/ProfileComparisonTests.java

@@ -1,18 +1,5 @@
 package org.hl7.fhir.validation.tests;
 
-import java.io.File;
-import java.util.UUID;
-
-import org.hl7.fhir.convertors.R5ToR5Loader;
-import org.hl7.fhir.r5.conformance.ProfileComparer;
-import org.hl7.fhir.r5.model.FhirPublication;
-import org.hl7.fhir.r5.model.StructureDefinition;
-import org.hl7.fhir.r5.utils.KeyGenerator;
-import org.hl7.fhir.utilities.cache.PackageCacheManager;
-import org.hl7.fhir.utilities.cache.ToolsVersion;
-import org.hl7.fhir.r5.test.utils.TestingUtilities;
-import org.hl7.fhir.validation.ValidationEngine;
-import org.hl7.fhir.validation.tests.utilities.TestUtilities;
 import org.junit.jupiter.api.Test;
 
 public class ProfileComparisonTests {

pom.xml

@@ -17,7 +17,7 @@
 
     <properties>
         <hapi_fhir_version>5.0.0</hapi_fhir_version>
-        <validator_test_case_version>1.1.18-SNAPSHOT</validator_test_case_version>
+        <validator_test_case_version>1.1.18</validator_test_case_version>
         <junit_jupiter_version>5.6.2</junit_jupiter_version>
         <maven_surefire_version>3.0.0-M4</maven_surefire_version>
         <jacoco_version>0.8.5</jacoco_version>
@@ -33,6 +33,20 @@
         <url>http://hl7.org</url>
     </organization>
 
+    <modules>
+        <module>org.hl7.fhir.utilities</module>
+        <module>org.hl7.fhir.dstu2</module>
+        <module>org.hl7.fhir.dstu2016may</module>
+        <module>org.hl7.fhir.dstu3</module>
+        <module>org.hl7.fhir.r4</module>
+        <module>org.hl7.fhir.r5</module>
+        <module>org.hl7.fhir.convertors</module>
+        <module>org.hl7.fhir.validation</module>
+        <module>org.hl7.fhir.validation.cli</module>
+        <!-- The report project exists only to aggregate the test results from the other projects into a central report. -->
+        <module>org.hl7.fhir.report</module>
+    </modules>
+
     <repositories>
         <repository>
             <snapshots>
@@ -122,21 +136,6 @@
         </dependencies>
     </dependencyManagement>
 
-    <modules>
-        <module>org.hl7.fhir.utilities</module>
-        <module>org.hl7.fhir.dstu2</module>
-        <module>org.hl7.fhir.dstu2016may</module>
-        <module>org.hl7.fhir.dstu3</module>
-        <module>org.hl7.fhir.r4</module>
-        <module>org.hl7.fhir.r5</module>
-        <module>org.hl7.fhir.convertors</module>
-        <module>org.hl7.fhir.validation</module>
-        <module>org.hl7.fhir.validation.cli</module>
-        <!-- The report project exists only to aggregate the test results from the other projects into a central report. -->
-        <module>org.hl7.fhir.report</module>
-    </modules>
-
-
     <build>
         <plugins>
             <plugin>
@@ -148,7 +147,7 @@
                     <target>1.8</target>
                     <forceJavacCompilerUse>false</forceJavacCompilerUse>
                     <encoding>UTF-8</encoding>
-                    <fork>true</fork>
+                    <fork>false</fork>
                     <meminitial>512m</meminitial>
                     <maxmem>4000m</maxmem>
                     <debug>true</debug>