fix bugs downloading packages

org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/NpmPackageTests.java

@@ -2,6 +2,8 @@ package org.hl7.fhir.r5.test;
 
 import static org.junit.Assert.*;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -54,6 +56,16 @@ public class NpmPackageTests {
     checkNpm(npm);
   }
 
+  @Test
+  public void testSave() throws IOException {
+    NpmPackage npm = NpmPackage.fromPackage(TestingUtilities.loadTestResourceStream("npm", "test.format.old.tgz"));
+    ByteArrayOutputStream bs = new ByteArrayOutputStream();
+    npm.save(bs);
+    npm = NpmPackage.fromPackage(new ByteArrayInputStream(bs.toByteArray()));
+    checkNpm(npm);
+  }
+
+
   private void checkNpm(NpmPackage npm) throws IOException {
     Assert.assertEquals(1, npm.list("other").size());
     Assert.assertEquals("help.png", npm.list("other").get(0));

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java

@@ -84,7 +84,7 @@ public class NpmPackage {
     return ver.matches("^[0-9]+\\.[0-9]+\\.[0-9]+$");
   }
 
-  private class NpmPackageFolder {
+  public class NpmPackageFolder {
     private String name;
     private Map<String, List<String>> types = new HashMap<>();
     private Map<String, byte[]> content = new HashMap<String, byte[]>(); 
@@ -96,6 +96,10 @@ public class NpmPackage {
       this.name = name;
     }
 
+    public String getName() {
+      return name;
+    }
+
     public void readIndex(JsonObject index) {
       this.index = index;
       for (JsonElement e : index.getAsJsonArray("files")) {
@@ -127,6 +131,10 @@ public class NpmPackage {
       return res;
     }
 
+    public Map<String, byte[]> getContent() {
+      return content;
+    }
+
     public byte[] fetchFile(String file) throws FileNotFoundException, IOException {
       if (folder != null) {
         File f = new File(Utilities.path(folder.getAbsolutePath(), file));
@@ -263,18 +271,17 @@ public class NpmPackage {
           fos.close();
           loadFile(n, fos.toByteArray());
         }
-      }
+        if (progress && i % 50 == 0) {
-      i++;
+          c++;
-      if (progress && i % 50 == 0) {
+          System.out.print(".");
-        c++;
+          if (c == 120) {
-        System.out.print(".");
+            System.out.println("");
-        if (c == 120) {
+            System.out.print("  ");
-          System.out.println("");
+            c = 2;
-          System.out.print("  ");
+          }
-          c = 2;
         }
-      }    
+      }
-    }
+    } 
     try {
       npm = JsonTrackingParser.parseJson(folders.get("package").fetchFile("package.json"));
     } catch (Exception e) {
@@ -569,8 +576,8 @@ public class NpmPackage {
   }
 
   /** special case when playing around inside the package **/
-  public Map<String, byte[]> getContent() {
+  public Map<String, NpmPackageFolder> getFolders() {
-    return folders.get("package").content;
+    return folders;
   }
 
   //
@@ -624,9 +631,9 @@ public class NpmPackage {
       for (String s : folder.content.keySet()) {
         byte[] b = folder.content.get(s);
         String name = n+"/"+s;
-        indexer.seeFile(n, b);
+        indexer.seeFile(s, b);
         if (!s.equals(".index.json") && !s.equals("package.json")) {
-          TarArchiveEntry entry = new TarArchiveEntry(s);
+          TarArchiveEntry entry = new TarArchiveEntry(name);
           entry.setSize(b.length);
           tar.putArchiveEntry(entry);
           tar.write(b);
@@ -634,7 +641,7 @@ public class NpmPackage {
         }
       }
       byte[] cnt = indexer.build().getBytes(Charset.forName("UTF-8"));
-      TarArchiveEntry entry = new TarArchiveEntry(n+".index.json");
+      TarArchiveEntry entry = new TarArchiveEntry(n+"/.index.json");
       entry.setSize(cnt.length);
       tar.putArchiveEntry(entry);
       tar.write(cnt);

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java

@@ -47,6 +47,7 @@ import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.utilities.IniFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder;
 import org.hl7.fhir.utilities.json.JSONUtil;
 
 import com.google.gson.GsonBuilder;
@@ -181,29 +182,7 @@ public class PackageCacheManager {
     return cacheFolder;
   }
 
-  private void analysePackage(String dir) throws IOException {
+
-    NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
-    indexer.start();
-    int i = 0;
-    int c = 0;
-    File[] packages = new File(Utilities.path(dir, "package")).listFiles();
-    for (File f : packages) {
-      if (!f.isDirectory()) {
-        indexer.seeFile(f.getName(), TextFile.fileToBytes(f));
-      }
-      i++;
-      if (progress && i % 50 == 0) {
-        c++;
-        System.out.print(".");
-        if (c == 120) {
-          System.out.println("");
-          System.out.print("  ");
-          c = 2;
-        }
-      }    
-    }
-    TextFile.stringToFile(indexer.build(), Utilities.path(dir, "package", ".index.json"));
-  }
 
 
 
@@ -531,8 +510,10 @@ public class PackageCacheManager {
     
     recordMap(npm.canonical(), npm.name());
     
-    if (progress )
+    if (progress ) {
-      System.out.print("|");
+      System.out.println();
+      System.out.print("  Installing: ");
+    }
     if (npm.name() == null || id == null || !id.equals(npm.name())) {
       if (!id.equals("hl7.fhir.r5.core")) {// temporary work around
         throw new IOException("Attempt to import a mis-identified package. Expected "+id+", got "+npm.name());
@@ -548,30 +529,28 @@ public class PackageCacheManager {
     int i = 0;
     int c = 0;
     int size = 0;
-    for (Entry<String, byte[]> e : npm.getContent().entrySet()) {
+    for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
-      String fn = Utilities.path(packRoot, "package", e.getKey());
+      String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package",  e.getKey());;
-      String dir = Utilities.getDirectoryForFile(fn);
       if (!(new File(dir).exists()))
         Utilities.createDirectory(dir);
-      TextFile.bytesToFile(e.getValue(), fn);
+      for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
-      size = size + e.getValue().length;
+        String fn = Utilities.path(dir, fe.getKey());
-      i++;
+        byte[] cnt = fe.getValue();
-      if (progress && i % 50 == 0) {
+        TextFile.bytesToFile(cnt, fn);
-        c++;
+        size = size + cnt.length;
-        System.out.print(".");
+        i++;
-        if (c == 120) {
+        if (progress && i % 50 == 0) {
-          System.out.println("");
+          c++;
-          System.out.print("  ");
+          System.out.print(".");
-          c = 2;
+          if (c == 120) {
-        }
+            System.out.println("");
-      }    
+            System.out.print("  ");
+            c = 2;
+          }
+        }    
+      }
     }
-    if (progress) {
+ 
-      System.out.println("");
-      System.out.print("  Analysing");
-    }      
-    
-    analysePackage(packRoot); // do this whether there's an index.json or not.
     
     IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
     ini.setTimeStampFormat("yyyyMMddhhmmss");