Minimal Memory model for NpmPackage

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java

@@ -197,7 +197,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
   }
 
   private NpmPackage loadPackageInfo(String path) throws IOException {
-    NpmPackage pi = NpmPackage.fromFolder(path);
+    NpmPackage pi = minimalMemory ?  NpmPackage.fromFolderMinimal(path) : NpmPackage.fromFolder(path);
     return pi;
   }
 

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java

@@ -68,6 +68,7 @@ import org.hl7.fhir.utilities.SimpleHTTPClient;
 import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.json.JsonException;
 import org.hl7.fhir.utilities.json.model.JsonArray;
 import org.hl7.fhir.utilities.json.model.JsonElement;
 import org.hl7.fhir.utilities.json.model.JsonObject;
@@ -162,36 +163,52 @@ public class NpmPackage {
   }
 
   public class NpmPackageFolder {
-    private String name;
+    private String folderName;
-    private Map<String, List<String>> types = new HashMap<>();
+    private Map<String, List<String>> types;
-    private Map<String, byte[]> content = new HashMap<>();
+    private Map<String, byte[]> content;
-    private JsonObject index;
+    private JsonObject cachedIndex;
     private File folder;
 
-    public NpmPackageFolder(String name) {
+    public NpmPackageFolder(String folderName) {
       super();
-      this.name = name;
+      this.folderName = folderName;
+      if (!minimalMemory) {
+        types = new HashMap<>();
+        content = new HashMap<>();
+      }
     }
 
-    public Map<String, List<String>> getTypes() {
+    private String fn(String name) throws IOException {
-      return types;
+      return Utilities.path(folder.getAbsolutePath(), name);
     }
 
-    public String getName() {
+    public Map<String, List<String>> getTypes() throws JsonException, IOException {
-      return name;
+      if (minimalMemory) {
+        Map<String, List<String>> typeMap = new HashMap<>();
+        readIndex(JsonParser.parseObjectFromFile(fn(".index.json")), typeMap);
+        return typeMap;
+      } else {
+        return types;
+      }
     }
 
-    public boolean readIndex(JsonObject index) {
+    public String getFolderName() {
+      return folderName;
+    }
+
+    public boolean readIndex(JsonObject index, Map<String, List<String>> typeMap) {
       if (!index.has("index-version") || (index.asInteger("index-version") != NpmPackageIndexBuilder.CURRENT_INDEX_VERSION)) {
         return false;
       }
-      this.index = index;
+      if (!minimalMemory) {
+        this.cachedIndex = index;
+      }
       for (JsonObject file : index.getJsonObjects("files")) {
         String type = file.asString("resourceType");
         String name = file.asString("filename");
-        if (!types.containsKey(type))
+        if (!typeMap.containsKey(type))
-          types.put(type, new ArrayList<>());
+          typeMap.put(type, new ArrayList<>());
-        types.get(type).add(name);
+        typeMap.get(type).add(name);
       }
       return true;
     }
@@ -216,6 +233,8 @@ public class NpmPackage {
     }
 
     public Map<String, byte[]> getContent() {
+      assert !minimalMemory;
+      
       return content;
     }
 
@@ -242,7 +261,7 @@ public class NpmPackage {
     }
 
     public String dump() {
-      return name + " ("+ (folder == null ? "null" : folder.toString())+") | "+Boolean.toString(index != null)+" | "+content.size()+" | "+types.size();
+      return folderName + " ("+ (folder == null ? "null" : folder.toString())+")"+(minimalMemory ? "" : " | "+Boolean.toString(cachedIndex != null)+" | "+content.size()+" | "+types.size());
     }
 
     public void removeFile(String n) throws IOException {
@@ -254,13 +273,28 @@ public class NpmPackage {
       changedByLoader = true;      
     }
 
+    public JsonObject index() throws IOException {
+      if (cachedIndex != null) {
+        return cachedIndex;
+      } else if (folder == null) {
+        return null;
+      } else {
+        File ij = new File(fn(".index.json"));
+        if (ij.exists()) {
+          return JsonParser.parseObject(ij);
+        } else {
+          return null;
+        }
+      }
+    }
   }
 
   private String path;
   private JsonObject npm;
   private Map<String, NpmPackageFolder> folders = new HashMap<>();
   private boolean changedByLoader; // internal qa only!
-  private Map<String, Object> userData = new HashMap<>();
+  private Map<String, Object> userData;
+  private boolean minimalMemory;
 
   /**
    * Constructor
@@ -279,6 +313,17 @@ public class NpmPackage {
     return res;
   }
 
+  /**
+   * Factory method that parses a package from an extracted folder
+   */
+  public static NpmPackage fromFolderMinimal(String path) throws IOException {
+    NpmPackage res = new NpmPackage();
+    res.minimalMemory = true;
+    res.loadFiles(path, new File(path));
+    res.checkIndexed(path);
+    return res;
+  }
+
   /**
    * Factory method that starts a new empty package using the given PackageGenerator to create the manifest
    */
@@ -297,6 +342,9 @@ public class NpmPackage {
   }
 
   public Map<String, Object> getUserData() {
+    if (userData == null) {
+      userData = new HashMap<>();
+    }
     return userData;
   }
 
@@ -312,17 +360,19 @@ public class NpmPackage {
           if (!d.equals("package")) {
             d = Utilities.path("package", d);
           }
-          NpmPackageFolder folder = this.new NpmPackageFolder(d);
-          folder.folder = f;
-          this.folders.put(d, folder);
           File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
           if (ij.exists()) {
-            try {
+            NpmPackageFolder folder = this.new NpmPackageFolder(d);
-              if (!folder.readIndex(JsonParser.parseObject(ij))) {
+            folder.folder = f;
-                indexFolder(folder.getName(), folder);
+            this.folders.put(d, folder);
+            if (!minimalMemory) {
+              try {
+                if (!folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
+                  indexFolder(folder.getFolderName(), folder);
+                }
+              } catch (Exception e) {
+                throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
               }
-            } catch (Exception e) {
-              throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
             }
           }
           loadSubFolders(dir.getAbsolutePath(), f);
@@ -347,17 +397,19 @@ public class NpmPackage {
         if (!d.startsWith("package")) {
           d = Utilities.path("package", d);
         }
-        NpmPackageFolder folder = this.new NpmPackageFolder(d);
-        folder.folder = f;
-        this.folders.put(d, folder);
         File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
         if (ij.exists()) {
-          try {
+          NpmPackageFolder folder = this.new NpmPackageFolder(d);
-            if (!folder.readIndex(JsonParser.parseObject(ij))) {
+          folder.folder = f;
-              indexFolder(folder.getName(), folder);
+          this.folders.put(d, folder);
+          if (!minimalMemory) {
+            try {
+              if (!folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
+                indexFolder(folder.getFolderName(), folder);
+              }
+            } catch (Exception e) {
+              throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
             }
-          } catch (Exception e) {
-            throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
           }
         }
         loadSubFolders(rootPath, f);        
@@ -465,7 +517,8 @@ public class NpmPackage {
 
   private void checkIndexed(String desc) throws IOException {
     for (NpmPackageFolder folder : folders.values()) {
-      if (folder.index == null || folder.index.forceArray("files").size() == 0) {
+      JsonObject index = folder.index();
+      if (index == null || index.forceArray("files").size() == 0) {
         indexFolder(desc, folder);
       } 
     }
@@ -484,14 +537,17 @@ public class NpmPackage {
       folder.removeFile(n);
     }
     String json = indexer.build();
+    
     try {
-      folder.readIndex(JsonParser.parseObject(json));
+      if (!minimalMemory) {
+        folder.readIndex(JsonParser.parseObject(json), folder.getTypes());
+      }
       if (folder.folder != null) {
         TextFile.stringToFile(json, Utilities.path(folder.folder.getAbsolutePath(), ".index.json"));
       }
     } catch (Exception e) {
       TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json"));
-      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.name+"/.index.json: "+e.getMessage(), e);
+      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e);
     }
   }
 
@@ -580,10 +636,11 @@ public class NpmPackage {
   public List<PackageResourceInformation> listIndexedResources(List<String> types) throws IOException {
     List<PackageResourceInformation> res = new ArrayList<PackageResourceInformation>();
     for (NpmPackageFolder folder : folders.values()) {
-      if (folder.index != null) {
+      JsonObject index = folder.index();
-        for (JsonObject fi : folder.index.getJsonObjects("files")) {
+      if (index != null) {
+        for (JsonObject fi : index.getJsonObjects("files")) {
           if (Utilities.existsInList(fi.asString("resourceType"), types) || types.isEmpty()) {
-            res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getName() : folder.folder.getAbsolutePath(), fi));
+            res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getFolderName() : folder.folder.getAbsolutePath(), fi));
           }
         }
       }
@@ -653,7 +710,7 @@ public class NpmPackage {
   public InputStream loadByCanonicalVersion(String folder, String canonical, String version) throws IOException {
     NpmPackageFolder f = folders.get(folder);
     List<JsonObject> matches = new ArrayList<>();
-    for (JsonObject file : f.index.getJsonObjects("files")) {
+    for (JsonObject file : f.index().getJsonObjects("files")) {
       if (canonical.equals(file.asString("url"))) {
         if (version != null && version.equals(file.asString("version"))) {
           return load("package", file.asString("filename"));
@@ -877,7 +934,7 @@ public class NpmPackage {
 
   public InputStream loadResource(String type, String id) throws IOException {
     NpmPackageFolder f = folders.get("package");
-    JsonArray files = f.index.getJsonArray("files");
+    JsonArray files = f.index().getJsonArray("files");
     for (JsonElement e : files.getItems()) {
       JsonObject i = (JsonObject) e;
       if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
@@ -893,7 +950,7 @@ public class NpmPackage {
       f = folders.get("package/example");      
     }
     if (f != null) {
-      JsonArray files = f.index.getJsonArray("files");
+      JsonArray files = f.index().getJsonArray("files");
       for (JsonElement e : files.getItems()) {
         JsonObject i = (JsonObject) e;
         if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
@@ -910,6 +967,7 @@ public class NpmPackage {
   }
 
   public void save(File directory) throws IOException {
+    assert !minimalMemory;
     File dir = new File(Utilities.path(directory.getAbsolutePath(), name()));
     if (!dir.exists()) {
       Utilities.createDirectory(dir.getAbsolutePath());
@@ -918,7 +976,7 @@ public class NpmPackage {
     }
     
     for (NpmPackageFolder folder : folders.values()) {
-      String n = folder.name;
+      String n = folder.folderName;
 
       File pd = new File(Utilities.path(dir.getAbsolutePath(), n));
       if (!pd.exists()) {
@@ -941,6 +999,7 @@ public class NpmPackage {
   }
   
   public void save(OutputStream stream) throws IOException {
+    assert !minimalMemory;
     TarArchiveOutputStream tar;
     ByteArrayOutputStream OutputStream;
     BufferedOutputStream bufferedOutputStream;
@@ -955,7 +1014,7 @@ public class NpmPackage {
 
 
     for (NpmPackageFolder folder : folders.values()) {
-      String n = folder.name;
+      String n = folder.folderName;
       if (!"package".equals(n) && !(n.startsWith("package/") || n.startsWith("package\\"))) {
         n = "package/"+n;
       }
@@ -1043,8 +1102,10 @@ public class NpmPackage {
   }
 
   public void unPack(String dir, boolean withAppend) throws IOException {
+    assert !minimalMemory;
+    
     for (NpmPackageFolder folder : folders.values()) {
-      String dn = folder.getName();
+      String dn = folder.getFolderName();
       if (!dn.equals("package") && (dn.startsWith("package/") || dn.startsWith("package\\"))) {
         dn = dn.substring(8);
       }
@@ -1099,6 +1160,8 @@ public class NpmPackage {
   }
 
   public void addFile(String folderName, String name, byte[] cnt, String type) {
+    assert !minimalMemory;
+    
     if (!folders.containsKey(folderName)) {
       folders.put(folderName, new NpmPackageFolder(folderName));
     }
@@ -1154,7 +1217,7 @@ public class NpmPackage {
     return npm.asString("name").startsWith("hl7.terminology");
   }
 
-  public boolean hasCanonical(String url) {
+  public boolean hasCanonical(String url) throws IOException {
     if (url == null) {
       return false;
     }
@@ -1162,7 +1225,7 @@ public class NpmPackage {
     String v = url.contains("|") ?  url.substring(url.indexOf("|")+1) : null;
     NpmPackageFolder folder = folders.get("package");
     if (folder != null) {
-      for (JsonObject o : folder.index.getJsonObjects("files")) {
+      for (JsonObject o : folder.index().getJsonObjects("files")) {
         if (u.equals(o.asString("url"))) {
           if (v == null || v.equals(o.asString("version"))) {
             return true;

org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java

@@ -4,11 +4,13 @@ import java.io.File;
 import java.io.IOException;
 import java.util.List;
 
+import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.npm.CommonPackages;
 import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager.FilesystemPackageCacheMode;
 import org.hl7.fhir.utilities.npm.NpmPackage;
+import org.hl7.fhir.utilities.npm.NpmPackage.NpmPackageFolder;
 import org.hl7.fhir.utilities.npm.ToolsVersion;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -66,4 +68,23 @@ public class PackageCacheTests {
     Assertions.assertEquals("0.0.8", cache.loadPackage(CommonPackages.ID_PUBPACK, "0.0.8").version());
     Assertions.assertEquals(CommonPackages.VER_PUBPACK, cache.loadPackage(CommonPackages.ID_PUBPACK).version());    
   }
+  
+  @Test
+  public void testMinimal() throws IOException {
+    FilesystemPackageCacheManager cache = new FilesystemPackageCacheManager(FilesystemPackageCacheMode.TESTING);
+    cache.clear();  
+    NpmPackage uscore = cache.loadPackage("hl7.fhir.us.core", "3.1.0");
+    cache.setMinimalMemory(true);
+    NpmPackage uscoreMin = cache.loadPackage("hl7.fhir.us.core", "3.1.0");
+    Assertions.assertEquals(uscore.version(), uscoreMin.version());
+    Assertions.assertEquals(uscore.getFolders().size(), uscoreMin.getFolders().size());
+    Assertions.assertEquals(uscore.list("package").size(), uscoreMin.list("package").size());
+    byte[] b1 = TextFile.streamToBytes(uscore.load(uscore.list("package").get(1)));
+    byte[] b2 = TextFile.streamToBytes(uscoreMin.load(uscore.list("package").get(1)));
+//    TextFile.bytesToFile(b1, "/Users/grahamegrieve/temp/b1.json");
+//    TextFile.bytesToFile(b2, "/Users/grahamegrieve/temp/b2.json");
+    Assertions.assertArrayEquals(b1, b2);
+    
+    
+  }
 }