honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Minimal Memory model for NpmPackage

This commit is contained in:
Grahame Grieve 2023-06-22 15:05:59 +10:00
parent 74c30d9143
commit 0615f2a163
3 changed files with 132 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java
View File

@ -197,7 +197,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
}
private NpmPackage loadPackageInfo(String path) throws IOException {
NpmPackage pi = NpmPackage.fromFolder(path);
NpmPackage pi = minimalMemory ? NpmPackage.fromFolderMinimal(path) : NpmPackage.fromFolder(path);
return pi;
}

157
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java
View File

@ -68,6 +68,7 @@ import org.hl7.fhir.utilities.SimpleHTTPClient;
import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.json.JsonException;
import org.hl7.fhir.utilities.json.model.JsonArray;
import org.hl7.fhir.utilities.json.model.JsonElement;
import org.hl7.fhir.utilities.json.model.JsonObject;
@ -162,36 +163,52 @@ public class NpmPackage {
}
public class NpmPackageFolder {
private String name;
private Map<String, List<String>> types = new HashMap<>();
private Map<String, byte[]> content = new HashMap<>();
private JsonObject index;
private String folderName;
private Map<String, List<String>> types;
private Map<String, byte[]> content;
private JsonObject cachedIndex;
private File folder;
public NpmPackageFolder(String name) {
public NpmPackageFolder(String folderName) {
super();
this.name = name;
this.folderName = folderName;
if (!minimalMemory) {
types = new HashMap<>();
content = new HashMap<>();
}
}
public Map<String, List<String>> getTypes() {
return types;
private String fn(String name) throws IOException {
return Utilities.path(folder.getAbsolutePath(), name);
}
public String getName() {
return name;
public Map<String, List<String>> getTypes() throws JsonException, IOException {
if (minimalMemory) {
Map<String, List<String>> typeMap = new HashMap<>();
readIndex(JsonParser.parseObjectFromFile(fn(".index.json")), typeMap);
return typeMap;
} else {
return types;
}
}
public boolean readIndex(JsonObject index) {
public String getFolderName() {
return folderName;
}
public boolean readIndex(JsonObject index, Map<String, List<String>> typeMap) {
if (!index.has("index-version") || (index.asInteger("index-version") != NpmPackageIndexBuilder.CURRENT_INDEX_VERSION)) {
return false;
}
this.index = index;
if (!minimalMemory) {
this.cachedIndex = index;
}
for (JsonObject file : index.getJsonObjects("files")) {
String type = file.asString("resourceType");
String name = file.asString("filename");
if (!types.containsKey(type))
types.put(type, new ArrayList<>());
types.get(type).add(name);
if (!typeMap.containsKey(type))
typeMap.put(type, new ArrayList<>());
typeMap.get(type).add(name);
}
return true;
}
@ -216,6 +233,8 @@ public class NpmPackage {
}
public Map<String, byte[]> getContent() {
assert !minimalMemory;
return content;
}
@ -242,7 +261,7 @@ public class NpmPackage {
}
public String dump() {
return name + " ("+ (folder == null ? "null" : folder.toString())+") | "+Boolean.toString(index != null)+" | "+content.size()+" | "+types.size();
return folderName + " ("+ (folder == null ? "null" : folder.toString())+")"+(minimalMemory ? "" : " | "+Boolean.toString(cachedIndex != null)+" | "+content.size()+" | "+types.size());
}
public void removeFile(String n) throws IOException {
@ -254,13 +273,28 @@ public class NpmPackage {
changedByLoader = true;
}
public JsonObject index() throws IOException {
if (cachedIndex != null) {
return cachedIndex;
} else if (folder == null) {
return null;
} else {
File ij = new File(fn(".index.json"));
if (ij.exists()) {
return JsonParser.parseObject(ij);
} else {
return null;
}
}
}
}
private String path;
private JsonObject npm;
private Map<String, NpmPackageFolder> folders = new HashMap<>();
private boolean changedByLoader; // internal qa only!
private Map<String, Object> userData = new HashMap<>();
private Map<String, Object> userData;
private boolean minimalMemory;
/**
* Constructor
@ -279,6 +313,17 @@ public class NpmPackage {
return res;
}
/**
* Factory method that parses a package from an extracted folder
*/
public static NpmPackage fromFolderMinimal(String path) throws IOException {
NpmPackage res = new NpmPackage();
res.minimalMemory = true;
res.loadFiles(path, new File(path));
res.checkIndexed(path);
return res;
}
/**
* Factory method that starts a new empty package using the given PackageGenerator to create the manifest
*/
@ -297,6 +342,9 @@ public class NpmPackage {
}
public Map<String, Object> getUserData() {
if (userData == null) {
userData = new HashMap<>();
}
return userData;
}
@ -312,17 +360,19 @@ public class NpmPackage {
if (!d.equals("package")) {
d = Utilities.path("package", d);
}
NpmPackageFolder folder = this.new NpmPackageFolder(d);
folder.folder = f;
this.folders.put(d, folder);
File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
if (ij.exists()) {
try {
if (!folder.readIndex(JsonParser.parseObject(ij))) {
indexFolder(folder.getName(), folder);
NpmPackageFolder folder = this.new NpmPackageFolder(d);
folder.folder = f;
this.folders.put(d, folder);
if (!minimalMemory) {
try {
if (!folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
indexFolder(folder.getFolderName(), folder);
}
} catch (Exception e) {
throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
}
} catch (Exception e) {
throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
}
}
loadSubFolders(dir.getAbsolutePath(), f);
@ -347,17 +397,19 @@ public class NpmPackage {
if (!d.startsWith("package")) {
d = Utilities.path("package", d);
}
NpmPackageFolder folder = this.new NpmPackageFolder(d);
folder.folder = f;
this.folders.put(d, folder);
File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
if (ij.exists()) {
try {
if (!folder.readIndex(JsonParser.parseObject(ij))) {
indexFolder(folder.getName(), folder);
NpmPackageFolder folder = this.new NpmPackageFolder(d);
folder.folder = f;
this.folders.put(d, folder);
if (!minimalMemory) {
try {
if (!folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
indexFolder(folder.getFolderName(), folder);
}
} catch (Exception e) {
throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
}
} catch (Exception e) {
throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
}
}
loadSubFolders(rootPath, f);
@ -465,7 +517,8 @@ public class NpmPackage {
private void checkIndexed(String desc) throws IOException {
for (NpmPackageFolder folder : folders.values()) {
if (folder.index == null || folder.index.forceArray("files").size() == 0) {
JsonObject index = folder.index();
if (index == null || index.forceArray("files").size() == 0) {
indexFolder(desc, folder);
}
}
@ -484,14 +537,17 @@ public class NpmPackage {
folder.removeFile(n);
}
String json = indexer.build();
try {
folder.readIndex(JsonParser.parseObject(json));
if (!minimalMemory) {
folder.readIndex(JsonParser.parseObject(json), folder.getTypes());
}
if (folder.folder != null) {
TextFile.stringToFile(json, Utilities.path(folder.folder.getAbsolutePath(), ".index.json"));
}
} catch (Exception e) {
TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json"));
throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.name+"/.index.json: "+e.getMessage(), e);
throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e);
}
}
@ -580,10 +636,11 @@ public class NpmPackage {
public List<PackageResourceInformation> listIndexedResources(List<String> types) throws IOException {
List<PackageResourceInformation> res = new ArrayList<PackageResourceInformation>();
for (NpmPackageFolder folder : folders.values()) {
if (folder.index != null) {
for (JsonObject fi : folder.index.getJsonObjects("files")) {
JsonObject index = folder.index();
if (index != null) {
for (JsonObject fi : index.getJsonObjects("files")) {
if (Utilities.existsInList(fi.asString("resourceType"), types) || types.isEmpty()) {
res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getName() : folder.folder.getAbsolutePath(), fi));
res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getFolderName() : folder.folder.getAbsolutePath(), fi));
}
}
}
@ -653,7 +710,7 @@ public class NpmPackage {
public InputStream loadByCanonicalVersion(String folder, String canonical, String version) throws IOException {
NpmPackageFolder f = folders.get(folder);
List<JsonObject> matches = new ArrayList<>();
for (JsonObject file : f.index.getJsonObjects("files")) {
for (JsonObject file : f.index().getJsonObjects("files")) {
if (canonical.equals(file.asString("url"))) {
if (version != null && version.equals(file.asString("version"))) {
return load("package", file.asString("filename"));
@ -877,7 +934,7 @@ public class NpmPackage {
public InputStream loadResource(String type, String id) throws IOException {
NpmPackageFolder f = folders.get("package");
JsonArray files = f.index.getJsonArray("files");
JsonArray files = f.index().getJsonArray("files");
for (JsonElement e : files.getItems()) {
JsonObject i = (JsonObject) e;
if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
@ -893,7 +950,7 @@ public class NpmPackage {
f = folders.get("package/example");
}
if (f != null) {
JsonArray files = f.index.getJsonArray("files");
JsonArray files = f.index().getJsonArray("files");
for (JsonElement e : files.getItems()) {
JsonObject i = (JsonObject) e;
if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
@ -910,6 +967,7 @@ public class NpmPackage {
}
public void save(File directory) throws IOException {
assert !minimalMemory;
File dir = new File(Utilities.path(directory.getAbsolutePath(), name()));
if (!dir.exists()) {
Utilities.createDirectory(dir.getAbsolutePath());
@ -918,7 +976,7 @@ public class NpmPackage {
}
for (NpmPackageFolder folder : folders.values()) {
String n = folder.name;
String n = folder.folderName;
File pd = new File(Utilities.path(dir.getAbsolutePath(), n));
if (!pd.exists()) {
@ -941,6 +999,7 @@ public class NpmPackage {
}
public void save(OutputStream stream) throws IOException {
assert !minimalMemory;
TarArchiveOutputStream tar;
ByteArrayOutputStream OutputStream;
BufferedOutputStream bufferedOutputStream;
@ -955,7 +1014,7 @@ public class NpmPackage {
for (NpmPackageFolder folder : folders.values()) {
String n = folder.name;
String n = folder.folderName;
if (!"package".equals(n) && !(n.startsWith("package/") || n.startsWith("package\\"))) {
n = "package/"+n;
}
@ -1043,8 +1102,10 @@ public class NpmPackage {
}
public void unPack(String dir, boolean withAppend) throws IOException {
assert !minimalMemory;
for (NpmPackageFolder folder : folders.values()) {
String dn = folder.getName();
String dn = folder.getFolderName();
if (!dn.equals("package") && (dn.startsWith("package/") || dn.startsWith("package\\"))) {
dn = dn.substring(8);
}
@ -1099,6 +1160,8 @@ public class NpmPackage {
}
public void addFile(String folderName, String name, byte[] cnt, String type) {
assert !minimalMemory;
if (!folders.containsKey(folderName)) {
folders.put(folderName, new NpmPackageFolder(folderName));
}
@ -1154,7 +1217,7 @@ public class NpmPackage {
return npm.asString("name").startsWith("hl7.terminology");
}
public boolean hasCanonical(String url) {
public boolean hasCanonical(String url) throws IOException {
if (url == null) {
return false;
}
@ -1162,7 +1225,7 @@ public class NpmPackage {
String v = url.contains("|") ? url.substring(url.indexOf("|")+1) : null;
NpmPackageFolder folder = folders.get("package");
if (folder != null) {
for (JsonObject o : folder.index.getJsonObjects("files")) {
for (JsonObject o : folder.index().getJsonObjects("files")) {
if (u.equals(o.asString("url"))) {
if (v == null || v.equals(o.asString("version"))) {
return true;

21
org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java
View File

@ -4,11 +4,13 @@ import java.io.File;
import java.io.IOException;
import java.util.List;
import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.npm.CommonPackages;
import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager.FilesystemPackageCacheMode;
import org.hl7.fhir.utilities.npm.NpmPackage;
import org.hl7.fhir.utilities.npm.NpmPackage.NpmPackageFolder;
import org.hl7.fhir.utilities.npm.ToolsVersion;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@ -66,4 +68,23 @@ public class PackageCacheTests {
Assertions.assertEquals("0.0.8", cache.loadPackage(CommonPackages.ID_PUBPACK, "0.0.8").version());
Assertions.assertEquals(CommonPackages.VER_PUBPACK, cache.loadPackage(CommonPackages.ID_PUBPACK).version());
}
@Test
public void testMinimal() throws IOException {
FilesystemPackageCacheManager cache = new FilesystemPackageCacheManager(FilesystemPackageCacheMode.TESTING);
cache.clear();
NpmPackage uscore = cache.loadPackage("hl7.fhir.us.core", "3.1.0");
cache.setMinimalMemory(true);
NpmPackage uscoreMin = cache.loadPackage("hl7.fhir.us.core", "3.1.0");
Assertions.assertEquals(uscore.version(), uscoreMin.version());
Assertions.assertEquals(uscore.getFolders().size(), uscoreMin.getFolders().size());
Assertions.assertEquals(uscore.list("package").size(), uscoreMin.list("package").size());
byte[] b1 = TextFile.streamToBytes(uscore.load(uscore.list("package").get(1)));
byte[] b2 = TextFile.streamToBytes(uscoreMin.load(uscore.list("package").get(1)));
// TextFile.bytesToFile(b1, "/Users/grahamegrieve/temp/b1.json");
// TextFile.bytesToFile(b2, "/Users/grahamegrieve/temp/b2.json");
Assertions.assertArrayEquals(b1, b2);
}
}