honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NPM rework

This commit is contained in:
jamesagnew 2020-05-27 05:27:44 -04:00
parent 37f6a6ed8c
commit c0f6366626
8 changed files with 179 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java vendored
View File

@ -1,14 +1,19 @@
package org.hl7.fhir.utilities.cache;
import org.apache.commons.lang3.Validate;
import org.hl7.fhir.utilities.Utilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
public abstract class BasePackageCacheManager implements IPackageCacheManager {
private static final Logger ourLog = LoggerFactory.getLogger(BasePackageCacheManager.class);
private List<String> myPackageServers = new ArrayList<>();
/**
@ -35,16 +40,37 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager {
/**
* Load the latest version of the identified package from the cache - it it exists
*
* @param id
* @return
* @throws IOException
*/
public NpmPackage loadPackageFromCacheOnly(String id) throws IOException {
return loadPackageFromCacheOnly(id, null);
}
protected abstract NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException;
/**
* Try to load a package from all registered package servers, and return <code>null</code>
* if it can not be found at any of them.
*/
@Nullable
protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
for (String nextPackageServer : getPackageServers()) {
CachingPackageClient packageClient = new CachingPackageClient(nextPackageServer);
try {
if (Utilities.noString(version)) {
version = packageClient.getLatestVersion(id);
}
InputStream stream = packageClient.fetch(id, version);
String url = packageClient.url(id, version);
return new InputStreamWithSrc(stream, packageClient.url(id, version), version);
} catch (IOException e) {
ourLog.info("Failed to resolve package {}#{} from server: {}", id, version, nextPackageServer);
}
}
return null;
}
protected abstract NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException;
@Override
public String getPackageUrl(String packageId) throws IOException {
@ -66,8 +92,8 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager {
private String getPackageUrl(String packageId, String server) throws IOException {
PackageClient pc = new PackageClient(server);
List<PackageClient.PackageInfo> res = pc.search(packageId, null, null, false);
CachingPackageClient pc = new CachingPackageClient(server);
List<CachingPackageClient.PackageInfo> res = pc.search(packageId, null, null, false);
if (res.size() == 0) {
return null;
} else {
@ -91,13 +117,13 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager {
}
private String getPackageId(String canonical, String server) throws IOException {
PackageClient pc = new PackageClient(server);
List<PackageClient.PackageInfo> res = pc.search(null, canonical, null, false);
CachingPackageClient pc = new CachingPackageClient(server);
List<CachingPackageClient.PackageInfo> res = pc.search(null, canonical, null, false);
if (res.size() == 0) {
return null;
} else {
// this is driven by HL7 Australia (http://hl7.org.au/fhir/ is the canonical url for the base package, and the root for all the others)
for (PackageClient.PackageInfo pi : res) {
for (CachingPackageClient.PackageInfo pi : res) {
if (canonical.equals(pi.getCanonical())) {
return pi.getId();
}
@ -106,5 +132,17 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager {
}
}
public class InputStreamWithSrc {
public InputStream stream;
public String url;
public String version;
public InputStreamWithSrc(InputStream stream, String url, String version) {
this.stream = stream;
this.url = url;
this.version = version;
}
}
}

34
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageClient.java → org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageClient.java vendored
View File

@ -1,5 +1,14 @@
package org.hl7.fhir.utilities.cache;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.VersionUtilities;
import org.hl7.fhir.utilities.json.JSONUtil;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@ -13,19 +22,7 @@ import java.util.Date;
import java.util.List;
import java.util.Set;
import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.VersionUtilities;
import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
import org.hl7.fhir.utilities.json.JSONUtil;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
public class PackageClient {
public abstract class BasePackageClient {
public class PackageInfo {
private String id;
@ -76,7 +73,7 @@ public class PackageClient {
private String cacheFolder;
public PackageClient(String address) {
public BasePackageClient(String address) {
super();
this.address = address;
try {
@ -109,15 +106,10 @@ public class PackageClient {
}
public InputStream fetchCached(String url) throws IOException, FileNotFoundException {
File cacheFile = new File(Utilities.path(cacheFolder, fn(url)));
if (cacheFile.exists()) {
return new FileInputStream(cacheFile);
}
TextFile.bytesToFile(TextFile.streamToBytes(fetchUrl(url, null)), cacheFile);
return new FileInputStream(cacheFile);
return fetchUrl(url, null);
}
private String fn(String url) {
protected String fn(String url) {
String[] p = url.split("\\/");
return p[2]+"-"+p[p.length-2]+"-"+p[p.length-1]+".tgz";
}

71
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java vendored Normal file
View File

@ -0,0 +1,71 @@
package org.hl7.fhir.utilities.cache;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.VersionUtilities;
import org.hl7.fhir.utilities.json.JSONUtil;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
/**
* Implementation of a package client that keeps a local disk cache of downloaded artifacts
* in order to avoid re-downloading things
*/
public class CachingPackageClient extends BasePackageClient {
private String cacheFolder;
public CachingPackageClient(String address) {
super(address);
try {
cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "package-client");
Utilities.createDirectory(cacheFolder);
} catch (IOException e) {
}
}
public boolean exists(String id, String ver) throws IOException {
List<PackageInfo> vl = getVersions(id);
for (PackageInfo pi : vl) {
if (ver.equals(pi.getVersion())) {
return true;
}
}
return false;
}
public InputStream fetchCached(String url) throws IOException {
File cacheFile = new File(Utilities.path(cacheFolder, fn(url)));
if (cacheFile.exists()) {
return new FileInputStream(cacheFile);
}
InputStream fetchedPackage = super.fetchCached(url);
TextFile.bytesToFile(TextFile.streamToBytes(fetchedPackage), cacheFile);
return new FileInputStream(cacheFile);
}
public Date getNewPackages(Date lastCalled, List<PackageInfo> updates) {
return null;
}
}

48
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java vendored
View File

@ -40,7 +40,6 @@ import org.hl7.fhir.utilities.IniFile;
import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder;
import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
import org.hl7.fhir.utilities.json.JSONUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -67,7 +66,8 @@ import java.util.Map;
import java.util.Map.Entry;
/**
* Package cache manager
* This is a package cache manager implementation that uses a local disk cache
*
* <p>
* API:
* <p>
@ -200,30 +200,19 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
private void listSpecs(Map<String, String> specList, String server) throws IOException {
PackageClient pc = new PackageClient(server);
List<PackageInfo> matches = pc.search(null, null, null, false);
for (PackageInfo m : matches) {
CachingPackageClient pc = new CachingPackageClient(server);
List<BasePackageClient.PackageInfo> matches = pc.search(null, null, null, false);
for (BasePackageClient.PackageInfo m : matches) {
if (!specList.containsKey(m.getId())) {
specList.put(m.getId(), m.getUrl());
}
}
}
private InputStreamWithSrc loadFromPackageServer(String id, String version) {
for (String nextPackageServer : getPackageServers()) {
PackageClient packageClient = new PackageClient(nextPackageServer);
try {
if (Utilities.noString(version)) {
version = packageClient.getLatestVersion(id);
}
InputStream stream = packageClient.fetch(id, version);
String url = packageClient.url(id, version);
return new InputStreamWithSrc(stream, packageClient.url(id, version), version);
} catch (IOException e) {
ourLog.info("Failed to resolve package {}#{} from server: {}", id, version, nextPackageServer);
}
protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
InputStreamWithSrc retVal = super.loadFromPackageServer(id, version);
if (retVal != null) {
return retVal;
}
// ok, well, we'll try the old way
@ -232,7 +221,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
public String getLatestVersion(String id) throws IOException {
for (String nextPackageServer : getPackageServers()) {
PackageClient pc = new PackageClient(nextPackageServer);
CachingPackageClient pc = new CachingPackageClient(nextPackageServer);
try {
return pc.getLatestVersion(id);
} catch (IOException e) {
@ -443,6 +432,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
if (!Utilities.noString(version) && version.startsWith("file:")) {
return loadPackageFromFile(id, version.substring(5));
}
NpmPackage p = loadPackageFromCacheOnly(id, version);
if (p != null) {
if ("current".equals(version)) {
@ -461,7 +451,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
}
// nup, don't have it locally (or it's expired)
InputStreamWithSrc source;
FilesystemPackageCacheManager.InputStreamWithSrc source;
if ("current".equals(version)) {
// special case - fetch from ci-build server
source = loadFromCIBuild(id);
@ -471,6 +461,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
return addPackageToCache(id, version == null ? source.version : version, source.stream, source.url);
}
private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
try {
URL url = new URL(source);
@ -732,19 +723,6 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
InputStream resolvePackage(String packageId, String version);
}
public class InputStreamWithSrc {
public InputStream stream;
public String url;
public String version;
public InputStreamWithSrc(InputStream stream, String url, String version) {
this.stream = stream;
this.url = url;
this.version = version;
}
}
public class BuildRecordSorter implements Comparator<BuildRecord> {
@Override

1
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java vendored
View File

@ -7,7 +7,6 @@ import java.io.InputStream;
public interface IPackageCacheManager {
String getPackageId(String canonicalUrl) throws IOException;
NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException;

21
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java vendored
View File

@ -102,7 +102,7 @@ public class NpmPackage {
public class NpmPackageFolder {
private String name;
private Map<String, List<String>> types = new HashMap<>();
private Map<String, byte[]> content = new HashMap<String, byte[]>();
private Map<String, byte[]> content = new HashMap<>();
private JsonObject index;
private File folder;
@ -111,6 +111,10 @@ public class NpmPackage {
this.name = name;
}
public Map<String, List<String>> getTypes() {
return types;
}
public String getName() {
return name;
}
@ -192,15 +196,12 @@ public class NpmPackage {
private Map<String, NpmPackageFolder> folders = new HashMap<>();
private boolean changedByLoader; // internal qa only!
/**
* Constructor
*/
private NpmPackage() {
super();
}
// public NpmPackage(JsonObject npm, Map<String, byte[]> content, List<String> folders) {
// this.path = null;
// this.content = content;
// this.npm = npm;
// this.folders = folders;
// }
public static NpmPackage fromFolder(String path) throws IOException {
NpmPackage res = new NpmPackage();
@ -839,7 +840,9 @@ public class NpmPackage {
stream.write(b);
}
/**
* Keys are resource type names, values are filenames
*/
public Map<String, List<String>> getTypes() {
return folders.get("package").types;
}

28
org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageClientTests.java → org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java
View File

@ -1,18 +1,18 @@
package org.hl7.fhir.utilities.tests;
import org.hl7.fhir.utilities.cache.PackageClient;
import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
import org.hl7.fhir.utilities.cache.CachingPackageClient;
import org.hl7.fhir.utilities.cache.CachingPackageClient.PackageInfo;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
public class PackageClientTests {
public class CachingPackageClientTests {
@Test
public void testExists() throws IOException {
PackageClient client = new PackageClient("http://packages.fhir.org");
CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
Assertions.assertTrue(client.exists("hl7.fhir.r4.core", "4.0.1"));
Assertions.assertTrue(!client.exists("hl7.fhir.r4.core", "1.0.2"));
Assertions.assertTrue(!client.exists("hl7.fhir.nothing", "1.0.1"));
@ -20,7 +20,7 @@ public class PackageClientTests {
@Test
public void testSearch() throws IOException {
PackageClient client = new PackageClient("http://packages.fhir.org");
CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
List<PackageInfo> matches = client.search("core", null, null, false);
for (PackageInfo pi : matches) {
System.out.println(pi.toString());
@ -30,14 +30,14 @@ public class PackageClientTests {
@Test
public void testSearchNoMatches() throws IOException {
PackageClient client = new PackageClient("http://packages.fhir.org");
CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
List<PackageInfo> matches = client.search("corezxxx", null, null, false);
Assertions.assertTrue(matches.size() == 0);
}
@Test
public void testVersions() throws IOException {
PackageClient client = new PackageClient("http://packages.fhir.org");
CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3");
for (PackageInfo pi : matches) {
System.out.println(pi.toString());
@ -47,14 +47,14 @@ public class PackageClientTests {
@Test
public void testVersionsNone() throws IOException {
PackageClient client = new PackageClient("http://packages.fhir.org");
CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org");
List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3X");
Assertions.assertTrue(matches.size() == 0);
}
@Test
public void testExists2() throws IOException {
PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
Assertions.assertTrue(client.exists("hl7.fhir.r4.core", "4.0.1"));
Assertions.assertTrue(!client.exists("hl7.fhir.r4.core", "1.0.2"));
Assertions.assertTrue(!client.exists("hl7.fhir.nothing", "1.0.1"));
@ -62,7 +62,7 @@ public class PackageClientTests {
@Test
public void testSearch2() throws IOException {
PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
List<PackageInfo> matches = client.search("core", null, null, false);
for (PackageInfo pi : matches) {
System.out.println(pi.toString());
@ -72,14 +72,14 @@ public class PackageClientTests {
@Test
public void testSearchNoMatches2() throws IOException {
PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
List<PackageInfo> matches = client.search("corezxxx", null, null, false);
Assertions.assertTrue(matches.size() == 0);
}
@Test
public void testVersions2() throws IOException {
PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3");
for (PackageInfo pi : matches) {
System.out.println(pi.toString());
@ -89,7 +89,7 @@ public class PackageClientTests {
@Test
public void testVersions2A() throws IOException {
PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
List<PackageInfo> matches = client.getVersions("hl7.fhir.us.core");
for (PackageInfo pi : matches) {
System.out.println(pi.toString());
@ -99,7 +99,7 @@ public class PackageClientTests {
@Test
public void testVersionsNone2() throws IOException {
PackageClient client = new PackageClient("http://packages2.fhir.org/packages");
CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages");
List<PackageInfo> matches = client.getVersions("Simplifier.Core.STU3X");
Assertions.assertTrue(matches.size() == 0);
}

6
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java
View File

@ -9,8 +9,8 @@ import org.hl7.fhir.exceptions.FHIRFormatError;
import org.hl7.fhir.utilities.VersionUtilities;
import org.hl7.fhir.utilities.cache.NpmPackage;
import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
import org.hl7.fhir.utilities.cache.PackageClient;
import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo;
import org.hl7.fhir.utilities.cache.CachingPackageClient;
import org.hl7.fhir.utilities.cache.CachingPackageClient.PackageInfo;
import org.hl7.fhir.utilities.cache.ToolsVersion;
public class PackageValidator {
@ -22,7 +22,7 @@ public class PackageValidator {
private void execute() throws IOException {
FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
PackageClient pc = new PackageClient(FilesystemPackageCacheManager.PRIMARY_SERVER);
CachingPackageClient pc = new CachingPackageClient(FilesystemPackageCacheManager.PRIMARY_SERVER);
for (PackageInfo t : pc.search(null, null, null, false)) {
System.out.println("Check Package "+t.getId());
List<PackageInfo> vl = pc.getVersions(t.getId());