Revert "Refactor package cache + more concurrency improvements"
This reverts commit 4c84ea1751
.
This commit is contained in:
parent
4c84ea1751
commit
a1fbd674d6
|
@ -67,6 +67,7 @@ import javax.annotation.Nullable;
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
import org.hl7.fhir.exceptions.FHIRException;
|
import org.hl7.fhir.exceptions.FHIRException;
|
||||||
import org.hl7.fhir.utilities.FileNotifier.FileNotifier2;
|
import org.hl7.fhir.utilities.FileNotifier.FileNotifier2;
|
||||||
|
import org.hl7.fhir.utilities.Utilities.CaseInsensitiveSorter;
|
||||||
import org.hl7.fhir.utilities.filesystem.CSFile;
|
import org.hl7.fhir.utilities.filesystem.CSFile;
|
||||||
import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
|
import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
|
||||||
import org.hl7.fhir.utilities.settings.FhirSettings;
|
import org.hl7.fhir.utilities.settings.FhirSettings;
|
||||||
|
@ -448,28 +449,6 @@ public class Utilities {
|
||||||
return s.toString();
|
return s.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete a directory atomically by first renaming it to a temp directory in
|
|
||||||
* its parent, and then deleting its contents.
|
|
||||||
*
|
|
||||||
* @param path The directory to delete.
|
|
||||||
*/
|
|
||||||
public static void atomicDeleteDirectory(String path) throws IOException {
|
|
||||||
|
|
||||||
File directory = ManagedFileAccess.file(path);
|
|
||||||
|
|
||||||
String tempDirectoryPath = generateUniqueRandomUUIDPath(directory.getParent());
|
|
||||||
File tempDirectory = ManagedFileAccess.file(tempDirectoryPath);
|
|
||||||
if (!directory.renameTo(tempDirectory)) {
|
|
||||||
throw new IOException("Unable to rename directory " + path + " to " + tempDirectory +" for atomic delete");
|
|
||||||
}
|
|
||||||
clearDirectory(tempDirectory.getAbsolutePath());
|
|
||||||
if (!tempDirectory.delete()) {
|
|
||||||
throw new IOException("Unable to delete temp directory " + tempDirectory + " when atomically deleting " + path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public static void clearDirectory(String folder, String... exemptions) throws IOException {
|
public static void clearDirectory(String folder, String... exemptions) throws IOException {
|
||||||
File dir = ManagedFileAccess.file(folder);
|
File dir = ManagedFileAccess.file(folder);
|
||||||
if (dir.exists()) {
|
if (dir.exists()) {
|
||||||
|
@ -491,21 +470,6 @@ public class Utilities {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String generateUniqueRandomUUIDPath(String path) throws IOException {
|
|
||||||
String randomUUIDPath = null;
|
|
||||||
|
|
||||||
while (randomUUIDPath == null) {
|
|
||||||
final String uuid = UUID.randomUUID().toString().toLowerCase();
|
|
||||||
final String pathCandidate = Utilities.path(path, uuid);
|
|
||||||
|
|
||||||
if (!ManagedFileAccess.file(pathCandidate).exists()) {
|
|
||||||
randomUUIDPath = pathCandidate;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return randomUUIDPath;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static File createDirectory(String path) throws IOException {
|
public static File createDirectory(String path) throws IOException {
|
||||||
ManagedFileAccess.csfile(path).mkdirs();
|
ManagedFileAccess.csfile(path).mkdirs();
|
||||||
return ManagedFileAccess.file(path);
|
return ManagedFileAccess.file(path);
|
||||||
|
|
|
@ -99,14 +99,9 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
private final File cacheFolder;
|
private final File cacheFolder;
|
||||||
|
|
||||||
private final List<NpmPackage> temporaryPackages = new ArrayList<>();
|
private final List<NpmPackage> temporaryPackages = new ArrayList<>();
|
||||||
private long ciQueryTimeStamp = 0;
|
private boolean buildLoaded = false;
|
||||||
private final long ciQueryInterval;
|
private final Map<String, String> ciList = new HashMap<>();
|
||||||
private static final long DEFAULT_CI_QUERY_INTERVAL = 1000 * 60 * 60 * 24;
|
private JsonArray buildInfo;
|
||||||
|
|
||||||
/** key = packageId
|
|
||||||
* value = url of built package on https://build.fhir.org/ig/ */
|
|
||||||
private final Map<String, String> ciPackageList = new HashMap<>();
|
|
||||||
private JsonArray ciBuildInfo;
|
|
||||||
private boolean suppressErrors;
|
private boolean suppressErrors;
|
||||||
|
|
||||||
@Setter
|
@Setter
|
||||||
|
@ -122,10 +117,6 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
@Getter
|
@Getter
|
||||||
private final List<PackageServer> packageServers;
|
private final List<PackageServer> packageServers;
|
||||||
|
|
||||||
@With
|
|
||||||
@Getter
|
|
||||||
private final long ciQueryInterval;
|
|
||||||
|
|
||||||
@With
|
@With
|
||||||
@Getter
|
@Getter
|
||||||
private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters;
|
private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters;
|
||||||
|
@ -134,13 +125,11 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
this.cacheFolder = getUserCacheFolder();
|
this.cacheFolder = getUserCacheFolder();
|
||||||
this.packageServers = getPackageServersFromFHIRSettings();
|
this.packageServers = getPackageServersFromFHIRSettings();
|
||||||
this.lockParameters = null;
|
this.lockParameters = null;
|
||||||
this.ciQueryInterval = FilesystemPackageCacheManager.DEFAULT_CI_QUERY_INTERVAL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private Builder(File cacheFolder, List<PackageServer> packageServers, long ciQueryInterval, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) {
|
private Builder(File cacheFolder, List<PackageServer> packageServers, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) {
|
||||||
this.cacheFolder = cacheFolder;
|
this.cacheFolder = cacheFolder;
|
||||||
this.packageServers = packageServers;
|
this.packageServers = packageServers;
|
||||||
this.ciQueryInterval = ciQueryInterval;
|
|
||||||
this.lockParameters = lockParameters;
|
this.lockParameters = lockParameters;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -174,7 +163,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
if (!cacheFolder.exists()) {
|
if (!cacheFolder.exists()) {
|
||||||
throw new FHIRException("The folder '" + cacheFolder + "' could not be found");
|
throw new FHIRException("The folder '" + cacheFolder + "' could not be found");
|
||||||
}
|
}
|
||||||
return new Builder(cacheFolder, this.packageServers, this.ciQueryInterval, this.lockParameters);
|
return new Builder(cacheFolder, this.packageServers, this.lockParameters);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Builder withSystemCacheFolder() throws IOException {
|
public Builder withSystemCacheFolder() throws IOException {
|
||||||
|
@ -184,11 +173,11 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
} else {
|
} else {
|
||||||
systemCacheFolder = ManagedFileAccess.file(Utilities.path("/var", "lib", ".fhir", "packages"));
|
systemCacheFolder = ManagedFileAccess.file(Utilities.path("/var", "lib", ".fhir", "packages"));
|
||||||
}
|
}
|
||||||
return new Builder(systemCacheFolder, this.packageServers, this.ciQueryInterval, this.lockParameters);
|
return new Builder(systemCacheFolder, this.packageServers, this.lockParameters);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Builder withTestingCacheFolder() throws IOException {
|
public Builder withTestingCacheFolder() throws IOException {
|
||||||
return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.ciQueryInterval, this.lockParameters);
|
return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.lockParameters);
|
||||||
}
|
}
|
||||||
|
|
||||||
public FilesystemPackageCacheManager build() throws IOException {
|
public FilesystemPackageCacheManager build() throws IOException {
|
||||||
|
@ -202,14 +191,13 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return new FilesystemPackageCacheManager(cacheFolder, packageServers, ciQueryInterval, locks, lockParameters);
|
return new FilesystemPackageCacheManager(cacheFolder, packageServers, locks, lockParameters);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List<PackageServer> packageServers, long ciQueryInterval,@Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException {
|
private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List<PackageServer> packageServers, @Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException {
|
||||||
super(packageServers);
|
super(packageServers);
|
||||||
this.cacheFolder = cacheFolder;
|
this.cacheFolder = cacheFolder;
|
||||||
this.ciQueryInterval = ciQueryInterval;
|
|
||||||
this.locks = locks;
|
this.locks = locks;
|
||||||
this.lockParameters = lockParameters;
|
this.lockParameters = lockParameters;
|
||||||
prepareCacheFolder();
|
prepareCacheFolder();
|
||||||
|
@ -292,6 +280,27 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void initPackageServers() {
|
||||||
|
myPackageServers.addAll(getConfiguredServers());
|
||||||
|
if (!isIgnoreDefaultPackageServers()) {
|
||||||
|
myPackageServers.addAll(getDefaultServers());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected boolean isIgnoreDefaultPackageServers() {
|
||||||
|
return FhirSettings.isIgnoreDefaultPackageServers();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Nonnull
|
||||||
|
protected List<PackageServer> getDefaultServers() {
|
||||||
|
return PackageServer.defaultServers();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected List<PackageServer> getConfiguredServers() {
|
||||||
|
return PackageServer.getConfiguredServers();
|
||||||
|
}
|
||||||
|
|
||||||
public String getFolder() {
|
public String getFolder() {
|
||||||
return cacheFolder.getAbsolutePath();
|
return cacheFolder.getAbsolutePath();
|
||||||
}
|
}
|
||||||
|
@ -303,7 +312,16 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
private void clearCache() throws IOException {
|
private void clearCache() throws IOException {
|
||||||
for (File f : Objects.requireNonNull(cacheFolder.listFiles())) {
|
for (File f : Objects.requireNonNull(cacheFolder.listFiles())) {
|
||||||
if (f.isDirectory()) {
|
if (f.isDirectory()) {
|
||||||
Utilities.atomicDeleteDirectory(f.getAbsolutePath());
|
Utilities.clearDirectory(f.getAbsolutePath());
|
||||||
|
try {
|
||||||
|
FileUtils.deleteDirectory(f);
|
||||||
|
} catch (Exception e1) {
|
||||||
|
try {
|
||||||
|
FileUtils.deleteDirectory(f);
|
||||||
|
} catch (Exception e2) {
|
||||||
|
// just give up
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
} else if (!f.getName().equals("packages.ini")) {
|
} else if (!f.getName().equals("packages.ini")) {
|
||||||
FileUtils.forceDelete(f);
|
FileUtils.forceDelete(f);
|
||||||
|
@ -433,8 +451,10 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
String f = Utilities.path(cacheFolder, id + "#" + version);
|
String f = Utilities.path(cacheFolder, id + "#" + version);
|
||||||
File ff = ManagedFileAccess.file(f);
|
File ff = ManagedFileAccess.file(f);
|
||||||
if (ff.exists()) {
|
if (ff.exists()) {
|
||||||
Utilities.atomicDeleteDirectory(f);
|
Utilities.clearDirectory(f);
|
||||||
|
ff.delete();
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}, lockParameters);
|
}, lockParameters);
|
||||||
}
|
}
|
||||||
|
@ -541,42 +561,25 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
public NpmPackage addPackageToCache(final String id, final String version, final InputStream packageTgzInputStream, final String sourceDesc) throws IOException {
|
public NpmPackage addPackageToCache(final String id, final String version, final InputStream packageTgzInputStream, final String sourceDesc) throws IOException {
|
||||||
checkValidVersionString(version, id);
|
checkValidVersionString(version, id);
|
||||||
return locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> {
|
return locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> {
|
||||||
|
String uuid = UUID.randomUUID().toString().toLowerCase();
|
||||||
|
String tempDir = Utilities.path(cacheFolder, uuid);
|
||||||
|
|
||||||
String tempDir = Utilities.generateUniqueRandomUUIDPath(cacheFolder.getAbsolutePath());
|
NpmPackage npm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory);
|
||||||
|
|
||||||
NpmPackage extractedNpm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory);
|
|
||||||
|
|
||||||
log("");
|
log("");
|
||||||
log("Installing " + id + "#" + version);
|
log("Installing " + id + "#" + version);
|
||||||
|
|
||||||
if ((extractedNpm.name() != null && id != null && !id.equalsIgnoreCase(extractedNpm.name()))) {
|
if ((npm.name() != null && id != null && !id.equalsIgnoreCase(npm.name()))) {
|
||||||
if (!suppressErrors && (!id.equals("hl7.fhir.r5.core") && !id.equals("hl7.fhir.us.immds"))) {// temporary work around
|
if (!suppressErrors && (!id.equals("hl7.fhir.r5.core") && !id.equals("hl7.fhir.us.immds"))) {// temporary work around
|
||||||
throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + extractedNpm.name());
|
throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
NpmPackage npmPackage = null;
|
NpmPackage npmPackage = null;
|
||||||
String packageRoot = Utilities.path(cacheFolder, id + "#" + version);
|
String packageRoot = Utilities.path(cacheFolder, id + "#" + version);
|
||||||
try {
|
try {
|
||||||
if (!id.equals(extractedNpm.getNpm().asString("name")) || !version.equals(extractedNpm.getNpm().asString("version"))) {
|
// ok, now we have a lock on it... check if something created it while we were waiting
|
||||||
if (!id.equals(extractedNpm.getNpm().asString("name"))) {
|
|
||||||
extractedNpm.getNpm().add("original-name", extractedNpm.getNpm().asString("name"));
|
|
||||||
extractedNpm.getNpm().remove("name");
|
|
||||||
extractedNpm.getNpm().add("name", id);
|
|
||||||
}
|
|
||||||
if (!version.equals(extractedNpm.getNpm().asString("version"))) {
|
|
||||||
extractedNpm.getNpm().add("original-version", extractedNpm.getNpm().asString("version"));
|
|
||||||
extractedNpm.getNpm().remove("version");
|
|
||||||
extractedNpm.getNpm().add("version", version);
|
|
||||||
}
|
|
||||||
TextFile.stringToFile(JsonParser.compose(extractedNpm.getNpm(), true), Utilities.path(tempDir, "package", "package.json"));
|
|
||||||
}
|
|
||||||
|
|
||||||
final NpmPackage tempPackage = loadPackageInfo(tempDir);
|
|
||||||
if (tempPackage != null && !tempPackage.isIndexed()) {
|
|
||||||
tempPackage.checkIndexed(packageRoot);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ManagedFileAccess.file(packageRoot).exists() || Utilities.existsInList(version, "current", "dev")) {
|
if (!ManagedFileAccess.file(packageRoot).exists() || Utilities.existsInList(version, "current", "dev")) {
|
||||||
Utilities.createDirectory(packageRoot);
|
Utilities.createDirectory(packageRoot);
|
||||||
try {
|
try {
|
||||||
|
@ -584,18 +587,30 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
} catch (Throwable t) {
|
} catch (Throwable t) {
|
||||||
log("Unable to clear directory: " + packageRoot + ": " + t.getMessage() + " - this may cause problems later");
|
log("Unable to clear directory: " + packageRoot + ": " + t.getMessage() + " - this may cause problems later");
|
||||||
}
|
}
|
||||||
|
|
||||||
Utilities.renameDirectory(tempDir, packageRoot);
|
Utilities.renameDirectory(tempDir, packageRoot);
|
||||||
|
|
||||||
npmPackage = loadPackageInfo(packageRoot);
|
|
||||||
|
|
||||||
log(" done.");
|
log(" done.");
|
||||||
} else {
|
} else {
|
||||||
Utilities.clearDirectory(tempDir);
|
Utilities.clearDirectory(tempDir);
|
||||||
ManagedFileAccess.file(tempDir).delete();
|
ManagedFileAccess.file(tempDir).delete();
|
||||||
}
|
}
|
||||||
|
if (!id.equals(npm.getNpm().asString("name")) || !version.equals(npm.getNpm().asString("version"))) {
|
||||||
|
if (!id.equals(npm.getNpm().asString("name"))) {
|
||||||
|
npm.getNpm().add("original-name", npm.getNpm().asString("name"));
|
||||||
|
npm.getNpm().remove("name");
|
||||||
|
npm.getNpm().add("name", id);
|
||||||
|
}
|
||||||
|
if (!version.equals(npm.getNpm().asString("version"))) {
|
||||||
|
npm.getNpm().add("original-version", npm.getNpm().asString("version"));
|
||||||
|
npm.getNpm().remove("version");
|
||||||
|
npm.getNpm().add("version", version);
|
||||||
|
}
|
||||||
|
TextFile.stringToFile(JsonParser.compose(npm.getNpm(), true), Utilities.path(cacheFolder, id + "#" + version, "package", "package.json"));
|
||||||
|
}
|
||||||
npmPackage = loadPackageInfo(packageRoot);
|
npmPackage = loadPackageInfo(packageRoot);
|
||||||
|
if (npmPackage != null && !npmPackage.isIndexed()) {
|
||||||
|
npmPackage.checkIndexed(packageRoot);
|
||||||
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
try {
|
try {
|
||||||
// don't leave a half extracted package behind
|
// don't leave a half extracted package behind
|
||||||
|
@ -726,31 +741,31 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
}
|
}
|
||||||
|
|
||||||
private InputStreamWithSrc loadFromCIBuild(String id, String branch) throws IOException {
|
private InputStreamWithSrc loadFromCIBuild(String id, String branch) throws IOException {
|
||||||
checkCIServerQueried();
|
checkBuildLoaded();
|
||||||
if (ciPackageList.containsKey(id)) {
|
if (ciList.containsKey(id)) {
|
||||||
if (branch == null) {
|
if (branch == null) {
|
||||||
InputStream stream;
|
InputStream stream;
|
||||||
try {
|
try {
|
||||||
stream = fetchFromUrlSpecific(Utilities.pathURL(ciPackageList.get(id), "package.tgz"), false);
|
stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
stream = fetchFromUrlSpecific(Utilities.pathURL(ciPackageList.get(id), "branches", "main", "package.tgz"), false);
|
stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", "main", "package.tgz"), false);
|
||||||
}
|
}
|
||||||
return new InputStreamWithSrc(stream, Utilities.pathURL(ciPackageList.get(id), "package.tgz"), "current");
|
return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current");
|
||||||
} else {
|
} else {
|
||||||
InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciPackageList.get(id), "branches", branch, "package.tgz"), false);
|
InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), false);
|
||||||
return new InputStreamWithSrc(stream, Utilities.pathURL(ciPackageList.get(id), "branches", branch, "package.tgz"), "current$" + branch);
|
return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), "current$" + branch);
|
||||||
}
|
}
|
||||||
} else if (id.startsWith("hl7.fhir.r6")) {
|
} else if (id.startsWith("hl7.fhir.r6")) {
|
||||||
InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("https://build.fhir.org", id + ".tgz"), false);
|
InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("https://build.fhir.org", id + ".tgz"), false);
|
||||||
return new InputStreamWithSrc(stream, Utilities.pathURL("https://build.fhir.org", id + ".tgz"), "current");
|
return new InputStreamWithSrc(stream, Utilities.pathURL("https://build.fhir.org", id + ".tgz"), "current");
|
||||||
} else {
|
} else {
|
||||||
throw new FHIRException("The package '" + id + "' has no entry on the current build server (" + ciPackageList + ")");
|
throw new FHIRException("The package '" + id + "' has no entry on the current build server (" + ciList + ")");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getPackageUrlFromBuildList(String packageId) throws IOException {
|
private String getPackageUrlFromBuildList(String packageId) throws IOException {
|
||||||
checkCIServerQueried();
|
checkBuildLoaded();
|
||||||
for (JsonObject o : ciBuildInfo.asJsonObjects()) {
|
for (JsonObject o : buildInfo.asJsonObjects()) {
|
||||||
if (packageId.equals(o.asString("package-id"))) {
|
if (packageId.equals(o.asString("package-id"))) {
|
||||||
return o.asString("url");
|
return o.asString("url");
|
||||||
}
|
}
|
||||||
|
@ -795,15 +810,15 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
if (canonical == null) {
|
if (canonical == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
checkCIServerQueried();
|
checkBuildLoaded();
|
||||||
if (ciBuildInfo != null) {
|
if (buildInfo != null) {
|
||||||
for (JsonElement n : ciBuildInfo) {
|
for (JsonElement n : buildInfo) {
|
||||||
JsonObject o = (JsonObject) n;
|
JsonObject o = (JsonObject) n;
|
||||||
if (canonical.equals(o.asString("url"))) {
|
if (canonical.equals(o.asString("url"))) {
|
||||||
return o.asString("package-id");
|
return o.asString("package-id");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (JsonElement n : ciBuildInfo) {
|
for (JsonElement n : buildInfo) {
|
||||||
JsonObject o = (JsonObject) n;
|
JsonObject o = (JsonObject) n;
|
||||||
if (o.asString("url").startsWith(canonical + "/ImplementationGuide/")) {
|
if (o.asString("url").startsWith(canonical + "/ImplementationGuide/")) {
|
||||||
return o.asString("package-id");
|
return o.asString("package-id");
|
||||||
|
@ -813,39 +828,32 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private NpmPackage checkCurrency(String id, NpmPackage p) {
|
||||||
* Checks https://
|
checkBuildLoaded();
|
||||||
*
|
|
||||||
* @param id
|
|
||||||
* @param npmPackage
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
private NpmPackage checkCurrency(String id, NpmPackage npmPackage) {
|
|
||||||
checkCIServerQueried();
|
|
||||||
// special case: current versions roll over, and we have to check their currency
|
// special case: current versions roll over, and we have to check their currency
|
||||||
try {
|
try {
|
||||||
String packageManifestUrl = ciPackageList.get(id);
|
String url = ciList.get(id);
|
||||||
JsonObject packageManifestJson = JsonParser.parseObjectFromUrl(Utilities.pathURL(packageManifestUrl, "package.manifest.json"));
|
JsonObject json = JsonParser.parseObjectFromUrl(Utilities.pathURL(url, "package.manifest.json"));
|
||||||
String currentDate = packageManifestJson.asString("date");
|
String currDate = json.asString("date");
|
||||||
String packageDate = npmPackage.date();
|
String packDate = p.date();
|
||||||
if (!currentDate.equals(packageDate)) {
|
if (!currDate.equals(packDate)) {
|
||||||
return null; // nup, we need a new copy
|
return null; // nup, we need a new copy
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log("Unable to check package currency: " + id + ": " + id);
|
log("Unable to check package currency: " + id + ": " + id);
|
||||||
}
|
}
|
||||||
return npmPackage;
|
return p;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void checkCIServerQueried() {
|
private void checkBuildLoaded() {
|
||||||
if (System.currentTimeMillis() - ciQueryTimeStamp > ciQueryInterval) {
|
if (!buildLoaded) {
|
||||||
try {
|
try {
|
||||||
queryCIServer();
|
loadFromBuildServer();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
try {
|
try {
|
||||||
// we always pause a second and try again - the most common reason to be here is that the file was being changed on the server
|
// we always pause a second and try again - the most common reason to be here is that the file was being changed on the server
|
||||||
Thread.sleep(1000);
|
Thread.sleep(1000);
|
||||||
queryCIServer();
|
loadFromBuildServer();
|
||||||
} catch (Exception e2) {
|
} catch (Exception e2) {
|
||||||
log("Error connecting to build server - running without build (" + e2.getMessage() + ")");
|
log("Error connecting to build server - running without build (" + e2.getMessage() + ")");
|
||||||
}
|
}
|
||||||
|
@ -853,31 +861,31 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void queryCIServer() throws IOException {
|
private void loadFromBuildServer() throws IOException {
|
||||||
|
|
||||||
HTTPResult res = ManagedWebAccess.get("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
|
HTTPResult res = ManagedWebAccess.get("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
|
||||||
res.checkThrowException();
|
res.checkThrowException();
|
||||||
|
|
||||||
ciBuildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent()));
|
buildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent()));
|
||||||
|
|
||||||
List<BuildRecord> builds = new ArrayList<>();
|
List<BuildRecord> builds = new ArrayList<>();
|
||||||
|
|
||||||
for (JsonElement n : ciBuildInfo) {
|
for (JsonElement n : buildInfo) {
|
||||||
JsonObject j = (JsonObject) n;
|
JsonObject o = (JsonObject) n;
|
||||||
if (j.has("url") && j.has("package-id") && j.asString("package-id").contains(".")) {
|
if (o.has("url") && o.has("package-id") && o.asString("package-id").contains(".")) {
|
||||||
String packageUrl = j.asString("url");
|
String u = o.asString("url");
|
||||||
if (packageUrl.contains("/ImplementationGuide/"))
|
if (u.contains("/ImplementationGuide/"))
|
||||||
packageUrl = packageUrl.substring(0, packageUrl.indexOf("/ImplementationGuide/"));
|
u = u.substring(0, u.indexOf("/ImplementationGuide/"));
|
||||||
builds.add(new BuildRecord(packageUrl, j.asString("package-id"), getRepo(j.asString("repo")), readDate(j.asString("date"))));
|
builds.add(new BuildRecord(u, o.asString("package-id"), getRepo(o.asString("repo")), readDate(o.asString("date"))));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Collections.sort(builds, new BuildRecordSorter());
|
Collections.sort(builds, new BuildRecordSorter());
|
||||||
for (BuildRecord build : builds) {
|
for (BuildRecord bld : builds) {
|
||||||
if (!ciPackageList.containsKey(build.getPackageId())) {
|
if (!ciList.containsKey(bld.getPackageId())) {
|
||||||
ciPackageList.put(build.getPackageId(), "https://build.fhir.org/ig/" + build.getRepo());
|
ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ciQueryTimeStamp = System.currentTimeMillis();
|
buildLoaded = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getRepo(String path) {
|
private String getRepo(String path) {
|
||||||
|
|
|
@ -640,11 +640,11 @@ public class NpmPackage {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void checkIndexed(String path) throws IOException {
|
public void checkIndexed(String desc) throws IOException {
|
||||||
for (NpmPackageFolder folder : folders.values()) {
|
for (NpmPackageFolder folder : folders.values()) {
|
||||||
JsonObject index = folder.index();
|
JsonObject index = folder.index();
|
||||||
if (index == null || index.forceArray("files").size() == 0) {
|
if (index == null || index.forceArray("files").size() == 0) {
|
||||||
indexFolder(path, folder);
|
indexFolder(desc, folder);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -656,18 +656,18 @@ public class NpmPackage {
|
||||||
* See <a href="https://hl7.org/fhir/packages.html#2.1.10.4">the FHIR specification</a> for details on .index.json
|
* See <a href="https://hl7.org/fhir/packages.html#2.1.10.4">the FHIR specification</a> for details on .index.json
|
||||||
* format and usage.
|
* format and usage.
|
||||||
*
|
*
|
||||||
* @param path
|
* @param desc
|
||||||
* @param folder
|
* @param folder
|
||||||
* @throws FileNotFoundException
|
* @throws FileNotFoundException
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public void indexFolder(String path, NpmPackageFolder folder) throws FileNotFoundException, IOException {
|
public void indexFolder(String desc, NpmPackageFolder folder) throws FileNotFoundException, IOException {
|
||||||
List<String> remove = new ArrayList<>();
|
List<String> remove = new ArrayList<>();
|
||||||
NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
|
NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
|
||||||
indexer.start(folder.folder != null ? Utilities.path(folder.folder.getAbsolutePath(), ".index.db") : null);
|
indexer.start(folder.folder != null ? Utilities.path(folder.folder.getAbsolutePath(), ".index.db") : null);
|
||||||
for (String file : folder.listFiles()) {
|
for (String n : folder.listFiles()) {
|
||||||
if (!indexer.seeFile(file, folder.fetchFile(file))) {
|
if (!indexer.seeFile(n, folder.fetchFile(n))) {
|
||||||
remove.add(file);
|
remove.add(n);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (String n : remove) {
|
for (String n : remove) {
|
||||||
|
@ -684,7 +684,7 @@ public class NpmPackage {
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json"));
|
TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json"));
|
||||||
throw new IOException("Error parsing "+(path == null ? "" : path+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e);
|
throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -50,10 +50,7 @@ public class FilesystemPackageManagerTests {
|
||||||
new PackageServer(DUMMY_URL_4)
|
new PackageServer(DUMMY_URL_4)
|
||||||
);
|
);
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testCheckCurrentPackage() {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testDefaultServers() throws IOException {
|
public void testDefaultServers() throws IOException {
|
||||||
|
|
|
@ -105,7 +105,7 @@ public class LockfileTestProcessUtility {
|
||||||
System.out.println("File "+lockFileName+" is locked. Waiting for " + seconds + " seconds to release. ");
|
System.out.println("File "+lockFileName+" is locked. Waiting for " + seconds + " seconds to release. ");
|
||||||
Thread.sleep(seconds * 1000L);
|
Thread.sleep(seconds * 1000L);
|
||||||
|
|
||||||
lockFile.renameTo(File.createTempFile(lockFile.getName(), ".lock-renamed"));
|
lockFile.renameTo(ManagedFileAccess.file(File.createTempFile(lockFile.getName(), ".lock-renamed").getAbsolutePath()));
|
||||||
|
|
||||||
fileLock.release();
|
fileLock.release();
|
||||||
channel.close();
|
channel.close();
|
||||||
|
|
2
pom.xml
2
pom.xml
|
@ -21,7 +21,7 @@
|
||||||
<commons_compress_version>1.26.0</commons_compress_version>
|
<commons_compress_version>1.26.0</commons_compress_version>
|
||||||
<guava_version>32.0.1-jre</guava_version>
|
<guava_version>32.0.1-jre</guava_version>
|
||||||
<hapi_fhir_version>6.4.1</hapi_fhir_version>
|
<hapi_fhir_version>6.4.1</hapi_fhir_version>
|
||||||
<validator_test_case_version>1.5.25-SNAPSHOT</validator_test_case_version>
|
<validator_test_case_version>1.5.24</validator_test_case_version>
|
||||||
<jackson_version>2.17.0</jackson_version>
|
<jackson_version>2.17.0</jackson_version>
|
||||||
<junit_jupiter_version>5.9.2</junit_jupiter_version>
|
<junit_jupiter_version>5.9.2</junit_jupiter_version>
|
||||||
<junit_platform_launcher_version>1.8.2</junit_platform_launcher_version>
|
<junit_platform_launcher_version>1.8.2</junit_platform_launcher_version>
|
||||||
|
|
Loading…
Reference in New Issue