diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java index 4042ee2f1..3bb9031e8 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java @@ -67,7 +67,6 @@ import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.utilities.FileNotifier.FileNotifier2; -import org.hl7.fhir.utilities.Utilities.CaseInsensitiveSorter; import org.hl7.fhir.utilities.filesystem.CSFile; import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; import org.hl7.fhir.utilities.settings.FhirSettings; @@ -449,6 +448,28 @@ public class Utilities { return s.toString(); } + /** + * Delete a directory atomically by first renaming it to a temp directory in + * its parent, and then deleting its contents. + * + * @param path The directory to delete. + */ + public static void atomicDeleteDirectory(String path) throws IOException { + + File directory = ManagedFileAccess.file(path); + + String tempDirectoryPath = generateUniqueRandomUUIDPath(directory.getParent()); + File tempDirectory = ManagedFileAccess.file(tempDirectoryPath); + if (!directory.renameTo(tempDirectory)) { + throw new IOException("Unable to rename directory " + path + " to " + tempDirectory +" for atomic delete"); + } + clearDirectory(tempDirectory.getAbsolutePath()); + if (!tempDirectory.delete()) { + throw new IOException("Unable to delete temp directory " + tempDirectory + " when atomically deleting " + path); + } + } + + public static void clearDirectory(String folder, String... exemptions) throws IOException { File dir = ManagedFileAccess.file(folder); if (dir.exists()) { @@ -470,6 +491,21 @@ public class Utilities { } } + public static String generateUniqueRandomUUIDPath(String path) throws IOException { + String randomUUIDPath = null; + + while (randomUUIDPath == null) { + final String uuid = UUID.randomUUID().toString().toLowerCase(); + final String pathCandidate = Utilities.path(path, uuid); + + if (!ManagedFileAccess.file(pathCandidate).exists()) { + randomUUIDPath = pathCandidate; + } + } + + return randomUUIDPath; + } + public static File createDirectory(String path) throws IOException { ManagedFileAccess.csfile(path).mkdirs(); return ManagedFileAccess.file(path); diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java index 206d90782..046cd80d0 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java @@ -99,9 +99,14 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple private final File cacheFolder; private final List temporaryPackages = new ArrayList<>(); - private boolean buildLoaded = false; - private final Map ciList = new HashMap<>(); - private JsonArray buildInfo; + private long ciQueryTimeStamp = 0; + private final long ciQueryInterval; + private static final long DEFAULT_CI_QUERY_INTERVAL = 1000 * 60 * 60 * 24; + + /** key = packageId + * value = url of built package on https://build.fhir.org/ig/ */ + private final Map ciPackageList = new HashMap<>(); + private JsonArray ciBuildInfo; private boolean suppressErrors; @Setter @@ -117,6 +122,10 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple @Getter private final List packageServers; + @With + @Getter + private final long ciQueryInterval; + @With @Getter private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters; @@ -125,11 +134,13 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple this.cacheFolder = getUserCacheFolder(); this.packageServers = getPackageServersFromFHIRSettings(); this.lockParameters = null; + this.ciQueryInterval = FilesystemPackageCacheManager.DEFAULT_CI_QUERY_INTERVAL; } - private Builder(File cacheFolder, List packageServers, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) { + private Builder(File cacheFolder, List packageServers, long ciQueryInterval, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) { this.cacheFolder = cacheFolder; this.packageServers = packageServers; + this.ciQueryInterval = ciQueryInterval; this.lockParameters = lockParameters; } @@ -163,7 +174,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple if (!cacheFolder.exists()) { throw new FHIRException("The folder '" + cacheFolder + "' could not be found"); } - return new Builder(cacheFolder, this.packageServers, this.lockParameters); + return new Builder(cacheFolder, this.packageServers, this.ciQueryInterval, this.lockParameters); } public Builder withSystemCacheFolder() throws IOException { @@ -173,11 +184,11 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } else { systemCacheFolder = ManagedFileAccess.file(Utilities.path("/var", "lib", ".fhir", "packages")); } - return new Builder(systemCacheFolder, this.packageServers, this.lockParameters); + return new Builder(systemCacheFolder, this.packageServers, this.ciQueryInterval, this.lockParameters); } public Builder withTestingCacheFolder() throws IOException { - return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.lockParameters); + return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.ciQueryInterval, this.lockParameters); } public FilesystemPackageCacheManager build() throws IOException { @@ -191,13 +202,14 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple throw e; } } - return new FilesystemPackageCacheManager(cacheFolder, packageServers, locks, lockParameters); + return new FilesystemPackageCacheManager(cacheFolder, packageServers, ciQueryInterval, locks, lockParameters); } } - private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List packageServers, @Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException { + private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List packageServers, long ciQueryInterval,@Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException { super(packageServers); this.cacheFolder = cacheFolder; + this.ciQueryInterval = ciQueryInterval; this.locks = locks; this.lockParameters = lockParameters; prepareCacheFolder(); @@ -280,27 +292,6 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } } - - private void initPackageServers() { - myPackageServers.addAll(getConfiguredServers()); - if (!isIgnoreDefaultPackageServers()) { - myPackageServers.addAll(getDefaultServers()); - } - } - - protected boolean isIgnoreDefaultPackageServers() { - return FhirSettings.isIgnoreDefaultPackageServers(); - } - - @Nonnull - protected List getDefaultServers() { - return PackageServer.defaultServers(); - } - - protected List getConfiguredServers() { - return PackageServer.getConfiguredServers(); - } - public String getFolder() { return cacheFolder.getAbsolutePath(); } @@ -312,16 +303,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple private void clearCache() throws IOException { for (File f : Objects.requireNonNull(cacheFolder.listFiles())) { if (f.isDirectory()) { - Utilities.clearDirectory(f.getAbsolutePath()); - try { - FileUtils.deleteDirectory(f); - } catch (Exception e1) { - try { - FileUtils.deleteDirectory(f); - } catch (Exception e2) { - // just give up - } - } + Utilities.atomicDeleteDirectory(f.getAbsolutePath()); } else if (!f.getName().equals("packages.ini")) { FileUtils.forceDelete(f); @@ -449,12 +431,10 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> { String f = Utilities.path(cacheFolder, id + "#" + version); - File ff = ManagedFileAccess.file(f); - if (ff.exists()) { - Utilities.clearDirectory(f); - ff.delete(); - } - + File ff = ManagedFileAccess.file(f); + if (ff.exists()) { + Utilities.atomicDeleteDirectory(f); + } return null; }, lockParameters); } @@ -561,25 +541,42 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple public NpmPackage addPackageToCache(final String id, final String version, final InputStream packageTgzInputStream, final String sourceDesc) throws IOException { checkValidVersionString(version, id); return locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> { - String uuid = UUID.randomUUID().toString().toLowerCase(); - String tempDir = Utilities.path(cacheFolder, uuid); - NpmPackage npm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory); + String tempDir = Utilities.generateUniqueRandomUUIDPath(cacheFolder.getAbsolutePath()); + + NpmPackage extractedNpm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory); log(""); log("Installing " + id + "#" + version); - if ((npm.name() != null && id != null && !id.equalsIgnoreCase(npm.name()))) { + if ((extractedNpm.name() != null && id != null && !id.equalsIgnoreCase(extractedNpm.name()))) { if (!suppressErrors && (!id.equals("hl7.fhir.r5.core") && !id.equals("hl7.fhir.us.immds"))) {// temporary work around - throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name()); + throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + extractedNpm.name()); } } - NpmPackage npmPackage = null; String packageRoot = Utilities.path(cacheFolder, id + "#" + version); try { - // ok, now we have a lock on it... check if something created it while we were waiting + if (!id.equals(extractedNpm.getNpm().asString("name")) || !version.equals(extractedNpm.getNpm().asString("version"))) { + if (!id.equals(extractedNpm.getNpm().asString("name"))) { + extractedNpm.getNpm().add("original-name", extractedNpm.getNpm().asString("name")); + extractedNpm.getNpm().remove("name"); + extractedNpm.getNpm().add("name", id); + } + if (!version.equals(extractedNpm.getNpm().asString("version"))) { + extractedNpm.getNpm().add("original-version", extractedNpm.getNpm().asString("version")); + extractedNpm.getNpm().remove("version"); + extractedNpm.getNpm().add("version", version); + } + TextFile.stringToFile(JsonParser.compose(extractedNpm.getNpm(), true), Utilities.path(tempDir, "package", "package.json")); + } + + final NpmPackage tempPackage = loadPackageInfo(tempDir); + if (tempPackage != null && !tempPackage.isIndexed()) { + tempPackage.checkIndexed(packageRoot); + } + if (!ManagedFileAccess.file(packageRoot).exists() || Utilities.existsInList(version, "current", "dev")) { Utilities.createDirectory(packageRoot); try { @@ -587,30 +584,18 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } catch (Throwable t) { log("Unable to clear directory: " + packageRoot + ": " + t.getMessage() + " - this may cause problems later"); } + Utilities.renameDirectory(tempDir, packageRoot); + npmPackage = loadPackageInfo(packageRoot); + log(" done."); } else { Utilities.clearDirectory(tempDir); ManagedFileAccess.file(tempDir).delete(); } - if (!id.equals(npm.getNpm().asString("name")) || !version.equals(npm.getNpm().asString("version"))) { - if (!id.equals(npm.getNpm().asString("name"))) { - npm.getNpm().add("original-name", npm.getNpm().asString("name")); - npm.getNpm().remove("name"); - npm.getNpm().add("name", id); - } - if (!version.equals(npm.getNpm().asString("version"))) { - npm.getNpm().add("original-version", npm.getNpm().asString("version")); - npm.getNpm().remove("version"); - npm.getNpm().add("version", version); - } - TextFile.stringToFile(JsonParser.compose(npm.getNpm(), true), Utilities.path(cacheFolder, id + "#" + version, "package", "package.json")); - } npmPackage = loadPackageInfo(packageRoot); - if (npmPackage != null && !npmPackage.isIndexed()) { - npmPackage.checkIndexed(packageRoot); - } + } catch (Exception e) { try { // don't leave a half extracted package behind @@ -741,31 +726,31 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } private InputStreamWithSrc loadFromCIBuild(String id, String branch) throws IOException { - checkBuildLoaded(); - if (ciList.containsKey(id)) { + checkCIServerQueried(); + if (ciPackageList.containsKey(id)) { if (branch == null) { InputStream stream; try { - stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false); + stream = fetchFromUrlSpecific(Utilities.pathURL(ciPackageList.get(id), "package.tgz"), false); } catch (Exception e) { - stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", "main", "package.tgz"), false); + stream = fetchFromUrlSpecific(Utilities.pathURL(ciPackageList.get(id), "branches", "main", "package.tgz"), false); } - return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current"); + return new InputStreamWithSrc(stream, Utilities.pathURL(ciPackageList.get(id), "package.tgz"), "current"); } else { - InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), false); - return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), "current$" + branch); + InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciPackageList.get(id), "branches", branch, "package.tgz"), false); + return new InputStreamWithSrc(stream, Utilities.pathURL(ciPackageList.get(id), "branches", branch, "package.tgz"), "current$" + branch); } } else if (id.startsWith("hl7.fhir.r6")) { InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("https://build.fhir.org", id + ".tgz"), false); return new InputStreamWithSrc(stream, Utilities.pathURL("https://build.fhir.org", id + ".tgz"), "current"); } else { - throw new FHIRException("The package '" + id + "' has no entry on the current build server (" + ciList + ")"); + throw new FHIRException("The package '" + id + "' has no entry on the current build server (" + ciPackageList + ")"); } } private String getPackageUrlFromBuildList(String packageId) throws IOException { - checkBuildLoaded(); - for (JsonObject o : buildInfo.asJsonObjects()) { + checkCIServerQueried(); + for (JsonObject o : ciBuildInfo.asJsonObjects()) { if (packageId.equals(o.asString("package-id"))) { return o.asString("url"); } @@ -810,15 +795,15 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple if (canonical == null) { return null; } - checkBuildLoaded(); - if (buildInfo != null) { - for (JsonElement n : buildInfo) { + checkCIServerQueried(); + if (ciBuildInfo != null) { + for (JsonElement n : ciBuildInfo) { JsonObject o = (JsonObject) n; if (canonical.equals(o.asString("url"))) { return o.asString("package-id"); } } - for (JsonElement n : buildInfo) { + for (JsonElement n : ciBuildInfo) { JsonObject o = (JsonObject) n; if (o.asString("url").startsWith(canonical + "/ImplementationGuide/")) { return o.asString("package-id"); @@ -828,32 +813,39 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple return null; } - private NpmPackage checkCurrency(String id, NpmPackage p) { - checkBuildLoaded(); + /** + * Checks https:// + * + * @param id + * @param npmPackage + * @return + */ + private NpmPackage checkCurrency(String id, NpmPackage npmPackage) { + checkCIServerQueried(); // special case: current versions roll over, and we have to check their currency try { - String url = ciList.get(id); - JsonObject json = JsonParser.parseObjectFromUrl(Utilities.pathURL(url, "package.manifest.json")); - String currDate = json.asString("date"); - String packDate = p.date(); - if (!currDate.equals(packDate)) { + String packageManifestUrl = ciPackageList.get(id); + JsonObject packageManifestJson = JsonParser.parseObjectFromUrl(Utilities.pathURL(packageManifestUrl, "package.manifest.json")); + String currentDate = packageManifestJson.asString("date"); + String packageDate = npmPackage.date(); + if (!currentDate.equals(packageDate)) { return null; // nup, we need a new copy } } catch (Exception e) { log("Unable to check package currency: " + id + ": " + id); } - return p; + return npmPackage; } - private void checkBuildLoaded() { - if (!buildLoaded) { + private void checkCIServerQueried() { + if (System.currentTimeMillis() - ciQueryTimeStamp > ciQueryInterval) { try { - loadFromBuildServer(); + queryCIServer(); } catch (Exception e) { try { // we always pause a second and try again - the most common reason to be here is that the file was being changed on the server Thread.sleep(1000); - loadFromBuildServer(); + queryCIServer(); } catch (Exception e2) { log("Error connecting to build server - running without build (" + e2.getMessage() + ")"); } @@ -861,31 +853,31 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } } - private void loadFromBuildServer() throws IOException { + private void queryCIServer() throws IOException { HTTPResult res = ManagedWebAccess.get("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis()); res.checkThrowException(); - buildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent())); + ciBuildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent())); List builds = new ArrayList<>(); - for (JsonElement n : buildInfo) { - JsonObject o = (JsonObject) n; - if (o.has("url") && o.has("package-id") && o.asString("package-id").contains(".")) { - String u = o.asString("url"); - if (u.contains("/ImplementationGuide/")) - u = u.substring(0, u.indexOf("/ImplementationGuide/")); - builds.add(new BuildRecord(u, o.asString("package-id"), getRepo(o.asString("repo")), readDate(o.asString("date")))); + for (JsonElement n : ciBuildInfo) { + JsonObject j = (JsonObject) n; + if (j.has("url") && j.has("package-id") && j.asString("package-id").contains(".")) { + String packageUrl = j.asString("url"); + if (packageUrl.contains("/ImplementationGuide/")) + packageUrl = packageUrl.substring(0, packageUrl.indexOf("/ImplementationGuide/")); + builds.add(new BuildRecord(packageUrl, j.asString("package-id"), getRepo(j.asString("repo")), readDate(j.asString("date")))); } } Collections.sort(builds, new BuildRecordSorter()); - for (BuildRecord bld : builds) { - if (!ciList.containsKey(bld.getPackageId())) { - ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo()); + for (BuildRecord build : builds) { + if (!ciPackageList.containsKey(build.getPackageId())) { + ciPackageList.put(build.getPackageId(), "https://build.fhir.org/ig/" + build.getRepo()); } } - buildLoaded = true; + ciQueryTimeStamp = System.currentTimeMillis(); } private String getRepo(String path) { diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java index 54c8ee917..b41b26d05 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java @@ -640,11 +640,11 @@ public class NpmPackage { } - public void checkIndexed(String desc) throws IOException { + public void checkIndexed(String path) throws IOException { for (NpmPackageFolder folder : folders.values()) { JsonObject index = folder.index(); if (index == null || index.forceArray("files").size() == 0) { - indexFolder(desc, folder); + indexFolder(path, folder); } } } @@ -656,18 +656,18 @@ public class NpmPackage { * See the FHIR specification for details on .index.json * format and usage. * - * @param desc + * @param path * @param folder * @throws FileNotFoundException * @throws IOException */ - public void indexFolder(String desc, NpmPackageFolder folder) throws FileNotFoundException, IOException { + public void indexFolder(String path, NpmPackageFolder folder) throws FileNotFoundException, IOException { List remove = new ArrayList<>(); NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder(); indexer.start(folder.folder != null ? Utilities.path(folder.folder.getAbsolutePath(), ".index.db") : null); - for (String n : folder.listFiles()) { - if (!indexer.seeFile(n, folder.fetchFile(n))) { - remove.add(n); + for (String file : folder.listFiles()) { + if (!indexer.seeFile(file, folder.fetchFile(file))) { + remove.add(file); } } for (String n : remove) { @@ -684,7 +684,7 @@ public class NpmPackage { } } catch (Exception e) { TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json")); - throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e); + throw new IOException("Error parsing "+(path == null ? "" : path+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e); } } diff --git a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerTests.java b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerTests.java index 1221aeb66..30eaba253 100644 --- a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerTests.java +++ b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerTests.java @@ -50,7 +50,10 @@ public class FilesystemPackageManagerTests { new PackageServer(DUMMY_URL_4) ); + @Test + public void testCheckCurrentPackage() { + } @Test public void testDefaultServers() throws IOException { diff --git a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestProcessUtility.java b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestProcessUtility.java index d8f054f3d..74b4dde0e 100644 --- a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestProcessUtility.java +++ b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestProcessUtility.java @@ -105,7 +105,7 @@ public class LockfileTestProcessUtility { System.out.println("File "+lockFileName+" is locked. Waiting for " + seconds + " seconds to release. "); Thread.sleep(seconds * 1000L); - lockFile.renameTo(ManagedFileAccess.file(File.createTempFile(lockFile.getName(), ".lock-renamed").getAbsolutePath())); + lockFile.renameTo(File.createTempFile(lockFile.getName(), ".lock-renamed")); fileLock.release(); channel.close(); diff --git a/pom.xml b/pom.xml index e3fa0de1b..632e1337a 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ 1.26.0 32.0.1-jre 6.4.1 - 1.5.24 + 1.5.25-SNAPSHOT 2.17.0 5.9.2 1.8.2