From 70c75fde6273084fc5144e15f92b8f40ce733f5c Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Wed, 27 May 2020 07:08:36 +1000 Subject: [PATCH] fix problem with package cache corruption when different processes install the same package at the same time --- .../utilities/cache/PackageCacheManager.java | 190 +++++++++++------- 1 file changed, 121 insertions(+), 69 deletions(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java index c35172733..c2a99edd3 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java @@ -35,11 +35,15 @@ import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -186,6 +190,42 @@ public class PackageCacheManager { } } + public interface CacheLockFunction { + T get() throws IOException; + } + public class CacheLock { + + private final File lockFile; + + public CacheLock(String name) throws IOException { + this.lockFile = new File(cacheFolder, name + ".lock"); + if (!lockFile.isFile()) { + TextFile.stringToFile("", lockFile); + } + } + + public T doWithLock(CacheLockFunction f) throws FileNotFoundException, IOException { + try (FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel()) { + final FileLock fileLock = channel.lock(); + T result = null; + try { + result = f.get(); + } finally { + fileLock.release(); + } + try { + if (!lockFile.delete()) { + lockFile.deleteOnExit(); + } + } catch (Throwable t) { + System.out.println("unable to clean up lock file for "+lockFile.getName()+": "+t.getMessage()); + // nothing + } + return result; + } + } + } + public static final String PRIMARY_SERVER = "http://packages.fhir.org"; public static final String SECONDARY_SERVER = "http://packages2.fhir.org/packages"; // private static final String SECONDARY_SERVER = "http://local.fhir.org:960/packages"; @@ -256,16 +296,19 @@ public class PackageCacheManager { private void clearCache() throws IOException { for (File f : new File(cacheFolder).listFiles()) { if (f.isDirectory()) { - Utilities.clearDirectory(f.getAbsolutePath()); - try { - FileUtils.deleteDirectory(f); - } catch (Exception e1) { + new CacheLock(f.getName()).doWithLock(() -> { + Utilities.clearDirectory(f.getAbsolutePath()); try { FileUtils.deleteDirectory(f); + } catch (Exception e1) { + try { + FileUtils.deleteDirectory(f); } catch (Exception e2) { // just give up } - } + } + return null; // must return something + }); } else if (!f.getName().equals("packages.ini")) FileUtils.forceDelete(f); @@ -413,15 +456,18 @@ public class PackageCacheManager { * @throws IOException */ public void removePackage(String id, String ver) throws IOException { - String f = Utilities.path(cacheFolder, id+"#"+ver); - File ff = new File(f); - if (ff.exists()) { - Utilities.clearDirectory(f); - IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); - ini.removeProperty("packages", id+"#"+ver); - ini.save(); - ff.delete(); - } + new CacheLock(id+"#"+ver).doWithLock(() -> { + String f = Utilities.path(cacheFolder, id+"#"+ver); + File ff = new File(f); + if (ff.exists()) { + Utilities.clearDirectory(f); + IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); + ini.removeProperty("packages", id+"#"+ver); + ini.save(); + ff.delete(); + } + return null; + }); } /** @@ -494,70 +540,76 @@ public class PackageCacheManager { if (version == null) version = npm.version(); - String packRoot = Utilities.path(cacheFolder, id+"#"+version); - try { - Utilities.createDirectory(packRoot); - Utilities.clearDirectory(packRoot); + String v = version; + return new CacheLock(id+"#"+version).doWithLock(() -> { + NpmPackage pck = null; + String packRoot = Utilities.path(cacheFolder, id+"#"+v); + try { + Utilities.createDirectory(packRoot); + Utilities.clearDirectory(packRoot); - int i = 0; - int c = 0; - int size = 0; - for (Entry e : npm.getFolders().entrySet()) { - String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());; - if (!(new File(dir).exists())) - Utilities.createDirectory(dir); - for (Entry fe : e.getValue().getContent().entrySet()) { - String fn = Utilities.path(dir, fe.getKey()); - byte[] cnt = fe.getValue(); - TextFile.bytesToFile(cnt, fn); - size = size + cnt.length; - i++; - if (progress && i % 50 == 0) { - c++; - System.out.print("."); - if (c == 120) { - System.out.println(""); - System.out.print(" "); - c = 2; - } - } + int i = 0; + int c = 0; + int size = 0; + for (Entry e : npm.getFolders().entrySet()) { + String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());; + if (!(new File(dir).exists())) + Utilities.createDirectory(dir); + for (Entry fe : e.getValue().getContent().entrySet()) { + String fn = Utilities.path(dir, fe.getKey()); + byte[] cnt = fe.getValue(); + TextFile.bytesToFile(cnt, fn); + size = size + cnt.length; + i++; + if (progress && i % 50 == 0) { + c++; + System.out.print("."); + if (c == 120) { + System.out.println(""); + System.out.print(" "); + c = 2; + } + } + } } - } - IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); - ini.setTimeStampFormat("yyyyMMddhhmmss"); - ini.setTimestampProperty("packages", id+"#"+version, Timestamp.from(Instant.now()), null); - ini.setIntegerProperty("package-sizes", id+"#"+version, size, null); - ini.save(); - if (progress) - System.out.println(" done."); + IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); + ini.setTimeStampFormat("yyyyMMddhhmmss"); + ini.setTimestampProperty("packages", id+"#"+v, Timestamp.from(Instant.now()), null); + ini.setIntegerProperty("package-sizes", id+"#"+v, size, null); + ini.save(); + if (progress) + System.out.println(" done."); - NpmPackage pck = loadPackageInfo(packRoot); - if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !version.equals(JSONUtil.str(npm.getNpm(), "version"))) { - if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) { - npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name")); - npm.getNpm().remove("name"); - npm.getNpm().addProperty("name", id); + pck = loadPackageInfo(packRoot); + if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !v.equals(JSONUtil.str(npm.getNpm(), "version"))) { + if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) { + npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name")); + npm.getNpm().remove("name"); + npm.getNpm().addProperty("name", id); + } + if (!v.equals(JSONUtil.str(npm.getNpm(), "version"))) { + npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version")); + npm.getNpm().remove("version"); + npm.getNpm().addProperty("version", v); + } + TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+v, "package", "package.json"), false); } - if (!version.equals(JSONUtil.str(npm.getNpm(), "version"))) { - npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version")); - npm.getNpm().remove("version"); - npm.getNpm().addProperty("version", version); + } catch (Exception e) { + try { + // don't leave a half extracted package behind + System.out.println("Clean up package "+packRoot+" because installation failed: "+e.getMessage()); + e.printStackTrace(); + Utilities.clearDirectory(packRoot); + new File(packRoot).delete(); + } catch (Exception ei) { + // nothing } - TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+version, "package", "package.json"), false); + throw e; } return pck; - } catch (Exception e) { - try { - // don't leave a half extracted package behind - Utilities.clearDirectory(packRoot); - new File(packRoot).delete(); - } catch (Exception ei) { - // nothing - } - throw e; - } + }); } public String getPackageId(String canonical) throws IOException {