honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix problem with package cache corruption when different processes install the same package at the same time

This commit is contained in:
Grahame Grieve 2020-05-27 07:08:36 +10:00
parent ccb6b067b3
commit 70c75fde62
1 changed files with 121 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java vendored
View File

@ -35,11 +35,15 @@ import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
@ -186,6 +190,42 @@ public class PackageCacheManager {
} }
} }
public interface CacheLockFunction<T> {
T get() throws IOException;
}
public class CacheLock {
private final File lockFile;
public CacheLock(String name) throws IOException {
this.lockFile = new File(cacheFolder, name + ".lock");
if (!lockFile.isFile()) {
TextFile.stringToFile("", lockFile);
}
}
public <T> T doWithLock(CacheLockFunction<T> f) throws FileNotFoundException, IOException {
try (FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel()) {
final FileLock fileLock = channel.lock();
T result = null;
try {
result = f.get();
} finally {
fileLock.release();
}
try {
if (!lockFile.delete()) {
lockFile.deleteOnExit();
}
} catch (Throwable t) {
System.out.println("unable to clean up lock file for "+lockFile.getName()+": "+t.getMessage());
// nothing
}
return result;
}
}
}
public static final String PRIMARY_SERVER = "http://packages.fhir.org"; public static final String PRIMARY_SERVER = "http://packages.fhir.org";
public static final String SECONDARY_SERVER = "http://packages2.fhir.org/packages"; public static final String SECONDARY_SERVER = "http://packages2.fhir.org/packages";
// private static final String SECONDARY_SERVER = "http://local.fhir.org:960/packages"; // private static final String SECONDARY_SERVER = "http://local.fhir.org:960/packages";
@ -256,16 +296,19 @@ public class PackageCacheManager {
private void clearCache() throws IOException { private void clearCache() throws IOException {
for (File f : new File(cacheFolder).listFiles()) { for (File f : new File(cacheFolder).listFiles()) {
if (f.isDirectory()) { if (f.isDirectory()) {
Utilities.clearDirectory(f.getAbsolutePath()); new CacheLock(f.getName()).doWithLock(() -> {
try { Utilities.clearDirectory(f.getAbsolutePath());
FileUtils.deleteDirectory(f);
} catch (Exception e1) {
try { try {
FileUtils.deleteDirectory(f); FileUtils.deleteDirectory(f);
} catch (Exception e1) {
try {
FileUtils.deleteDirectory(f);
} catch (Exception e2) { } catch (Exception e2) {
// just give up // just give up
} }
} }
return null; // must return something
});
} }
else if (!f.getName().equals("packages.ini")) else if (!f.getName().equals("packages.ini"))
FileUtils.forceDelete(f); FileUtils.forceDelete(f);
@ -413,15 +456,18 @@ public class PackageCacheManager {
* @throws IOException * @throws IOException
*/ */
public void removePackage(String id, String ver) throws IOException { public void removePackage(String id, String ver) throws IOException {
String f = Utilities.path(cacheFolder, id+"#"+ver); new CacheLock(id+"#"+ver).doWithLock(() -> {
File ff = new File(f); String f = Utilities.path(cacheFolder, id+"#"+ver);
if (ff.exists()) { File ff = new File(f);
Utilities.clearDirectory(f); if (ff.exists()) {
IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); Utilities.clearDirectory(f);
ini.removeProperty("packages", id+"#"+ver); IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
ini.save(); ini.removeProperty("packages", id+"#"+ver);
ff.delete(); ini.save();
} ff.delete();
}
return null;
});
} }
/** /**
@ -494,70 +540,76 @@ public class PackageCacheManager {
if (version == null) if (version == null)
version = npm.version(); version = npm.version();
String packRoot = Utilities.path(cacheFolder, id+"#"+version); String v = version;
try { return new CacheLock(id+"#"+version).doWithLock(() -> {
Utilities.createDirectory(packRoot); NpmPackage pck = null;
Utilities.clearDirectory(packRoot); String packRoot = Utilities.path(cacheFolder, id+"#"+v);
try {
Utilities.createDirectory(packRoot);
Utilities.clearDirectory(packRoot);
int i = 0; int i = 0;
int c = 0; int c = 0;
int size = 0; int size = 0;
for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) { for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());; String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());;
if (!(new File(dir).exists())) if (!(new File(dir).exists()))
Utilities.createDirectory(dir); Utilities.createDirectory(dir);
for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) { for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
String fn = Utilities.path(dir, fe.getKey()); String fn = Utilities.path(dir, fe.getKey());
byte[] cnt = fe.getValue(); byte[] cnt = fe.getValue();
TextFile.bytesToFile(cnt, fn); TextFile.bytesToFile(cnt, fn);
size = size + cnt.length; size = size + cnt.length;
i++; i++;
if (progress && i % 50 == 0) { if (progress && i % 50 == 0) {
c++; c++;
System.out.print("."); System.out.print(".");
if (c == 120) { if (c == 120) {
System.out.println(""); System.out.println("");
System.out.print(" "); System.out.print(" ");
c = 2; c = 2;
}
} }
} }
} }
}
IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
ini.setTimeStampFormat("yyyyMMddhhmmss"); ini.setTimeStampFormat("yyyyMMddhhmmss");
ini.setTimestampProperty("packages", id+"#"+version, Timestamp.from(Instant.now()), null); ini.setTimestampProperty("packages", id+"#"+v, Timestamp.from(Instant.now()), null);
ini.setIntegerProperty("package-sizes", id+"#"+version, size, null); ini.setIntegerProperty("package-sizes", id+"#"+v, size, null);
ini.save(); ini.save();
if (progress) if (progress)
System.out.println(" done."); System.out.println(" done.");
NpmPackage pck = loadPackageInfo(packRoot); pck = loadPackageInfo(packRoot);
if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !version.equals(JSONUtil.str(npm.getNpm(), "version"))) { if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !v.equals(JSONUtil.str(npm.getNpm(), "version"))) {
if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) { if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) {
npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name")); npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name"));
npm.getNpm().remove("name"); npm.getNpm().remove("name");
npm.getNpm().addProperty("name", id); npm.getNpm().addProperty("name", id);
}
if (!v.equals(JSONUtil.str(npm.getNpm(), "version"))) {
npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version"));
npm.getNpm().remove("version");
npm.getNpm().addProperty("version", v);
}
TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+v, "package", "package.json"), false);
} }
if (!version.equals(JSONUtil.str(npm.getNpm(), "version"))) { } catch (Exception e) {
npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version")); try {
npm.getNpm().remove("version"); // don't leave a half extracted package behind
npm.getNpm().addProperty("version", version); System.out.println("Clean up package "+packRoot+" because installation failed: "+e.getMessage());
e.printStackTrace();
Utilities.clearDirectory(packRoot);
new File(packRoot).delete();
} catch (Exception ei) {
// nothing
} }
TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+version, "package", "package.json"), false); throw e;
} }
return pck; return pck;
} catch (Exception e) { });
try {
// don't leave a half extracted package behind
Utilities.clearDirectory(packRoot);
new File(packRoot).delete();
} catch (Exception ei) {
// nothing
}
throw e;
}
} }
public String getPackageId(String canonical) throws IOException { public String getPackageId(String canonical) throws IOException {