fix problem with package cache corruption when different processes install the same package at the same time
This commit is contained in:
Grahame Grieve
parent
ccb6b067b3
commit
70c75fde62
|
|
@ -35,11 +35,15 @@ import java.io.BufferedOutputStream;
|
|||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.channels.FileLock;
|
||||
import java.sql.Timestamp;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
|
|
@ -186,6 +190,42 @@ public class PackageCacheManager {
|
|||
}
|
||||
}
|
||||
|
||||
public interface CacheLockFunction<T> {
|
||||
T get() throws IOException;
|
||||
}
|
||||
public class CacheLock {
|
||||
|
||||
private final File lockFile;
|
||||
|
||||
public CacheLock(String name) throws IOException {
|
||||
this.lockFile = new File(cacheFolder, name + ".lock");
|
||||
if (!lockFile.isFile()) {
|
||||
TextFile.stringToFile("", lockFile);
|
||||
}
|
||||
}
|
||||
|
||||
public <T> T doWithLock(CacheLockFunction<T> f) throws FileNotFoundException, IOException {
|
||||
try (FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel()) {
|
||||
final FileLock fileLock = channel.lock();
|
||||
T result = null;
|
||||
try {
|
||||
result = f.get();
|
||||
} finally {
|
||||
fileLock.release();
|
||||
}
|
||||
try {
|
||||
if (!lockFile.delete()) {
|
||||
lockFile.deleteOnExit();
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
System.out.println("unable to clean up lock file for "+lockFile.getName()+": "+t.getMessage());
|
||||
// nothing
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static final String PRIMARY_SERVER = "http://packages.fhir.org";
|
||||
public static final String SECONDARY_SERVER = "http://packages2.fhir.org/packages";
|
||||
// private static final String SECONDARY_SERVER = "http://local.fhir.org:960/packages";
|
||||
|
|
@ -256,16 +296,19 @@ public class PackageCacheManager {
|
|||
private void clearCache() throws IOException {
|
||||
for (File f : new File(cacheFolder).listFiles()) {
|
||||
if (f.isDirectory()) {
|
||||
Utilities.clearDirectory(f.getAbsolutePath());
|
||||
try {
|
||||
FileUtils.deleteDirectory(f);
|
||||
} catch (Exception e1) {
|
||||
new CacheLock(f.getName()).doWithLock(() -> {
|
||||
Utilities.clearDirectory(f.getAbsolutePath());
|
||||
try {
|
||||
FileUtils.deleteDirectory(f);
|
||||
} catch (Exception e1) {
|
||||
try {
|
||||
FileUtils.deleteDirectory(f);
|
||||
} catch (Exception e2) {
|
||||
// just give up
|
||||
}
|
||||
}
|
||||
}
|
||||
return null; // must return something
|
||||
});
|
||||
}
|
||||
else if (!f.getName().equals("packages.ini"))
|
||||
FileUtils.forceDelete(f);
|
||||
|
|
@ -413,15 +456,18 @@ public class PackageCacheManager {
|
|||
* @throws IOException
|
||||
*/
|
||||
public void removePackage(String id, String ver) throws IOException {
|
||||
String f = Utilities.path(cacheFolder, id+"#"+ver);
|
||||
File ff = new File(f);
|
||||
if (ff.exists()) {
|
||||
Utilities.clearDirectory(f);
|
||||
IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
|
||||
ini.removeProperty("packages", id+"#"+ver);
|
||||
ini.save();
|
||||
ff.delete();
|
||||
}
|
||||
new CacheLock(id+"#"+ver).doWithLock(() -> {
|
||||
String f = Utilities.path(cacheFolder, id+"#"+ver);
|
||||
File ff = new File(f);
|
||||
if (ff.exists()) {
|
||||
Utilities.clearDirectory(f);
|
||||
IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
|
||||
ini.removeProperty("packages", id+"#"+ver);
|
||||
ini.save();
|
||||
ff.delete();
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -494,70 +540,76 @@ public class PackageCacheManager {
|
|||
if (version == null)
|
||||
version = npm.version();
|
||||
|
||||
String packRoot = Utilities.path(cacheFolder, id+"#"+version);
|
||||
try {
|
||||
Utilities.createDirectory(packRoot);
|
||||
Utilities.clearDirectory(packRoot);
|
||||
String v = version;
|
||||
return new CacheLock(id+"#"+version).doWithLock(() -> {
|
||||
NpmPackage pck = null;
|
||||
String packRoot = Utilities.path(cacheFolder, id+"#"+v);
|
||||
try {
|
||||
Utilities.createDirectory(packRoot);
|
||||
Utilities.clearDirectory(packRoot);
|
||||
|
||||
int i = 0;
|
||||
int c = 0;
|
||||
int size = 0;
|
||||
for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
|
||||
String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());;
|
||||
if (!(new File(dir).exists()))
|
||||
Utilities.createDirectory(dir);
|
||||
for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
|
||||
String fn = Utilities.path(dir, fe.getKey());
|
||||
byte[] cnt = fe.getValue();
|
||||
TextFile.bytesToFile(cnt, fn);
|
||||
size = size + cnt.length;
|
||||
i++;
|
||||
if (progress && i % 50 == 0) {
|
||||
c++;
|
||||
System.out.print(".");
|
||||
if (c == 120) {
|
||||
System.out.println("");
|
||||
System.out.print(" ");
|
||||
c = 2;
|
||||
}
|
||||
}
|
||||
int i = 0;
|
||||
int c = 0;
|
||||
int size = 0;
|
||||
for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
|
||||
String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());;
|
||||
if (!(new File(dir).exists()))
|
||||
Utilities.createDirectory(dir);
|
||||
for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
|
||||
String fn = Utilities.path(dir, fe.getKey());
|
||||
byte[] cnt = fe.getValue();
|
||||
TextFile.bytesToFile(cnt, fn);
|
||||
size = size + cnt.length;
|
||||
i++;
|
||||
if (progress && i % 50 == 0) {
|
||||
c++;
|
||||
System.out.print(".");
|
||||
if (c == 120) {
|
||||
System.out.println("");
|
||||
System.out.print(" ");
|
||||
c = 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
|
||||
ini.setTimeStampFormat("yyyyMMddhhmmss");
|
||||
ini.setTimestampProperty("packages", id+"#"+version, Timestamp.from(Instant.now()), null);
|
||||
ini.setIntegerProperty("package-sizes", id+"#"+version, size, null);
|
||||
ini.save();
|
||||
if (progress)
|
||||
System.out.println(" done.");
|
||||
IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
|
||||
ini.setTimeStampFormat("yyyyMMddhhmmss");
|
||||
ini.setTimestampProperty("packages", id+"#"+v, Timestamp.from(Instant.now()), null);
|
||||
ini.setIntegerProperty("package-sizes", id+"#"+v, size, null);
|
||||
ini.save();
|
||||
if (progress)
|
||||
System.out.println(" done.");
|
||||
|
||||
NpmPackage pck = loadPackageInfo(packRoot);
|
||||
if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !version.equals(JSONUtil.str(npm.getNpm(), "version"))) {
|
||||
if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) {
|
||||
npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name"));
|
||||
npm.getNpm().remove("name");
|
||||
npm.getNpm().addProperty("name", id);
|
||||
pck = loadPackageInfo(packRoot);
|
||||
if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !v.equals(JSONUtil.str(npm.getNpm(), "version"))) {
|
||||
if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) {
|
||||
npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name"));
|
||||
npm.getNpm().remove("name");
|
||||
npm.getNpm().addProperty("name", id);
|
||||
}
|
||||
if (!v.equals(JSONUtil.str(npm.getNpm(), "version"))) {
|
||||
npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version"));
|
||||
npm.getNpm().remove("version");
|
||||
npm.getNpm().addProperty("version", v);
|
||||
}
|
||||
TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+v, "package", "package.json"), false);
|
||||
}
|
||||
if (!version.equals(JSONUtil.str(npm.getNpm(), "version"))) {
|
||||
npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version"));
|
||||
npm.getNpm().remove("version");
|
||||
npm.getNpm().addProperty("version", version);
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
// don't leave a half extracted package behind
|
||||
System.out.println("Clean up package "+packRoot+" because installation failed: "+e.getMessage());
|
||||
e.printStackTrace();
|
||||
Utilities.clearDirectory(packRoot);
|
||||
new File(packRoot).delete();
|
||||
} catch (Exception ei) {
|
||||
// nothing
|
||||
}
|
||||
TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+version, "package", "package.json"), false);
|
||||
throw e;
|
||||
}
|
||||
return pck;
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
// don't leave a half extracted package behind
|
||||
Utilities.clearDirectory(packRoot);
|
||||
new File(packRoot).delete();
|
||||
} catch (Exception ei) {
|
||||
// nothing
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public String getPackageId(String canonical) throws IOException {
|
||||
|
|
|
|||
Loading…
Reference in New Issue