honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WIP Switch to apache commons instead of nio for directory monitor

This commit is contained in:
dotasek 2024-09-18 18:40:13 -04:00
parent 048aa2abe5
commit b30134abfc
5 changed files with 175 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java
View File

@ -244,12 +244,15 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
if (file.getName().endsWith(".lock")) {
if (locks.getCacheLock().canLockFileBeHeldByThisProcess(file)) {
String packageDirectoryName = file.getName().substring(0, file.getName().length() - 5);
log("Detected potential incomplete package installed in cache: " + packageDirectoryName + ". Attempting to delete");
File packageDirectory = ManagedFileAccess.file(Utilities.path(cacheFolder, packageDirectoryName));
if (packageDirectory.exists()) {
Utilities.clearDirectory(packageDirectory.getAbsolutePath());
packageDirectory.delete();
}
file.delete();
log("Deleted potential incomplete package: " + packageDirectoryName);
}
}
}

24
org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerLockTests.java
View File

@ -124,9 +124,9 @@ public class FilesystemPackageManagerLockTests {
Assertions.assertTrue(filesystemPackageCacheLockManager.getCacheLock().canLockFileBeHeldByThisProcess(lockFile));
}
@Test void testWhenLockIsHelp_canLockFileBeHeldByThisProcessIsFalse() throws IOException, InterruptedException, TimeoutException {
@Test void testWhenLockIsHelp_canLockFileBeHeldByThisProcessIsFalse() throws InterruptedException, TimeoutException, IOException {
File lockFile = getPackageLockFile();
Thread lockThread = LockfileTestUtility.lockWaitAndDeleteInNewProcess(cachePath, DUMMY_PACKAGE + ".lock", 2);
Thread lockThread = LockfileTestProcessUtility.lockWaitAndDeleteInNewProcess(cachePath, DUMMY_PACKAGE + ".lock", 2);
LockfileTestUtility.waitForLockfileCreation(cacheDirectory.getAbsolutePath(), DUMMY_PACKAGE + ".lock");
@ -198,11 +198,11 @@ public class FilesystemPackageManagerLockTests {
}
@Test
public void testReadWhenLockedByFileTimesOut() throws IOException, InterruptedException, TimeoutException {
public void testReadWhenLockedByFileTimesOut() throws InterruptedException, TimeoutException, IOException {
FilesystemPackageCacheManagerLocks shorterTimeoutManager = filesystemPackageCacheLockManager;
final FilesystemPackageCacheManagerLocks.PackageLock packageLock = shorterTimeoutManager.getPackageLock(DUMMY_PACKAGE);
File lockFile = getPackageLockFile();
Thread lockThread = LockfileTestUtility.lockWaitAndDeleteInNewProcess(cachePath, lockFile.getName(), 5);
Thread lockThread = LockfileTestProcessUtility.lockWaitAndDeleteInNewProcess(cachePath, lockFile.getName(), 5);
LockfileTestUtility.waitForLockfileCreation(cachePath,lockFile.getName());
Exception exception = assertThrows(IOException.class, () -> {
@ -219,13 +219,25 @@ public class FilesystemPackageManagerLockTests {
}
@Test
public void testReadWhenLockFileIsDeleted() throws IOException, InterruptedException, TimeoutException {
public void apacheFileAlterationMonitorTest() {
// Use Apache FileAlterationMonitor to monitor the cache directory for file deletions
// and create a lock file in a separate thread
// Create a lock file in a separate thread
}
@Test
public void testReadWhenLockFileIsDeleted() throws InterruptedException, TimeoutException, IOException {
final FilesystemPackageCacheManagerLocks.PackageLock packageLock = filesystemPackageCacheLockManager.getPackageLock(DUMMY_PACKAGE);
final File lockFile = getPackageLockFile();
Thread lockThread = LockfileTestUtility.lockWaitAndDeleteInNewProcess(cachePath, lockFile.getName(), 5);
Thread lockThread = LockfileTestProcessUtility.lockWaitAndDeleteInNewProcess(cachePath, lockFile.getName(), 5);
LockfileTestUtility.waitForLockfileCreation(cachePath,lockFile.getName());
packageLock.doReadWithLock(() -> {

6
org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerTests.java
View File

@ -140,7 +140,7 @@ public class FilesystemPackageManagerTests {
File dummyPackage = createDummyPackage(cacheDirectory, "example.fhir.uv.myig", "1.2.3");
Thread lockThread = LockfileTestUtility.lockWaitAndDeleteInNewProcess(cacheDirectory.getAbsolutePath(), "example.fhir.uv.myig#1.2.3.lock", 2);
Thread lockThread = LockfileTestProcessUtility.lockWaitAndDeleteInNewProcess(cacheDirectory.getAbsolutePath(), "example.fhir.uv.myig#1.2.3.lock", 2);
LockfileTestUtility.waitForLockfileCreation(cacheDirectory.getAbsolutePath(), "example.fhir.uv.myig#1.2.3.lock");
File dummyLockFile = ManagedFileAccess.file(cacheDirectory.getAbsolutePath(), "example.fhir.uv.myig#1.2.3.lock");
@ -168,7 +168,7 @@ public class FilesystemPackageManagerTests {
Assertions.assertTrue(pcm.listPackages().isEmpty());
Thread lockThread = LockfileTestUtility.lockWaitAndDeleteInNewProcess(pcmPath, "example.fhir.uv.myig#1.2.3.lock", 10);
Thread lockThread = LockfileTestProcessUtility.lockWaitAndDeleteInNewProcess(pcmPath, "example.fhir.uv.myig#1.2.3.lock", 10);
File directory = ManagedFileAccess.file(pcmPath, "example.fhir.uv.myig#1.2.3" );
directory.mkdir();
@ -198,7 +198,7 @@ public class FilesystemPackageManagerTests {
File directory = ManagedFileAccess.file(pcmPath, packageAndVersion);
directory.mkdir();
Thread lockThread = LockfileTestUtility.lockWaitAndDeleteInNewProcess(pcmPath, "example.fhir.uv.myig#1.2.3.lock", 5);
Thread lockThread = LockfileTestProcessUtility.lockWaitAndDeleteInNewProcess(pcmPath, "example.fhir.uv.myig#1.2.3.lock", 5);
LockfileTestUtility.waitForLockfileCreation(pcmPath, "example.fhir.uv.myig#1.2.3.lock");
NpmPackage npmPackage = pcm.loadPackageFromCacheOnly("example.fhir.uv.myig", "1.2.3");

117
org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestProcessUtility.java Normal file
View File

@ -0,0 +1,117 @@
package org.hl7.fhir.utilities.npm;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
/**
* FilesystemPackageCacheManagerLocks relies on the existence of .lock files to prevent access to packages being written
* by processes outside the current JVM. Testing this functionality means creating a process outside the JUnit test JVM,
* which is achieved by running a separate Java process.
* <p/>
* Intended usage:
* <p/>
* The helper method {@link #lockWaitAndDeleteInNewProcess(String, String, int)} is the intended starting point for
* using this class.
* <p/>
*
*
* This class deliberately avoids using any dependencies outside java.*, which avoids having to construct a classpath
* for the separate process.
*/
public class LockfileTestProcessUtility {
/**
* Main method to allow running this class.
* <p/
* It is not recommended to call this method directly. Instead, use the provided {@link #lockWaitAndDeleteInNewProcess(String, String, int)} method.
*
* @param args The arguments to the main method. The first argument is the path to create the lockfile in, the second
* argument is the name of the lockfile, and the third argument is the number of seconds to wait before
* deleting the lockfile.
*/
public static void main(String[] args) {
String path = args[0];
String lockFileName = args[1];
int seconds = Integer.parseInt(args[2]);
try {
lockWaitAndDelete(path, lockFileName, seconds);
} catch (InterruptedException | IOException e) {
throw new RuntimeException(e);
}
}
/**
* Static helper method that starts a new process, creates a lock file in the path and waits for a specified number of
* seconds before deleting it.
* <p/>
* This method calls the {@link #main(String[])} method in a new process.
*
* @param path The path to create the lockfile in
* @param lockFileName The name of the lockfile
* @param seconds The number of seconds to wait before deleting the lockfile
* @return The thread wrapping the process execution. This can be used to wait for the process to complete, so that
* System.out and System.err can be processed before tests return results.
*/
public static Thread lockWaitAndDeleteInNewProcess(String path, String lockFileName, int seconds) {
Thread t = new Thread(() -> {
ProcessBuilder processBuilder = new ProcessBuilder("java", "-cp", "target/test-classes:.", LockfileTestProcessUtility.class.getName(), path, lockFileName, Integer.toString(seconds));
try {
Process process = processBuilder.start();
process.getErrorStream().transferTo(System.err);
process.getInputStream().transferTo(System.out);
process.waitFor();
} catch (IOException | InterruptedException e) {
throw new RuntimeException(e);
}
});
t.start();
return t;
}
/**
* The actual logic to create a .lock file.
* <p/>
* This should match the logic in FilesystemPackageCacheManagerLocks
* <p/>
*
* @param path The path to create the lockfile in
* @param lockFileName The name of the lockfile
* @param seconds The number of seconds to wait before deleting the lockfile
* @throws InterruptedException If the thread is interrupted while waiting
* @throws IOException If there is an error accessing the file system
*/
/* TODO Eventually, this logic should exist in a Lockfile class so that it isn't duplicated between the main code and
the test code.
*/
private static void lockWaitAndDelete(String path, String lockFileName, int seconds) throws InterruptedException, IOException {
File file = Paths.get(path,lockFileName).toFile();
try (FileChannel channel = new RandomAccessFile(file.getAbsolutePath(), "rw").getChannel()) {
FileLock fileLock = channel.tryLock(0, Long.MAX_VALUE, false);
if (fileLock != null) {
final ByteBuffer buff = ByteBuffer.wrap("Hello world".getBytes(StandardCharsets.UTF_8));
channel.write(buff);
System.out.println("File "+lockFileName+" is locked. Waiting for " + seconds + " seconds to release. ");
Thread.sleep(seconds * 1000L);
file.delete();
fileLock.release();
System.out.println(System.currentTimeMillis());
System.out.println("File "+lockFileName+" is released.");
channel.close();
}}finally {
if (file.exists()) {
file.delete();
}
}
}
}

155
org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestUtility.java
View File

@ -1,52 +1,21 @@
package org.hl7.fhir.utilities.npm;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationMonitor;
import org.apache.commons.io.monitor.FileAlterationObserver;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* FilesystemPackageCacheManagerLocks relies on the existence of .lock files to prevent access to packages being written
* by processes outside the current JVM. Testing this functionality means creating a process outside the JUnit test JVM,
* which is achieved by running a separate Java process.
* <p/>
* Intended usage:
* <p/>
* The helper method {@link #lockWaitAndDeleteInNewProcess(String, String, int)} is the intended starting point for
* using this class.
* <p/>
*
*
* This class deliberately avoids using any dependencies outside java.*, which avoids having to construct a classpath
* for the separate process.
*/
public class LockfileTestUtility {
/**
* Main method to allow running this class.
*
* It is not recommended to call this method directly. Instead, use the provided {@link LockfileTestUtility.l} method.
*
*
* @param args
*/
public static void main(String[] args) {
String lockFileName = args[1];
String path = args[0];
int seconds = Integer.parseInt(args[2]);
try {
lockWaitAndDelete(path, lockFileName, seconds);
} catch (InterruptedException | IOException e) {
throw new RuntimeException(e);
}
}
/**
* Wait for the lock file to be created in the given path.
@ -57,98 +26,42 @@ public class LockfileTestUtility {
* @param path The path containing the lock file
* @param lockFileName The name of the lock file
* @throws InterruptedException If the thread is interrupted while waiting
* @throws IOException If there is an error accessing the file system
* @throws TimeoutException If the lock file is not created within 10 seconds
*/
public static void waitForLockfileCreation(String path, String lockFileName) throws InterruptedException, IOException, TimeoutException {
public static void waitForLockfileCreation(String path, String lockFileName) throws InterruptedException, TimeoutException {
if (Files.exists(Paths.get(path, lockFileName))) {
return;
}
try (WatchService watchService = FileSystems.getDefault().newWatchService()) {
Path dir = Paths.get(path);
dir.register(watchService, StandardWatchEventKinds.ENTRY_CREATE);
CountDownLatch latch = new CountDownLatch(1);
FileAlterationMonitor monitor = new FileAlterationMonitor(100);
FileAlterationObserver observer = new FileAlterationObserver(path);
WatchKey key = watchService.poll(10, TimeUnit.SECONDS);
if (key == null) {
throw new TimeoutException("Timeout waiting for lock file creation: " + lockFileName);
}
for (WatchEvent<?> event : key.pollEvents()) {
WatchEvent.Kind<?> kind = event.kind();
if (kind == StandardWatchEventKinds.ENTRY_CREATE) {
Path createdFile = (Path) event.context();
if (createdFile.toString().equals(lockFileName)) {
System.out.println("Lock file created: " + lockFileName);
return;
}
}
}
throw new TimeoutException("Timeout waiting for lock file creation: " + lockFileName);
}
}
/**
* Static helper method that starts a new process, creates a lock file in the path and waits for a specified number of
* seconds before deleting it.
* <p/>
* This method calls the {@link #main(String[])} method in a new process.
*
* @param path The path to create the lockfile in
* @param lockFileName The name of the lockfile
* @param seconds The number of seconds to wait before deleting the lockfile
* @return The thread wrapping the process execution. This can be used to wait for the process to complete, so that
* System.out and System.err can be processed before tests return results.
*/
public static Thread lockWaitAndDeleteInNewProcess(String path, String lockFileName, int seconds) {
Thread t = new Thread(() -> {
ProcessBuilder processBuilder = new ProcessBuilder("java", "-cp", "target/test-classes:.", LockfileTestUtility.class.getName(), path, lockFileName, Integer.toString(seconds));
try {
Process process = processBuilder.start();
process.getErrorStream().transferTo(System.err);
process.getInputStream().transferTo(System.out);
process.waitFor();
} catch (IOException | InterruptedException e) {
throw new RuntimeException(e);
observer.addListener(new FileAlterationListenerAdaptor(){
@Override
public void onFileCreate(File file) {
System.out.println("File created: " + file.getName());
latch.countDown();
}
});
t.start();
return t;
}
monitor.addObserver(observer);
/**
* The actual logic to create a .lock file.
* <p/>
* This should match the logic in FilesystemPackageCacheManagerLocks
* <p/>
*
* @param path The path to create the lockfile in
* @param lockFileName The name of the lockfile
* @param seconds The number of seconds to wait before deleting the lockfile
* @throws InterruptedException If the thread is interrupted while waiting
* @throws IOException If there is an error accessing the file system
*/
/* TODO Eventually, this logic should exist in a Lockfile class so that it isn't duplicated between the main code and
the test code.
*/
private static void lockWaitAndDelete(String path, String lockFileName, int seconds) throws InterruptedException, IOException {
File file = Paths.get(path,lockFileName).toFile();
try (FileChannel channel = new RandomAccessFile(file.getAbsolutePath(), "rw").getChannel()) {
FileLock fileLock = channel.tryLock(0, Long.MAX_VALUE, false);
if (fileLock != null) {
final ByteBuffer buff = ByteBuffer.wrap("Hello world".getBytes(StandardCharsets.UTF_8));
channel.write(buff);
System.out.println("File "+lockFileName+" is locked. Waiting for " + seconds + " seconds to release. ");
Thread.sleep(seconds * 1000L);
fileLock.release();
System.out.println(System.currentTimeMillis());
System.out.println("File "+lockFileName+" is released.");
channel.close();
}}finally {
if (file.exists()) {
file.delete();
}
try {
monitor.start();
} catch (Exception e) {
throw new RuntimeException(e);
}
latch.await(10, TimeUnit.SECONDS);
try {
monitor.stop();
} catch (Exception e) {
throw new RuntimeException(e);
}
if (!Files.exists(Paths.get(path, lockFileName))) {
throw new TimeoutException("Lock file not created within 10 seconds");
}
}
}