ARTEMIS-1058 Improving web temporary cleanup

* Using SpawnedVMSupport (used to be on testsuite, moving it to Utils)
* Building the classpath for ./lib, similar to what happens on Bootstrap
* Using Path as much as possible to avoid issues encoding files
This commit is contained in:
Clebert Suconic 2019-02-04 10:20:01 -05:00
parent 8873beb828
commit 59ada66033
23 changed files with 160 additions and 109 deletions

View File

@ -14,35 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.artemis.tests.util;
package org.apache.activemq.artemis.utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.activemq.artemis.tests.unit.UnitTestLogger;
import org.apache.activemq.artemis.utils.ActiveMQThreadFactory;
import org.junit.Assert;
import org.jboss.logging.Logger;
import static java.util.concurrent.TimeUnit.SECONDS;
public final class SpawnedVMSupport {
public class SpawnedVMSupport {
static ConcurrentHashMap<Process, String> startedProcesses = null;
private static final UnitTestLogger log = UnitTestLogger.LOGGER;
private static final Logger log = Logger.getLogger(SpawnedVMSupport.class);
public static Process spawnVM(final String className, final String... args) throws Exception {
return SpawnedVMSupport.spawnVM(className, new String[0], true, args);
@ -53,6 +44,12 @@ public final class SpawnedVMSupport {
final String... args) throws Exception {
return SpawnedVMSupport.spawnVM(className, new String[0], logOutput, args);
}
public static Process spawnVM(final String classPath,
final String className,
final boolean logOutput,
final String... args) throws Exception {
return SpawnedVMSupport.spawnVM(classPath, className, new String[0], logOutput, args);
}
public static Process spawnVM(final String className, final String[] vmargs, final String... args) throws Exception {
return SpawnedVMSupport.spawnVM(className, vmargs, true, args);
@ -65,6 +62,14 @@ public final class SpawnedVMSupport {
return SpawnedVMSupport.spawnVM(className, "-Xms512m", "-Xmx512m", vmargs, logOutput, true, true, args);
}
public static Process spawnVM(final String classpath,
final String className,
final String[] vmargs,
final boolean logOutput,
final String... args) throws Exception {
return SpawnedVMSupport.spawnVM(classpath, className, "-Xms512m", "-Xmx512m", vmargs, logOutput, true, true, args);
}
public static Process spawnVMWithLogMacher(String wordMatch,
Runnable runnable,
final String className,
@ -85,6 +90,18 @@ public final class SpawnedVMSupport {
return spawnVM(null, null, className, memoryArg1, memoryArg2, vmargs, logOutput, logErrorOutput, useLogging, args);
}
public static Process spawnVM(final String classPath,
final String className,
final String memoryArg1,
final String memoryArg2,
final String[] vmargs,
final boolean logOutput,
final boolean logErrorOutput,
final boolean useLogging,
final String... args) throws Exception {
return spawnVM(classPath, null, null, className, memoryArg1, memoryArg2, vmargs, logOutput, logErrorOutput, useLogging, args);
}
public static Process spawnVM(final String wordMatch,
final Runnable wordRunning,
final String className,
@ -95,7 +112,7 @@ public final class SpawnedVMSupport {
final boolean logErrorOutput,
final boolean useLogging,
final String... args) throws Exception {
return spawnVM(System.getProperty("java.class.path"), wordMatch, wordRunning, className, memoryArg1, memoryArg2, vmargs, logOutput, logErrorOutput, useLogging, args);
return spawnVM(getClassPath(), wordMatch, wordRunning, className, memoryArg1, memoryArg2, vmargs, logOutput, logErrorOutput, useLogging, args);
}
@ -113,6 +130,30 @@ public final class SpawnedVMSupport {
return spawnVM(classPath, wordMatch, wordRunning, className, memoryArg1, memoryArg2, vmargs, logOutput, logErrorOutput, useLogging, -1, args);
}
public static String getClassPath() {
return System.getProperty("java.class.path");
}
public static String getClassPath(File libfolder) {
if (libfolder == null) {
return getClassPath();
}
StringBuilder stringBuilder = new StringBuilder();
boolean empty = true;
for (File f : libfolder.listFiles()) {
if (f.getName().endsWith(".jar") || f.getName().endsWith(".zip")) {
if (!empty) {
stringBuilder.append(File.pathSeparator);
}
empty = false;
stringBuilder.append(f.toString());
}
}
return stringBuilder.toString();
}
/**
* @param classPath
* @param wordMatch
@ -259,7 +300,10 @@ public final class SpawnedVMSupport {
startedProcesses = new ConcurrentHashMap<>();
}
public static void checkProcess() {
/**
* Check if all spawned processes are finished.
*/
public static boolean checkProcess() {
HashSet<Process> aliveProcess = getAliveProcesses();
@ -270,11 +314,13 @@ public final class SpawnedVMSupport {
alive.destroyForcibly();
buffer.append(startedProcesses.get(alive) + " ");
}
Assert.fail("There are " + aliveProcess.size() + " processes alive :: " + buffer.toString());
return false;
}
} finally {
startedProcesses = null;
}
return true;
}
/**
@ -300,36 +346,6 @@ public final class SpawnedVMSupport {
startLogger(true, null, null, className, process);
}
/**
* Assert that a process exits with the expected value (or not depending if
* the <code>sameValue</code> is expected or not). The method waits 5
* seconds for the process to exit, then an Exception is thrown. In any case,
* the process is destroyed before the method returns.
*/
public static void assertProcessExits(final boolean sameValue,
final int value,
final Process p) throws InterruptedException, ExecutionException, TimeoutException {
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(ActiveMQThreadFactory.defaultThreadFactory());
Future<Integer> future = executor.submit(new Callable<Integer>() {
@Override
public Integer call() throws Exception {
p.waitFor();
return p.exitValue();
}
});
try {
int exitValue = future.get(10, SECONDS);
if (sameValue) {
Assert.assertSame(value, exitValue);
} else {
Assert.assertNotSame(value, exitValue);
}
} finally {
p.destroy();
}
}
/**
* Redirect the input stream to a logger (as debug logs)
*/

View File

@ -30,7 +30,6 @@ import org.apache.activemq.artemis.components.ExternalComponent;
import org.apache.activemq.artemis.dto.AppDTO;
import org.apache.activemq.artemis.dto.ComponentDTO;
import org.apache.activemq.artemis.dto.WebServerDTO;
import org.apache.activemq.artemis.utils.FileUtil;
import org.eclipse.jetty.server.ConnectionFactory;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
@ -60,12 +59,11 @@ public class WebServerComponent implements ExternalComponent {
private String consoleUrl;
private List<WebAppContext> webContexts;
private ServerConnector connector;
private String artemisHome;
private Path artemisHomePath;
@Override
public void configure(ComponentDTO config, String artemisInstance, String artemisHome) throws Exception {
webServerConfig = (WebServerDTO) config;
this.artemisHome = artemisHome;
uri = new URI(webServerConfig.bind);
server = new Server();
String scheme = uri.getScheme();
@ -104,7 +102,8 @@ public class WebServerComponent implements ExternalComponent {
handlers = new HandlerList();
Path homeWarDir = Paths.get(artemisHome != null ? artemisHome : ".").resolve(webServerConfig.path).toAbsolutePath();
this.artemisHomePath = Paths.get(artemisHome != null ? artemisHome : ".");
Path homeWarDir = artemisHomePath.resolve(webServerConfig.path).toAbsolutePath();
Path instanceWarDir = Paths.get(artemisInstance != null ? artemisInstance : ".").resolve(webServerConfig.path).toAbsolutePath();
if (webServerConfig.apps != null && webServerConfig.apps.size() > 0) {
@ -236,44 +235,33 @@ public class WebServerComponent implements ExternalComponent {
}
public void internalStop() throws Exception {
System.out.println("Stopping");
server.stop();
if (webContexts != null) {
File tmpdir = null;
StringBuilder strBuilder = new StringBuilder();
boolean found = false;
for (WebAppContext context : webContexts) {
tmpdir = context.getTempDirectory();
if (tmpdir != null && tmpdir.exists() && !context.isPersistTempDirectory()) {
//tmpdir will be removed by deleteOnExit()
//However because the URLClassLoader never release/close its opened
//jars the jar file won't be able to get deleted on Windows platform
//until after the process fully terminated. To fix this here arranges
//a separate process to try clean up the temp dir
FileUtil.deleteDirectory(tmpdir);
if (tmpdir.exists()) {
ActiveMQWebLogger.LOGGER.tmpFileNotDeleted(tmpdir);
strBuilder.append(tmpdir);
strBuilder.append(",");
found = true;
}
}
}
if (found) {
String bootJar = artemisHome + File.separator + "lib" + File.separator + "artemis-boot.jar";
String[] command = {"java", "-cp", bootJar, "org.apache.activemq.artemis.boot.WebTmpCleaner", strBuilder.toString()};
ProcessBuilder pb = new ProcessBuilder(command);
pb.start();
}
cleanupWebTemporaryFiles(webContexts);
webContexts.clear();
}
}
private File getLibFolder() {
Path lib = artemisHomePath.resolve("lib");
File libFolder = new File(lib.toUri());
return libFolder;
}
public void cleanupWebTemporaryFiles(List<WebAppContext> webContexts) throws Exception {
List<File> temporaryFiles = new ArrayList<>();
for (WebAppContext context : webContexts) {
File tmpdir = context.getTempDirectory();
temporaryFiles.add(tmpdir);
}
if (!temporaryFiles.isEmpty()) {
WebTmpCleaner.cleanupTmpFiles(getLibFolder(), temporaryFiles);
}
}
@Override
public boolean isStarted() {
return server != null && server.isStarted();

View File

@ -14,9 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.artemis.boot;
package org.apache.activemq.artemis.component;
import java.io.File;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
/**
* This class is used to remove the jar files
@ -26,10 +31,7 @@ import java.io.File;
*/
public class WebTmpCleaner {
public static void main(String[] args) throws Exception {
String[] filesToClean = args[0].split(",");
public static void main(String[] filesToClean) throws Exception {
//It needs to retry a bit as we are not sure
//when the main VM exists.
boolean allCleaned = false;
@ -38,9 +40,10 @@ public class WebTmpCleaner {
allCleaned = true;
for (String f : filesToClean) {
if (!f.trim().isEmpty()) {
File file = new File(f);
URI url = new URI(f);
File file = new File(url);
if (file.exists()) {
deleteFile(file);
deleteFolder(file);
allCleaned = false;
}
}
@ -49,13 +52,22 @@ public class WebTmpCleaner {
}
}
public static Process cleanupTmpFiles(File libFolder, List<File> temporaryFiles) throws Exception {
ArrayList<String> files = new ArrayList<>(temporaryFiles.size());
for (File f : temporaryFiles) {
files.add(f.toURI().toString());
}
public static final void deleteFile(final File file) {
String classPath = SpawnedVMSupport.getClassPath(libFolder);
return SpawnedVMSupport.spawnVM(classPath, WebTmpCleaner.class.getName(), false, (String[]) files.toArray(new String[files.size()]));
}
public static final void deleteFolder(final File file) {
if (file.isDirectory()) {
String[] files = file.list();
for (String path : files) {
File f = new File(file, path);
deleteFile(f);
deleteFolder(f);
}
}
file.delete();

View File

@ -17,6 +17,10 @@
package org.apache.activemq.cli.test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@ -25,6 +29,7 @@ import org.apache.activemq.artemis.api.core.client.ActiveMQClient;
import org.apache.activemq.artemis.cli.Artemis;
import org.apache.activemq.artemis.cli.commands.Run;
import org.apache.activemq.artemis.cli.commands.tools.LockAbstract;
import org.apache.activemq.artemis.component.WebTmpCleaner;
import org.apache.activemq.artemis.core.server.ActiveMQComponent;
import org.apache.activemq.artemis.core.server.ActiveMQServer;
import org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl;
@ -34,6 +39,7 @@ import org.apache.activemq.artemis.junit.Wait;
import org.apache.activemq.artemis.spi.core.security.jaas.PropertiesLoader;
import org.apache.activemq.artemis.utils.ThreadLeakCheckRule;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@ -104,6 +110,34 @@ public class WebServerCLITest {
stopServer();
}
@Test
public void testCleanupFolder() throws Exception {
System.out.println("temporary folder = " + temporaryFolder.getRoot());
List<File> fileList = new ArrayList<>();
for (int i = 0; i < 10; i++) {
File directory = temporaryFolder.newFolder("test & output " + i);
fillup(directory);
}
Process process = WebTmpCleaner.cleanupTmpFiles(null, fileList);
Assert.assertEquals(0, process.waitFor());
for (File f : fileList) {
Assert.assertFalse(f.exists());
}
}
private void fillup(File file) throws Exception {
System.out.println("Creating file " + file);
file.mkdirs();
for (int i = 0; i < 10; i++) {
File fi = new File(file, "file" + i + ".txt");
PrintStream str = new PrintStream(new FileOutputStream(fi));
str.println("hello");
str.close();
}
}
private void stopServer() throws Exception {
Artemis.internalExecute("stop");
assertTrue(Run.latchRunning.await(5, TimeUnit.SECONDS));

View File

@ -23,7 +23,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.activemq.artemis.core.server.ActiveMQServer;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.apache.activemq.transport.amqp.client.AmqpClient;
import org.apache.activemq.transport.amqp.client.AmqpConnection;
import org.apache.activemq.transport.amqp.client.AmqpMessage;

View File

@ -42,7 +42,7 @@ import org.apache.activemq.artemis.core.server.ActiveMQServers;
import org.apache.activemq.artemis.core.version.impl.VersionImpl;
import org.apache.activemq.artemis.tests.integration.IntegrationTestLogger;
import org.apache.activemq.artemis.tests.util.SpawnedTestBase;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.VersionLoader;
import org.junit.After;
import org.junit.Assert;

View File

@ -35,7 +35,7 @@ import org.apache.activemq.artemis.core.journal.RecordInfo;
import org.apache.activemq.artemis.core.journal.impl.JournalImpl;
import org.apache.activemq.artemis.core.server.ActiveMQServer;
import org.apache.activemq.artemis.tests.util.SpawnedTestBase;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.junit.Assert;
import org.junit.Test;

View File

@ -79,7 +79,7 @@ import org.apache.activemq.artemis.spi.core.security.ActiveMQJAASSecurityManager
import org.apache.activemq.artemis.spi.core.security.ActiveMQSecurityManager;
import org.apache.activemq.artemis.spi.core.security.jaas.InVMLoginModule;
import org.apache.activemq.artemis.tests.util.SpawnedTestBase;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.apache.activemq.artemis.tests.util.Wait;
import org.junit.After;
import org.junit.Assert;

View File

@ -30,7 +30,7 @@ import org.apache.activemq.artemis.core.settings.impl.AddressSettings;
import org.apache.activemq.artemis.jms.client.ActiveMQTextMessage;
import org.apache.activemq.artemis.tests.integration.IntegrationTestLogger;
import org.apache.activemq.artemis.tests.util.SpawnedVMCheck;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;

View File

@ -23,7 +23,7 @@ import org.apache.activemq.artemis.api.core.client.ClientSession;
import org.apache.activemq.artemis.api.core.client.ClientSessionFactory;
import org.apache.activemq.artemis.api.core.client.ServerLocator;
import org.apache.activemq.artemis.tests.integration.IntegrationTestLogger;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.RandomUtil;
import org.junit.Before;
import org.junit.Test;

View File

@ -30,7 +30,7 @@ import org.apache.activemq.artemis.api.core.RoutingType;
import org.apache.activemq.artemis.core.server.impl.AddressInfo;
import org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory;
import org.apache.activemq.artemis.logs.AssertionLoggerHandler;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.apache.qpid.jms.JmsConnectionFactory;
import org.junit.After;
import org.junit.Assert;

View File

@ -23,7 +23,7 @@ import java.util.List;
import org.apache.activemq.artemis.core.server.NodeManager;
import org.apache.activemq.artemis.core.server.impl.FileLockNodeManager;
import org.apache.activemq.artemis.tests.util.ActiveMQTestBase;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.UUID;
import org.junit.Assert;
import org.junit.Test;

View File

@ -48,7 +48,7 @@ import org.apache.activemq.artemis.junit.Wait;
import org.apache.activemq.artemis.spi.core.security.ActiveMQJAASSecurityManager;
import org.apache.activemq.artemis.spi.core.security.jaas.InVMLoginModule;
import org.apache.activemq.artemis.tests.util.ActiveMQTestBase;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.actors.ArtemisExecutor;
import org.junit.Assert;
import org.junit.Before;

View File

@ -36,7 +36,7 @@ import org.apache.activemq.artemis.spi.core.security.ActiveMQJAASSecurityManager
import org.apache.activemq.artemis.spi.core.security.ActiveMQSecurityManager;
import org.apache.activemq.artemis.spi.core.security.jaas.InVMLoginModule;
import org.apache.activemq.artemis.tests.util.SpawnedTestBase;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.junit.Assert;
import org.junit.Test;

View File

@ -33,7 +33,7 @@ import org.apache.activemq.artemis.core.journal.impl.JournalCompactor;
import org.apache.activemq.artemis.core.journal.impl.JournalFile;
import org.apache.activemq.artemis.core.journal.impl.JournalImpl;
import org.apache.activemq.artemis.tests.util.SpawnedTestBase;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

View File

@ -32,7 +32,7 @@ import org.apache.activemq.artemis.core.journal.RecordInfo;
import org.apache.activemq.artemis.core.journal.impl.JournalImpl;
import org.apache.activemq.artemis.jlibaio.LibaioContext;
import org.apache.activemq.artemis.tests.util.SpawnedTestBase;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.junit.Assert;
import org.junit.Test;

View File

@ -17,7 +17,7 @@
package org.apache.activemq.artemis.tests.integration.paging;
import org.apache.activemq.artemis.core.server.ActiveMQServer;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
/**
* This is a sub process of the test {@link PageCountSyncOnNonTXTest}

View File

@ -16,7 +16,7 @@
*/
package org.apache.activemq.artemis.tests.integration.paging;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
/**
* There is no difference between this class and {@link PagingWithFailoverServer}

View File

@ -17,7 +17,7 @@
package org.apache.activemq.artemis.tests.integration.paging;
import org.apache.activemq.artemis.core.server.ActiveMQServer;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
public class PagingWithFailoverServer extends SpawnedServerSupport {

View File

@ -25,7 +25,7 @@ import org.apache.activemq.artemis.core.config.impl.ConfigurationImpl;
import org.apache.activemq.artemis.core.server.ActiveMQServer;
import org.apache.activemq.artemis.tests.integration.IntegrationTestLogger;
import org.apache.activemq.artemis.tests.util.SpawnedTestBase;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

View File

@ -35,7 +35,7 @@ import org.apache.activemq.artemis.api.core.management.ManagementHelper;
import org.apache.activemq.artemis.api.core.management.ResourceNames;
import org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnectorFactory;
import org.apache.activemq.artemis.api.core.RoutingType;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.objectweb.jtests.jms.admin.Admin;
/**

View File

@ -20,8 +20,8 @@ package org.apache.activemq.artemis.tests.unit.core.util;
import java.util.HashSet;
import org.apache.activemq.artemis.tests.util.SpawnedVMCheck;
import org.apache.activemq.artemis.tests.util.SpawnedVMSupport;
import org.apache.activemq.artemis.utils.RandomUtil;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;

View File

@ -17,6 +17,7 @@
package org.apache.activemq.artemis.tests.util;
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
import org.junit.rules.ExternalResource;
public class SpawnedVMCheck extends ExternalResource {