From d0a2186e9c66be1799a60b6dfc33fc3b1bff591d Mon Sep 17 00:00:00 2001 From: Howard Gao Date: Thu, 23 Jan 2020 00:00:01 +0800 Subject: [PATCH] ARTEMIS-2596 (kill -9) AMQ causes tmp web dir space usage to increase If you kill the server without invoking a normal shutdown, tmp web files are not cleaned out. This leaves old webapp folders lingering until a normal shutdown. In a failover test environment that repeatedly kills the server, this causes disk space usage issues. The fix is to add a cleanup method before the web server starts. It searches the tmp web dir if there is any leftover files/dirs and delete them if any. --- .../artemis/component/WebServerComponent.java | 35 +++ .../artemis/component/WebTmpCleaner.java | 24 ++- .../cli/test/WebServerComponentTest.java | 201 +++++++++++++++++- artemis-web/src/test/webapp/WEB-INF/web.xml | 26 +++ artemis-web/src/test/webapp/index.html | 27 +++ 5 files changed, 303 insertions(+), 10 deletions(-) create mode 100644 artemis-web/src/test/webapp/WEB-INF/web.xml create mode 100644 artemis-web/src/test/webapp/index.html diff --git a/artemis-web/src/main/java/org/apache/activemq/artemis/component/WebServerComponent.java b/artemis-web/src/main/java/org/apache/activemq/artemis/component/WebServerComponent.java index 3d52393538..701ebd3c59 100644 --- a/artemis-web/src/main/java/org/apache/activemq/artemis/component/WebServerComponent.java +++ b/artemis-web/src/main/java/org/apache/activemq/artemis/component/WebServerComponent.java @@ -48,6 +48,7 @@ import org.eclipse.jetty.server.handler.ResourceHandler; import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.webapp.WebAppContext; +import org.eclipse.jetty.webapp.WebInfConfiguration; import org.jboss.logging.Logger; import javax.servlet.DispatcherType; @@ -252,6 +253,7 @@ public class WebServerComponent implements ExternalComponent { if (isStarted()) { return; } + cleanupTmp(); server.start(); ActiveMQWebLogger.LOGGER.webserverStarted(webServerConfig.bind); @@ -276,6 +278,35 @@ public class WebServerComponent implements ExternalComponent { return libFolder; } + private void cleanupTmp() { + if (webContexts == null || webContexts.size() == 0) { + //there is no webapp to be deployed (as in some tests) + return; + } + List temporaryFiles = new ArrayList<>(); + + for (WebAppContext context : webContexts) { + WebInfConfiguration config = new WebInfConfiguration(); + try { + config.resolveTempDirectory(context); + File webTmpBase = context.getTempDirectory().getParentFile(); + if (webTmpBase.exists()) { + webTmpBase.listFiles((f) -> { + temporaryFiles.add(f); + return false; + }); + } + if (temporaryFiles.size() > 0) { + WebTmpCleaner.cleanupTmpFiles(getLibFolder(), temporaryFiles, true); + } + //all web contexts share a same base dir. So we only do it once. + break; + } catch (Exception e) { + logger.warn("Failed to get base dir for tmp web files", e); + } + } + } + public void cleanupWebTemporaryFiles(List webContexts) throws Exception { List temporaryFiles = new ArrayList<>(); for (WebAppContext context : webContexts) { @@ -327,4 +358,8 @@ public class WebServerComponent implements ExternalComponent { internalStop(); } } + + public List getWebContexts() { + return this.webContexts; + } } diff --git a/artemis-web/src/main/java/org/apache/activemq/artemis/component/WebTmpCleaner.java b/artemis-web/src/main/java/org/apache/activemq/artemis/component/WebTmpCleaner.java index 478a0120b2..ee2b3439b5 100644 --- a/artemis-web/src/main/java/org/apache/activemq/artemis/component/WebTmpCleaner.java +++ b/artemis-web/src/main/java/org/apache/activemq/artemis/component/WebTmpCleaner.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.activemq.artemis.utils.SpawnedVMSupport; +import org.jboss.logging.Logger; /** * This class is used to remove the jar files @@ -31,11 +32,16 @@ import org.apache.activemq.artemis.utils.SpawnedVMSupport; */ public class WebTmpCleaner { + private static final Logger logger = Logger.getLogger(WebTmpCleaner.class); + public static void main(String[] filesToClean) throws Exception { //It needs to retry a bit as we are not sure //when the main VM exists. + cleanupFilesWithRetry(filesToClean, 100); + } + + private static boolean cleanupFilesWithRetry(String[] filesToClean, int maxRetries) throws Exception { boolean allCleaned = false; - int maxRetries = 100; while (!allCleaned && maxRetries-- > 0) { allCleaned = true; for (String f : filesToClean) { @@ -50,16 +56,28 @@ public class WebTmpCleaner { } Thread.sleep(200); } + if (!allCleaned) { + logger.warn("Some files in web temp dir are not cleaned up after " + maxRetries + " retries."); + } + return allCleaned; } public static Process cleanupTmpFiles(File libFolder, List temporaryFiles) throws Exception { + return cleanupTmpFiles(libFolder, temporaryFiles, false); + } + + public static Process cleanupTmpFiles(File libFolder, List temporaryFiles, boolean invm) throws Exception { ArrayList files = new ArrayList<>(temporaryFiles.size()); for (File f : temporaryFiles) { files.add(f.toURI().toString()); } - String classPath = SpawnedVMSupport.getClassPath(libFolder); - return SpawnedVMSupport.spawnVM(classPath, WebTmpCleaner.class.getName(), false, (String[]) files.toArray(new String[files.size()])); + if (!invm) { + String classPath = SpawnedVMSupport.getClassPath(libFolder); + return SpawnedVMSupport.spawnVM(classPath, WebTmpCleaner.class.getName(), false, (String[]) files.toArray(new String[files.size()])); + } + cleanupFilesWithRetry(files.toArray(new String[files.size()]), 2); + return null; } public static final void deleteFolder(final File file) { diff --git a/artemis-web/src/test/java/org/apache/activemq/cli/test/WebServerComponentTest.java b/artemis-web/src/test/java/org/apache/activemq/cli/test/WebServerComponentTest.java index 75b68ad113..e73d2c2672 100644 --- a/artemis-web/src/test/java/org/apache/activemq/cli/test/WebServerComponentTest.java +++ b/artemis-web/src/test/java/org/apache/activemq/cli/test/WebServerComponentTest.java @@ -18,14 +18,23 @@ package org.apache.activemq.cli.test; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; +import java.io.BufferedInputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; import java.util.regex.Pattern; import io.netty.bootstrap.Bootstrap; @@ -45,22 +54,31 @@ import io.netty.handler.codec.http.HttpObject; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.ssl.SslHandler; import io.netty.util.CharsetUtil; import org.apache.activemq.artemis.cli.factory.xml.XmlBrokerFactoryHandler; import org.apache.activemq.artemis.component.WebServerComponent; import org.apache.activemq.artemis.core.remoting.impl.ssl.SSLSupport; import org.apache.activemq.artemis.core.server.ActiveMQComponent; +import org.apache.activemq.artemis.dto.AppDTO; import org.apache.activemq.artemis.dto.BrokerDTO; import org.apache.activemq.artemis.dto.WebServerDTO; +import org.apache.activemq.artemis.utils.ThreadLeakCheckRule; import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.eclipse.jetty.webapp.WebAppContext; +import org.eclipse.jetty.webapp.WebInfConfiguration; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; public class WebServerComponentTest extends Assert { + @Rule + public ThreadLeakCheckRule leakCheckRule = new ThreadLeakCheckRule(); + static final String URL = System.getProperty("url", "http://localhost:8161/WebServerComponentTest.txt"); static final String SECURE_URL = System.getProperty("url", "https://localhost:8448/WebServerComponentTest.txt"); private Bootstrap bootstrap; @@ -69,6 +87,7 @@ public class WebServerComponentTest extends Assert { @Before public void setupNetty() throws URISyntaxException { + System.setProperty("jetty.base", "./target"); // Configure the client. group = new NioEventLoopGroup(); bootstrap = new Bootstrap(); @@ -77,9 +96,11 @@ public class WebServerComponentTest extends Assert { @After public void tearDown() throws Exception { + System.clearProperty("jetty.base"); for (ActiveMQComponent c : testedComponents) { c.stop(); } + testedComponents.clear(); } @Test @@ -125,13 +146,15 @@ public class WebServerComponentTest extends Assert { // Send the HTTP request. ch.writeAndFlush(request); assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertEquals("12345", clientHandler.body.toString()); if (useCustomizer) { assertEquals(1, TestCustomizer.count); } - assertEquals(clientHandler.body, "12345"); + assertEquals(clientHandler.body.toString(), "12345"); assertNull(clientHandler.serverHeader); // Wait for the server to close the connection. ch.close(); + ch.eventLoop().shutdownNow(); Assert.assertTrue(webServerComponent.isStarted()); webServerComponent.stop(true); Assert.assertFalse(webServerComponent.isStarted()); @@ -167,9 +190,10 @@ public class WebServerComponentTest extends Assert { // Send the HTTP request. ch.writeAndFlush(request); assertTrue(latch.await(5, TimeUnit.SECONDS)); - assertEquals(clientHandler.body, "12345"); + assertEquals("12345", clientHandler.body.toString()); // Wait for the server to close the connection. ch.close(); + ch.eventLoop().shutdownNow(); Assert.assertTrue(webServerComponent.isStarted()); //usual stop won't actually stop it @@ -224,6 +248,7 @@ public class WebServerComponentTest extends Assert { CountDownLatch latch = new CountDownLatch(1); final ClientHandler clientHandler = new ClientHandler(latch); + bootstrap.group(group).channel(NioSocketChannel.class).handler(new ChannelInitializer() { @Override protected void initChannel(Channel ch) throws Exception { @@ -242,10 +267,11 @@ public class WebServerComponentTest extends Assert { // Send the HTTP request. ch.writeAndFlush(request); assertTrue(latch.await(5, TimeUnit.SECONDS)); - assertEquals(clientHandler.body, "12345"); + assertEquals("12345", clientHandler.body.toString()); assertNull(clientHandler.serverHeader); // Wait for the server to close the connection. ch.close(); + ch.eventLoop().shutdownNow(); Assert.assertTrue(webServerComponent.isStarted()); webServerComponent.stop(true); Assert.assertFalse(webServerComponent.isStarted()); @@ -316,9 +342,10 @@ public class WebServerComponentTest extends Assert { // Send the HTTP request. ch.writeAndFlush(request); assertTrue(latch.await(5, TimeUnit.SECONDS)); - assertEquals(clientHandler.body, "12345"); + assertEquals("12345", clientHandler.body.toString()); // Wait for the server to close the connection. ch.close(); + ch.eventLoop().shutdownNow(); Assert.assertTrue(webServerComponent.isStarted()); webServerComponent.stop(true); Assert.assertFalse(webServerComponent.isStarted()); @@ -362,10 +389,168 @@ public class WebServerComponentTest extends Assert { assertEquals(trustPassword, broker.web.getTrustStorePassword()); } + @Test + public void testServerCleanupBeforeStart() throws Exception { + final String warName = "simple-app.war"; + createTestWar(warName); + WebServerDTO webServerDTO = new WebServerDTO(); + webServerDTO.bind = "http://localhost:0"; + webServerDTO.path = ""; + webServerDTO.apps = new ArrayList<>(); + AppDTO app = new AppDTO(); + app.url = "simple-app/"; + app.war = warName; + webServerDTO.apps.add(app); + WebServerComponent webServerComponent = new WebServerComponent(); + Assert.assertFalse(webServerComponent.isStarted()); + testedComponents.add(webServerComponent); + webServerComponent.configure(webServerDTO, "./target", "./target"); + //create some garbage + List contexts = webServerComponent.getWebContexts(); + + File targetDir = new File("./target"); + File workDir = new File(targetDir, "web-work"); + workDir.mkdir(); + + WebInfConfiguration cfg = new WebInfConfiguration(); + assertEquals(1, contexts.size()); + WebAppContext ctxt = contexts.get(0); + List garbage = new ArrayList<>(); + + ctxt.setAttribute("javax.servlet.context.tempdir", new File(workDir, "jetty-context0")); + + cfg.resolveTempDirectory(ctxt); + File tmpdir = ctxt.getTempDirectory(); + File testDir = tmpdir.getParentFile(); + createGarbagesInDir(testDir, garbage); + + assertTrue(garbage.size() > 0); + + for (File file : garbage) { + assertTrue(file.exists()); + } + + webServerComponent.start(); + + //make sure those garbage are gone + for (File file : garbage) { + assertFalse("file exist: " + file.getAbsolutePath(), file.exists()); + } + + //check the war is working + final int port = webServerComponent.getPort(); + + // Make the connection attempt. + CountDownLatch latch = new CountDownLatch(1); + final ClientHandler clientHandler = new ClientHandler(latch); + bootstrap.group(group).channel(NioSocketChannel.class).handler(new ChannelInitializer() { + @Override + protected void initChannel(Channel ch) throws Exception { + ch.pipeline().addLast(new HttpClientCodec(8192, 8192, 8192, false)); + ch.pipeline().addLast(clientHandler); + } + }); + + Channel ch = bootstrap.connect("localhost", port).sync().channel(); + + String warUrl = "http://localhost:" + port + "/" + app.url; + + URI uri = new URI(warUrl); + + // Prepare the HTTP request. + HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri.getRawPath()); + request.headers().set(HttpHeaderNames.HOST, "localhost"); + + // Send the HTTP request. + ch.writeAndFlush(request); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + + assertTrue("content: " + clientHandler.body.toString(), clientHandler.body.toString().contains("Hello Artemis Test")); + assertNull(clientHandler.serverHeader); + // Wait for the server to close the connection. + ch.close(); + ch.eventLoop().shutdownNow(); + Assert.assertTrue(webServerComponent.isStarted()); + webServerComponent.stop(true); + Assert.assertFalse(webServerComponent.isStarted()); + } + + private void createTestWar(String warName) throws Exception { + File warFile = new File("target", warName); + File srcFile = new File("src/test/webapp"); + createJarFile(srcFile, warFile); + } + + private void createJarFile(File srcFile, File jarFile) throws IOException { + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + try (JarOutputStream target = new JarOutputStream(new FileOutputStream(jarFile), manifest)) { + addFile(srcFile, target, "src/test/webapp"); + } + } + + private void addFile(File source, JarOutputStream target, String nameBase) throws IOException { + if (source.isDirectory()) { + String name = source.getPath().replace("\\", "/"); + if (!name.isEmpty()) { + name = name.substring(nameBase.length()); + if (!name.endsWith("/")) { + name += "/"; + } + JarEntry entry = new JarEntry(name); + entry.setTime(source.lastModified()); + target.putNextEntry(entry); + target.closeEntry(); + } + for (File nestedFile: source.listFiles()) { + addFile(nestedFile, target, nameBase); + } + return; + } + + String name = source.getPath().replace("\\", "/"); + name = name.substring(nameBase.length()); + JarEntry entry = new JarEntry(name); + entry.setTime(source.lastModified()); + target.putNextEntry(entry); + try (BufferedInputStream input = new BufferedInputStream(new FileInputStream(source))) { + byte[] buffer = new byte[1024]; + while (true) { + int count = input.read(buffer); + if (count == -1) + break; + target.write(buffer, 0, count); + } + target.closeEntry(); + } + } + + private void createGarbagesInDir(File tempDirectory, List garbage) throws IOException { + if (!tempDirectory.exists()) { + tempDirectory.mkdirs(); + } + createRandomJettyFiles(tempDirectory, 10, garbage); + } + + private void createRandomJettyFiles(File dir, int num, List collector) throws IOException { + for (int i = 0; i < num; i++) { + String randomName = "jetty-" + UUID.randomUUID().toString(); + File file = new File(dir, randomName); + if (i % 2 == 0) { + //create a dir + file.mkdir(); + } else { + //normal file + file.createNewFile(); + } + collector.add(file); + } + } + class ClientHandler extends SimpleChannelInboundHandler { private CountDownLatch latch; - private String body; + private StringBuilder body = new StringBuilder(); private String serverHeader; ClientHandler(CountDownLatch latch) { @@ -379,8 +564,10 @@ public class WebServerComponentTest extends Assert { serverHeader = response.headers().get("Server"); } else if (msg instanceof HttpContent) { HttpContent content = (HttpContent) msg; - body = content.content().toString(CharsetUtil.UTF_8); - latch.countDown(); + body.append(content.content().toString(CharsetUtil.UTF_8)); + if (msg instanceof LastHttpContent) { + latch.countDown(); + } } } diff --git a/artemis-web/src/test/webapp/WEB-INF/web.xml b/artemis-web/src/test/webapp/WEB-INF/web.xml new file mode 100644 index 0000000000..dc1d818dd0 --- /dev/null +++ b/artemis-web/src/test/webapp/WEB-INF/web.xml @@ -0,0 +1,26 @@ + + + + Simple War Application + + diff --git a/artemis-web/src/test/webapp/index.html b/artemis-web/src/test/webapp/index.html new file mode 100644 index 0000000000..5d69fb894d --- /dev/null +++ b/artemis-web/src/test/webapp/index.html @@ -0,0 +1,27 @@ + + + + + + simple web + + +

Hello Artemis Test

+ +