ARTEMIS-2596 (kill -9) AMQ causes tmp web dir space usage to increase

If you kill the server without invoking a normal shutdown, tmp
web files are not cleaned out.  This leaves old webapp folders
lingering until a normal shutdown.

In a failover test environment that repeatedly kills the server,
this causes disk space usage issues.

The fix is to add a cleanup method before the web server starts.
It searches the tmp web dir if there is any leftover files/dirs
and delete them if any.

artemis-web/src/main/java/org/apache/activemq/artemis/component/WebServerComponent.java

@@ -48,6 +48,7 @@ import org.eclipse.jetty.server.handler.ResourceHandler;
 import org.eclipse.jetty.servlet.FilterHolder;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.webapp.WebAppContext;
+import org.eclipse.jetty.webapp.WebInfConfiguration;
 import org.jboss.logging.Logger;
 
 import javax.servlet.DispatcherType;
@@ -252,6 +253,7 @@ public class WebServerComponent implements ExternalComponent {
       if (isStarted()) {
          return;
       }
+      cleanupTmp();
       server.start();
       ActiveMQWebLogger.LOGGER.webserverStarted(webServerConfig.bind);
 
@@ -276,6 +278,35 @@ public class WebServerComponent implements ExternalComponent {
       return libFolder;
    }
 
+   private void cleanupTmp() {
+      if (webContexts == null || webContexts.size() == 0) {
+         //there is no webapp to be deployed (as in some tests)
+         return;
+      }
+      List<File> temporaryFiles = new ArrayList<>();
+
+      for (WebAppContext context : webContexts) {
+         WebInfConfiguration config = new WebInfConfiguration();
+         try {
+            config.resolveTempDirectory(context);
+            File webTmpBase = context.getTempDirectory().getParentFile();
+            if (webTmpBase.exists()) {
+               webTmpBase.listFiles((f) -> {
+                  temporaryFiles.add(f);
+                  return false;
+               });
+            }
+            if (temporaryFiles.size() > 0) {
+               WebTmpCleaner.cleanupTmpFiles(getLibFolder(), temporaryFiles, true);
+            }
+            //all web contexts share a same base dir. So we only do it once.
+            break;
+         } catch (Exception e) {
+            logger.warn("Failed to get base dir for tmp web files", e);
+         }
+      }
+   }
+
    public void cleanupWebTemporaryFiles(List<WebAppContext> webContexts) throws Exception {
       List<File> temporaryFiles = new ArrayList<>();
       for (WebAppContext context : webContexts) {
@@ -327,4 +358,8 @@ public class WebServerComponent implements ExternalComponent {
          internalStop();
       }
    }
+
+   public List<WebAppContext> getWebContexts() {
+      return this.webContexts;
+   }
 }

artemis-web/src/main/java/org/apache/activemq/artemis/component/WebTmpCleaner.java

@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.activemq.artemis.utils.SpawnedVMSupport;
+import org.jboss.logging.Logger;
 
 /**
  * This class is used to remove the jar files
@@ -31,11 +32,16 @@ import org.apache.activemq.artemis.utils.SpawnedVMSupport;
  */
 public class WebTmpCleaner {
 
+   private static final Logger logger = Logger.getLogger(WebTmpCleaner.class);
+
    public static void main(String[] filesToClean) throws Exception {
       //It needs to retry a bit as we are not sure
       //when the main VM exists.
+      cleanupFilesWithRetry(filesToClean, 100);
+   }
+
+   private static boolean cleanupFilesWithRetry(String[] filesToClean, int maxRetries) throws Exception {
       boolean allCleaned = false;
-      int maxRetries = 100;
       while (!allCleaned && maxRetries-- > 0) {
          allCleaned = true;
          for (String f : filesToClean) {
@@ -50,16 +56,28 @@ public class WebTmpCleaner {
          }
          Thread.sleep(200);
       }
+      if (!allCleaned) {
+         logger.warn("Some files in web temp dir are not cleaned up after " + maxRetries + " retries.");
+      }
+      return allCleaned;
    }
 
    public static Process cleanupTmpFiles(File libFolder, List<File> temporaryFiles) throws Exception {
+      return cleanupTmpFiles(libFolder, temporaryFiles, false);
+   }
+
+   public static Process cleanupTmpFiles(File libFolder, List<File> temporaryFiles, boolean invm) throws Exception {
       ArrayList<String> files = new ArrayList<>(temporaryFiles.size());
       for (File f : temporaryFiles) {
          files.add(f.toURI().toString());
       }
 
-      String classPath = SpawnedVMSupport.getClassPath(libFolder);
+      if (!invm) {
-      return SpawnedVMSupport.spawnVM(classPath, WebTmpCleaner.class.getName(), false, (String[]) files.toArray(new String[files.size()]));
+         String classPath = SpawnedVMSupport.getClassPath(libFolder);
+         return SpawnedVMSupport.spawnVM(classPath, WebTmpCleaner.class.getName(), false, (String[]) files.toArray(new String[files.size()]));
+      }
+      cleanupFilesWithRetry(files.toArray(new String[files.size()]), 2);
+      return null;
    }
 
    public static final void deleteFolder(final File file) {

artemis-web/src/test/java/org/apache/activemq/cli/test/WebServerComponentTest.java

@@ -18,14 +18,23 @@ package org.apache.activemq.cli.test;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
+import java.io.BufferedInputStream;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.regex.Pattern;
 
 import io.netty.bootstrap.Bootstrap;
@@ -45,22 +54,31 @@ import io.netty.handler.codec.http.HttpObject;
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.HttpVersion;
+import io.netty.handler.codec.http.LastHttpContent;
 import io.netty.handler.ssl.SslHandler;
 import io.netty.util.CharsetUtil;
 import org.apache.activemq.artemis.cli.factory.xml.XmlBrokerFactoryHandler;
 import org.apache.activemq.artemis.component.WebServerComponent;
 import org.apache.activemq.artemis.core.remoting.impl.ssl.SSLSupport;
 import org.apache.activemq.artemis.core.server.ActiveMQComponent;
+import org.apache.activemq.artemis.dto.AppDTO;
 import org.apache.activemq.artemis.dto.BrokerDTO;
 import org.apache.activemq.artemis.dto.WebServerDTO;
+import org.apache.activemq.artemis.utils.ThreadLeakCheckRule;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.eclipse.jetty.webapp.WebInfConfiguration;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 
 public class WebServerComponentTest extends Assert {
 
+   @Rule
+   public ThreadLeakCheckRule leakCheckRule = new ThreadLeakCheckRule();
+
    static final String URL = System.getProperty("url", "http://localhost:8161/WebServerComponentTest.txt");
    static final String SECURE_URL = System.getProperty("url", "https://localhost:8448/WebServerComponentTest.txt");
    private Bootstrap bootstrap;
@@ -69,6 +87,7 @@ public class WebServerComponentTest extends Assert {
 
    @Before
    public void setupNetty() throws URISyntaxException {
+      System.setProperty("jetty.base", "./target");
       // Configure the client.
       group = new NioEventLoopGroup();
       bootstrap = new Bootstrap();
@@ -77,9 +96,11 @@ public class WebServerComponentTest extends Assert {
 
    @After
    public void tearDown() throws Exception {
+      System.clearProperty("jetty.base");
       for (ActiveMQComponent c : testedComponents) {
          c.stop();
       }
+      testedComponents.clear();
    }
 
    @Test
@@ -125,13 +146,15 @@ public class WebServerComponentTest extends Assert {
       // Send the HTTP request.
       ch.writeAndFlush(request);
       assertTrue(latch.await(5, TimeUnit.SECONDS));
+      assertEquals("12345", clientHandler.body.toString());
       if (useCustomizer) {
          assertEquals(1, TestCustomizer.count);
       }
-      assertEquals(clientHandler.body, "12345");
+      assertEquals(clientHandler.body.toString(), "12345");
       assertNull(clientHandler.serverHeader);
       // Wait for the server to close the connection.
       ch.close();
+      ch.eventLoop().shutdownNow();
       Assert.assertTrue(webServerComponent.isStarted());
       webServerComponent.stop(true);
       Assert.assertFalse(webServerComponent.isStarted());
@@ -167,9 +190,10 @@ public class WebServerComponentTest extends Assert {
       // Send the HTTP request.
       ch.writeAndFlush(request);
       assertTrue(latch.await(5, TimeUnit.SECONDS));
-      assertEquals(clientHandler.body, "12345");
+      assertEquals("12345", clientHandler.body.toString());
       // Wait for the server to close the connection.
       ch.close();
+      ch.eventLoop().shutdownNow();
       Assert.assertTrue(webServerComponent.isStarted());
 
       //usual stop won't actually stop it
@@ -224,6 +248,7 @@ public class WebServerComponentTest extends Assert {
 
       CountDownLatch latch = new CountDownLatch(1);
       final ClientHandler clientHandler = new ClientHandler(latch);
+
       bootstrap.group(group).channel(NioSocketChannel.class).handler(new ChannelInitializer() {
          @Override
          protected void initChannel(Channel ch) throws Exception {
@@ -242,10 +267,11 @@ public class WebServerComponentTest extends Assert {
       // Send the HTTP request.
       ch.writeAndFlush(request);
       assertTrue(latch.await(5, TimeUnit.SECONDS));
-      assertEquals(clientHandler.body, "12345");
+      assertEquals("12345", clientHandler.body.toString());
       assertNull(clientHandler.serverHeader);
       // Wait for the server to close the connection.
       ch.close();
+      ch.eventLoop().shutdownNow();
       Assert.assertTrue(webServerComponent.isStarted());
       webServerComponent.stop(true);
       Assert.assertFalse(webServerComponent.isStarted());
@@ -316,9 +342,10 @@ public class WebServerComponentTest extends Assert {
       // Send the HTTP request.
       ch.writeAndFlush(request);
       assertTrue(latch.await(5, TimeUnit.SECONDS));
-      assertEquals(clientHandler.body, "12345");
+      assertEquals("12345", clientHandler.body.toString());
       // Wait for the server to close the connection.
       ch.close();
+      ch.eventLoop().shutdownNow();
       Assert.assertTrue(webServerComponent.isStarted());
       webServerComponent.stop(true);
       Assert.assertFalse(webServerComponent.isStarted());
@@ -362,10 +389,168 @@ public class WebServerComponentTest extends Assert {
       assertEquals(trustPassword, broker.web.getTrustStorePassword());
    }
 
+   @Test
+   public void testServerCleanupBeforeStart() throws Exception {
+      final String warName = "simple-app.war";
+      createTestWar(warName);
+      WebServerDTO webServerDTO = new WebServerDTO();
+      webServerDTO.bind = "http://localhost:0";
+      webServerDTO.path = "";
+      webServerDTO.apps = new ArrayList<>();
+      AppDTO app = new AppDTO();
+      app.url = "simple-app/";
+      app.war = warName;
+      webServerDTO.apps.add(app);
+      WebServerComponent webServerComponent = new WebServerComponent();
+      Assert.assertFalse(webServerComponent.isStarted());
+      testedComponents.add(webServerComponent);
+      webServerComponent.configure(webServerDTO, "./target", "./target");
+      //create some garbage
+      List<WebAppContext> contexts = webServerComponent.getWebContexts();
+
+      File targetDir = new File("./target");
+      File workDir = new File(targetDir, "web-work");
+      workDir.mkdir();
+
+      WebInfConfiguration cfg = new WebInfConfiguration();
+      assertEquals(1, contexts.size());
+      WebAppContext ctxt = contexts.get(0);
+      List<File> garbage = new ArrayList<>();
+
+      ctxt.setAttribute("javax.servlet.context.tempdir", new File(workDir, "jetty-context0"));
+
+      cfg.resolveTempDirectory(ctxt);
+      File tmpdir = ctxt.getTempDirectory();
+      File testDir = tmpdir.getParentFile();
+      createGarbagesInDir(testDir, garbage);
+
+      assertTrue(garbage.size() > 0);
+
+      for (File file : garbage) {
+         assertTrue(file.exists());
+      }
+
+      webServerComponent.start();
+
+      //make sure those garbage are gone
+      for (File file : garbage) {
+         assertFalse("file exist: " + file.getAbsolutePath(), file.exists());
+      }
+
+      //check the war is working
+      final int port = webServerComponent.getPort();
+
+      // Make the connection attempt.
+      CountDownLatch latch = new CountDownLatch(1);
+      final ClientHandler clientHandler = new ClientHandler(latch);
+      bootstrap.group(group).channel(NioSocketChannel.class).handler(new ChannelInitializer() {
+         @Override
+         protected void initChannel(Channel ch) throws Exception {
+            ch.pipeline().addLast(new HttpClientCodec(8192, 8192, 8192, false));
+            ch.pipeline().addLast(clientHandler);
+         }
+      });
+
+      Channel ch = bootstrap.connect("localhost", port).sync().channel();
+
+      String warUrl = "http://localhost:" + port + "/" + app.url;
+
+      URI uri = new URI(warUrl);
+
+      // Prepare the HTTP request.
+      HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri.getRawPath());
+      request.headers().set(HttpHeaderNames.HOST, "localhost");
+
+      // Send the HTTP request.
+      ch.writeAndFlush(request);
+      assertTrue(latch.await(5, TimeUnit.SECONDS));
+
+      assertTrue("content: " + clientHandler.body.toString(), clientHandler.body.toString().contains("Hello Artemis Test"));
+      assertNull(clientHandler.serverHeader);
+      // Wait for the server to close the connection.
+      ch.close();
+      ch.eventLoop().shutdownNow();
+      Assert.assertTrue(webServerComponent.isStarted());
+      webServerComponent.stop(true);
+      Assert.assertFalse(webServerComponent.isStarted());
+   }
+
+   private void createTestWar(String warName) throws Exception {
+      File warFile = new File("target", warName);
+      File srcFile = new File("src/test/webapp");
+      createJarFile(srcFile, warFile);
+   }
+
+   private void createJarFile(File srcFile, File jarFile) throws IOException {
+      Manifest manifest = new Manifest();
+      manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+      try (JarOutputStream target = new JarOutputStream(new FileOutputStream(jarFile), manifest)) {
+         addFile(srcFile, target, "src/test/webapp");
+      }
+   }
+
+   private void addFile(File source, JarOutputStream target, String nameBase) throws IOException {
+      if (source.isDirectory()) {
+         String name = source.getPath().replace("\\", "/");
+         if (!name.isEmpty()) {
+            name = name.substring(nameBase.length());
+            if (!name.endsWith("/")) {
+               name += "/";
+            }
+            JarEntry entry = new JarEntry(name);
+            entry.setTime(source.lastModified());
+            target.putNextEntry(entry);
+            target.closeEntry();
+         }
+         for (File nestedFile: source.listFiles()) {
+            addFile(nestedFile, target, nameBase);
+         }
+         return;
+      }
+
+      String name = source.getPath().replace("\\", "/");
+      name = name.substring(nameBase.length());
+      JarEntry entry = new JarEntry(name);
+      entry.setTime(source.lastModified());
+      target.putNextEntry(entry);
+      try (BufferedInputStream input  = new BufferedInputStream(new FileInputStream(source))) {
+         byte[] buffer = new byte[1024];
+         while (true) {
+            int count = input.read(buffer);
+            if (count == -1)
+               break;
+            target.write(buffer, 0, count);
+         }
+         target.closeEntry();
+      }
+   }
+
+   private void createGarbagesInDir(File tempDirectory, List<File> garbage) throws IOException {
+      if (!tempDirectory.exists()) {
+         tempDirectory.mkdirs();
+      }
+      createRandomJettyFiles(tempDirectory, 10, garbage);
+   }
+
+   private void createRandomJettyFiles(File dir, int num, List<File> collector) throws IOException {
+      for (int i = 0; i < num; i++) {
+         String randomName = "jetty-" + UUID.randomUUID().toString();
+         File file = new File(dir, randomName);
+         if (i % 2 == 0) {
+            //create a dir
+            file.mkdir();
+         } else {
+            //normal file
+            file.createNewFile();
+         }
+         collector.add(file);
+      }
+   }
+
    class ClientHandler extends SimpleChannelInboundHandler<HttpObject> {
 
       private CountDownLatch latch;
-      private String body;
+      private StringBuilder body = new StringBuilder();
       private String serverHeader;
 
       ClientHandler(CountDownLatch latch) {
@@ -379,8 +564,10 @@ public class WebServerComponentTest extends Assert {
             serverHeader = response.headers().get("Server");
          } else if (msg instanceof HttpContent) {
             HttpContent content = (HttpContent) msg;
-            body = content.content().toString(CharsetUtil.UTF_8);
+            body.append(content.content().toString(CharsetUtil.UTF_8));
-            latch.countDown();
+            if (msg instanceof LastHttpContent) {
+               latch.countDown();
+            }
          }
       }
 

artemis-web/src/test/webapp/WEB-INF/web.xml

@@ -0,0 +1,26 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  Architecture
+-->
+<web-app xmlns="http://java.sun.com/xml/ns/j2ee"
+	      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	      xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee
+	      http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd"
+	      version="2.4">
+
+    <display-name>Simple War Application</display-name>
+
+</web-app>

artemis-web/src/test/webapp/index.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  Architecture
+-->
+<html>
+  <head>
+    <meta charset="utf-8"/>
+    <title>simple web</title>
+  </head>
+  <body>
+    <h1>Hello Artemis Test</h1>
+  </body>
+</html>