ARTEMIS-2596 (kill -9) AMQ causes tmp web dir space usage to increase
If you kill the server without invoking a normal shutdown, tmp web files are not cleaned out. This leaves old webapp folders lingering until a normal shutdown. In a failover test environment that repeatedly kills the server, this causes disk space usage issues. The fix is to add a cleanup method before the web server starts. It searches the tmp web dir if there is any leftover files/dirs and delete them if any.
This commit is contained in:
Howard Gao
Clebert Suconic
parent
faa83b2ba6
commit
d0a2186e9c
|
|
@ -48,6 +48,7 @@ import org.eclipse.jetty.server.handler.ResourceHandler;
|
|||
import org.eclipse.jetty.servlet.FilterHolder;
|
||||
import org.eclipse.jetty.util.ssl.SslContextFactory;
|
||||
import org.eclipse.jetty.webapp.WebAppContext;
|
||||
import org.eclipse.jetty.webapp.WebInfConfiguration;
|
||||
import org.jboss.logging.Logger;
|
||||
|
||||
import javax.servlet.DispatcherType;
|
||||
|
|
@ -252,6 +253,7 @@ public class WebServerComponent implements ExternalComponent {
|
|||
if (isStarted()) {
|
||||
return;
|
||||
}
|
||||
cleanupTmp();
|
||||
server.start();
|
||||
ActiveMQWebLogger.LOGGER.webserverStarted(webServerConfig.bind);
|
||||
|
||||
|
|
@ -276,6 +278,35 @@ public class WebServerComponent implements ExternalComponent {
|
|||
return libFolder;
|
||||
}
|
||||
|
||||
private void cleanupTmp() {
|
||||
if (webContexts == null || webContexts.size() == 0) {
|
||||
//there is no webapp to be deployed (as in some tests)
|
||||
return;
|
||||
}
|
||||
List<File> temporaryFiles = new ArrayList<>();
|
||||
|
||||
for (WebAppContext context : webContexts) {
|
||||
WebInfConfiguration config = new WebInfConfiguration();
|
||||
try {
|
||||
config.resolveTempDirectory(context);
|
||||
File webTmpBase = context.getTempDirectory().getParentFile();
|
||||
if (webTmpBase.exists()) {
|
||||
webTmpBase.listFiles((f) -> {
|
||||
temporaryFiles.add(f);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
if (temporaryFiles.size() > 0) {
|
||||
WebTmpCleaner.cleanupTmpFiles(getLibFolder(), temporaryFiles, true);
|
||||
}
|
||||
//all web contexts share a same base dir. So we only do it once.
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
logger.warn("Failed to get base dir for tmp web files", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void cleanupWebTemporaryFiles(List<WebAppContext> webContexts) throws Exception {
|
||||
List<File> temporaryFiles = new ArrayList<>();
|
||||
for (WebAppContext context : webContexts) {
|
||||
|
|
@ -327,4 +358,8 @@ public class WebServerComponent implements ExternalComponent {
|
|||
internalStop();
|
||||
}
|
||||
}
|
||||
|
||||
public List<WebAppContext> getWebContexts() {
|
||||
return this.webContexts;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
import org.apache.activemq.artemis.utils.SpawnedVMSupport;
|
||||
import org.jboss.logging.Logger;
|
||||
|
||||
/**
|
||||
* This class is used to remove the jar files
|
||||
|
|
@ -31,11 +32,16 @@ import org.apache.activemq.artemis.utils.SpawnedVMSupport;
|
|||
*/
|
||||
public class WebTmpCleaner {
|
||||
|
||||
private static final Logger logger = Logger.getLogger(WebTmpCleaner.class);
|
||||
|
||||
public static void main(String[] filesToClean) throws Exception {
|
||||
//It needs to retry a bit as we are not sure
|
||||
//when the main VM exists.
|
||||
cleanupFilesWithRetry(filesToClean, 100);
|
||||
}
|
||||
|
||||
private static boolean cleanupFilesWithRetry(String[] filesToClean, int maxRetries) throws Exception {
|
||||
boolean allCleaned = false;
|
||||
int maxRetries = 100;
|
||||
while (!allCleaned && maxRetries-- > 0) {
|
||||
allCleaned = true;
|
||||
for (String f : filesToClean) {
|
||||
|
|
@ -50,16 +56,28 @@ public class WebTmpCleaner {
|
|||
}
|
||||
Thread.sleep(200);
|
||||
}
|
||||
if (!allCleaned) {
|
||||
logger.warn("Some files in web temp dir are not cleaned up after " + maxRetries + " retries.");
|
||||
}
|
||||
return allCleaned;
|
||||
}
|
||||
|
||||
public static Process cleanupTmpFiles(File libFolder, List<File> temporaryFiles) throws Exception {
|
||||
return cleanupTmpFiles(libFolder, temporaryFiles, false);
|
||||
}
|
||||
|
||||
public static Process cleanupTmpFiles(File libFolder, List<File> temporaryFiles, boolean invm) throws Exception {
|
||||
ArrayList<String> files = new ArrayList<>(temporaryFiles.size());
|
||||
for (File f : temporaryFiles) {
|
||||
files.add(f.toURI().toString());
|
||||
}
|
||||
|
||||
String classPath = SpawnedVMSupport.getClassPath(libFolder);
|
||||
return SpawnedVMSupport.spawnVM(classPath, WebTmpCleaner.class.getName(), false, (String[]) files.toArray(new String[files.size()]));
|
||||
if (!invm) {
|
||||
String classPath = SpawnedVMSupport.getClassPath(libFolder);
|
||||
return SpawnedVMSupport.spawnVM(classPath, WebTmpCleaner.class.getName(), false, (String[]) files.toArray(new String[files.size()]));
|
||||
}
|
||||
cleanupFilesWithRetry(files.toArray(new String[files.size()]), 2);
|
||||
return null;
|
||||
}
|
||||
|
||||
public static final void deleteFolder(final File file) {
|
||||
|
|
|
|||
|
|
@ -18,14 +18,23 @@ package org.apache.activemq.cli.test;
|
|||
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLEngine;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.jar.Attributes;
|
||||
import java.util.jar.JarEntry;
|
||||
import java.util.jar.JarOutputStream;
|
||||
import java.util.jar.Manifest;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import io.netty.bootstrap.Bootstrap;
|
||||
|
|
@ -45,22 +54,31 @@ import io.netty.handler.codec.http.HttpObject;
|
|||
import io.netty.handler.codec.http.HttpRequest;
|
||||
import io.netty.handler.codec.http.HttpResponse;
|
||||
import io.netty.handler.codec.http.HttpVersion;
|
||||
import io.netty.handler.codec.http.LastHttpContent;
|
||||
import io.netty.handler.ssl.SslHandler;
|
||||
import io.netty.util.CharsetUtil;
|
||||
import org.apache.activemq.artemis.cli.factory.xml.XmlBrokerFactoryHandler;
|
||||
import org.apache.activemq.artemis.component.WebServerComponent;
|
||||
import org.apache.activemq.artemis.core.remoting.impl.ssl.SSLSupport;
|
||||
import org.apache.activemq.artemis.core.server.ActiveMQComponent;
|
||||
import org.apache.activemq.artemis.dto.AppDTO;
|
||||
import org.apache.activemq.artemis.dto.BrokerDTO;
|
||||
import org.apache.activemq.artemis.dto.WebServerDTO;
|
||||
import org.apache.activemq.artemis.utils.ThreadLeakCheckRule;
|
||||
import org.eclipse.jetty.util.ssl.SslContextFactory;
|
||||
import org.eclipse.jetty.webapp.WebAppContext;
|
||||
import org.eclipse.jetty.webapp.WebInfConfiguration;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
|
||||
public class WebServerComponentTest extends Assert {
|
||||
|
||||
@Rule
|
||||
public ThreadLeakCheckRule leakCheckRule = new ThreadLeakCheckRule();
|
||||
|
||||
static final String URL = System.getProperty("url", "http://localhost:8161/WebServerComponentTest.txt");
|
||||
static final String SECURE_URL = System.getProperty("url", "https://localhost:8448/WebServerComponentTest.txt");
|
||||
private Bootstrap bootstrap;
|
||||
|
|
@ -69,6 +87,7 @@ public class WebServerComponentTest extends Assert {
|
|||
|
||||
@Before
|
||||
public void setupNetty() throws URISyntaxException {
|
||||
System.setProperty("jetty.base", "./target");
|
||||
// Configure the client.
|
||||
group = new NioEventLoopGroup();
|
||||
bootstrap = new Bootstrap();
|
||||
|
|
@ -77,9 +96,11 @@ public class WebServerComponentTest extends Assert {
|
|||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
System.clearProperty("jetty.base");
|
||||
for (ActiveMQComponent c : testedComponents) {
|
||||
c.stop();
|
||||
}
|
||||
testedComponents.clear();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -125,13 +146,15 @@ public class WebServerComponentTest extends Assert {
|
|||
// Send the HTTP request.
|
||||
ch.writeAndFlush(request);
|
||||
assertTrue(latch.await(5, TimeUnit.SECONDS));
|
||||
assertEquals("12345", clientHandler.body.toString());
|
||||
if (useCustomizer) {
|
||||
assertEquals(1, TestCustomizer.count);
|
||||
}
|
||||
assertEquals(clientHandler.body, "12345");
|
||||
assertEquals(clientHandler.body.toString(), "12345");
|
||||
assertNull(clientHandler.serverHeader);
|
||||
// Wait for the server to close the connection.
|
||||
ch.close();
|
||||
ch.eventLoop().shutdownNow();
|
||||
Assert.assertTrue(webServerComponent.isStarted());
|
||||
webServerComponent.stop(true);
|
||||
Assert.assertFalse(webServerComponent.isStarted());
|
||||
|
|
@ -167,9 +190,10 @@ public class WebServerComponentTest extends Assert {
|
|||
// Send the HTTP request.
|
||||
ch.writeAndFlush(request);
|
||||
assertTrue(latch.await(5, TimeUnit.SECONDS));
|
||||
assertEquals(clientHandler.body, "12345");
|
||||
assertEquals("12345", clientHandler.body.toString());
|
||||
// Wait for the server to close the connection.
|
||||
ch.close();
|
||||
ch.eventLoop().shutdownNow();
|
||||
Assert.assertTrue(webServerComponent.isStarted());
|
||||
|
||||
//usual stop won't actually stop it
|
||||
|
|
@ -224,6 +248,7 @@ public class WebServerComponentTest extends Assert {
|
|||
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
final ClientHandler clientHandler = new ClientHandler(latch);
|
||||
|
||||
bootstrap.group(group).channel(NioSocketChannel.class).handler(new ChannelInitializer() {
|
||||
@Override
|
||||
protected void initChannel(Channel ch) throws Exception {
|
||||
|
|
@ -242,10 +267,11 @@ public class WebServerComponentTest extends Assert {
|
|||
// Send the HTTP request.
|
||||
ch.writeAndFlush(request);
|
||||
assertTrue(latch.await(5, TimeUnit.SECONDS));
|
||||
assertEquals(clientHandler.body, "12345");
|
||||
assertEquals("12345", clientHandler.body.toString());
|
||||
assertNull(clientHandler.serverHeader);
|
||||
// Wait for the server to close the connection.
|
||||
ch.close();
|
||||
ch.eventLoop().shutdownNow();
|
||||
Assert.assertTrue(webServerComponent.isStarted());
|
||||
webServerComponent.stop(true);
|
||||
Assert.assertFalse(webServerComponent.isStarted());
|
||||
|
|
@ -316,9 +342,10 @@ public class WebServerComponentTest extends Assert {
|
|||
// Send the HTTP request.
|
||||
ch.writeAndFlush(request);
|
||||
assertTrue(latch.await(5, TimeUnit.SECONDS));
|
||||
assertEquals(clientHandler.body, "12345");
|
||||
assertEquals("12345", clientHandler.body.toString());
|
||||
// Wait for the server to close the connection.
|
||||
ch.close();
|
||||
ch.eventLoop().shutdownNow();
|
||||
Assert.assertTrue(webServerComponent.isStarted());
|
||||
webServerComponent.stop(true);
|
||||
Assert.assertFalse(webServerComponent.isStarted());
|
||||
|
|
@ -362,10 +389,168 @@ public class WebServerComponentTest extends Assert {
|
|||
assertEquals(trustPassword, broker.web.getTrustStorePassword());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testServerCleanupBeforeStart() throws Exception {
|
||||
final String warName = "simple-app.war";
|
||||
createTestWar(warName);
|
||||
WebServerDTO webServerDTO = new WebServerDTO();
|
||||
webServerDTO.bind = "http://localhost:0";
|
||||
webServerDTO.path = "";
|
||||
webServerDTO.apps = new ArrayList<>();
|
||||
AppDTO app = new AppDTO();
|
||||
app.url = "simple-app/";
|
||||
app.war = warName;
|
||||
webServerDTO.apps.add(app);
|
||||
WebServerComponent webServerComponent = new WebServerComponent();
|
||||
Assert.assertFalse(webServerComponent.isStarted());
|
||||
testedComponents.add(webServerComponent);
|
||||
webServerComponent.configure(webServerDTO, "./target", "./target");
|
||||
//create some garbage
|
||||
List<WebAppContext> contexts = webServerComponent.getWebContexts();
|
||||
|
||||
File targetDir = new File("./target");
|
||||
File workDir = new File(targetDir, "web-work");
|
||||
workDir.mkdir();
|
||||
|
||||
WebInfConfiguration cfg = new WebInfConfiguration();
|
||||
assertEquals(1, contexts.size());
|
||||
WebAppContext ctxt = contexts.get(0);
|
||||
List<File> garbage = new ArrayList<>();
|
||||
|
||||
ctxt.setAttribute("javax.servlet.context.tempdir", new File(workDir, "jetty-context0"));
|
||||
|
||||
cfg.resolveTempDirectory(ctxt);
|
||||
File tmpdir = ctxt.getTempDirectory();
|
||||
File testDir = tmpdir.getParentFile();
|
||||
createGarbagesInDir(testDir, garbage);
|
||||
|
||||
assertTrue(garbage.size() > 0);
|
||||
|
||||
for (File file : garbage) {
|
||||
assertTrue(file.exists());
|
||||
}
|
||||
|
||||
webServerComponent.start();
|
||||
|
||||
//make sure those garbage are gone
|
||||
for (File file : garbage) {
|
||||
assertFalse("file exist: " + file.getAbsolutePath(), file.exists());
|
||||
}
|
||||
|
||||
//check the war is working
|
||||
final int port = webServerComponent.getPort();
|
||||
|
||||
// Make the connection attempt.
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
final ClientHandler clientHandler = new ClientHandler(latch);
|
||||
bootstrap.group(group).channel(NioSocketChannel.class).handler(new ChannelInitializer() {
|
||||
@Override
|
||||
protected void initChannel(Channel ch) throws Exception {
|
||||
ch.pipeline().addLast(new HttpClientCodec(8192, 8192, 8192, false));
|
||||
ch.pipeline().addLast(clientHandler);
|
||||
}
|
||||
});
|
||||
|
||||
Channel ch = bootstrap.connect("localhost", port).sync().channel();
|
||||
|
||||
String warUrl = "http://localhost:" + port + "/" + app.url;
|
||||
|
||||
URI uri = new URI(warUrl);
|
||||
|
||||
// Prepare the HTTP request.
|
||||
HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri.getRawPath());
|
||||
request.headers().set(HttpHeaderNames.HOST, "localhost");
|
||||
|
||||
// Send the HTTP request.
|
||||
ch.writeAndFlush(request);
|
||||
assertTrue(latch.await(5, TimeUnit.SECONDS));
|
||||
|
||||
assertTrue("content: " + clientHandler.body.toString(), clientHandler.body.toString().contains("Hello Artemis Test"));
|
||||
assertNull(clientHandler.serverHeader);
|
||||
// Wait for the server to close the connection.
|
||||
ch.close();
|
||||
ch.eventLoop().shutdownNow();
|
||||
Assert.assertTrue(webServerComponent.isStarted());
|
||||
webServerComponent.stop(true);
|
||||
Assert.assertFalse(webServerComponent.isStarted());
|
||||
}
|
||||
|
||||
private void createTestWar(String warName) throws Exception {
|
||||
File warFile = new File("target", warName);
|
||||
File srcFile = new File("src/test/webapp");
|
||||
createJarFile(srcFile, warFile);
|
||||
}
|
||||
|
||||
private void createJarFile(File srcFile, File jarFile) throws IOException {
|
||||
Manifest manifest = new Manifest();
|
||||
manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
|
||||
try (JarOutputStream target = new JarOutputStream(new FileOutputStream(jarFile), manifest)) {
|
||||
addFile(srcFile, target, "src/test/webapp");
|
||||
}
|
||||
}
|
||||
|
||||
private void addFile(File source, JarOutputStream target, String nameBase) throws IOException {
|
||||
if (source.isDirectory()) {
|
||||
String name = source.getPath().replace("\\", "/");
|
||||
if (!name.isEmpty()) {
|
||||
name = name.substring(nameBase.length());
|
||||
if (!name.endsWith("/")) {
|
||||
name += "/";
|
||||
}
|
||||
JarEntry entry = new JarEntry(name);
|
||||
entry.setTime(source.lastModified());
|
||||
target.putNextEntry(entry);
|
||||
target.closeEntry();
|
||||
}
|
||||
for (File nestedFile: source.listFiles()) {
|
||||
addFile(nestedFile, target, nameBase);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
String name = source.getPath().replace("\\", "/");
|
||||
name = name.substring(nameBase.length());
|
||||
JarEntry entry = new JarEntry(name);
|
||||
entry.setTime(source.lastModified());
|
||||
target.putNextEntry(entry);
|
||||
try (BufferedInputStream input = new BufferedInputStream(new FileInputStream(source))) {
|
||||
byte[] buffer = new byte[1024];
|
||||
while (true) {
|
||||
int count = input.read(buffer);
|
||||
if (count == -1)
|
||||
break;
|
||||
target.write(buffer, 0, count);
|
||||
}
|
||||
target.closeEntry();
|
||||
}
|
||||
}
|
||||
|
||||
private void createGarbagesInDir(File tempDirectory, List<File> garbage) throws IOException {
|
||||
if (!tempDirectory.exists()) {
|
||||
tempDirectory.mkdirs();
|
||||
}
|
||||
createRandomJettyFiles(tempDirectory, 10, garbage);
|
||||
}
|
||||
|
||||
private void createRandomJettyFiles(File dir, int num, List<File> collector) throws IOException {
|
||||
for (int i = 0; i < num; i++) {
|
||||
String randomName = "jetty-" + UUID.randomUUID().toString();
|
||||
File file = new File(dir, randomName);
|
||||
if (i % 2 == 0) {
|
||||
//create a dir
|
||||
file.mkdir();
|
||||
} else {
|
||||
//normal file
|
||||
file.createNewFile();
|
||||
}
|
||||
collector.add(file);
|
||||
}
|
||||
}
|
||||
|
||||
class ClientHandler extends SimpleChannelInboundHandler<HttpObject> {
|
||||
|
||||
private CountDownLatch latch;
|
||||
private String body;
|
||||
private StringBuilder body = new StringBuilder();
|
||||
private String serverHeader;
|
||||
|
||||
ClientHandler(CountDownLatch latch) {
|
||||
|
|
@ -379,8 +564,10 @@ public class WebServerComponentTest extends Assert {
|
|||
serverHeader = response.headers().get("Server");
|
||||
} else if (msg instanceof HttpContent) {
|
||||
HttpContent content = (HttpContent) msg;
|
||||
body = content.content().toString(CharsetUtil.UTF_8);
|
||||
latch.countDown();
|
||||
body.append(content.content().toString(CharsetUtil.UTF_8));
|
||||
if (msg instanceof LastHttpContent) {
|
||||
latch.countDown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
Architecture
|
||||
-->
|
||||
<web-app xmlns="http://java.sun.com/xml/ns/j2ee"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee
|
||||
http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd"
|
||||
version="2.4">
|
||||
|
||||
<display-name>Simple War Application</display-name>
|
||||
|
||||
</web-app>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
<!DOCTYPE html>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
Architecture
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>simple web</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Hello Artemis Test</h1>
|
||||
</body>
|
||||
</html>
|
||||
Loading…
Reference in New Issue