Issue #402 (Don't use Thread.isAlive() in ShutdownMonitor).

Refactored completely ShutdownMonitor, fixing synchronization, race
conditions, cleaning up code, deleting unnecessary code, etc.
This commit is contained in:
Simone Bordet 2016-03-08 14:29:49 +01:00
parent 6ac2c4a701
commit 87f011d98b
2 changed files with 368 additions and 455 deletions

View File

@ -27,9 +27,12 @@ import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.function.Predicate;
import org.eclipse.jetty.util.component.Destroyable;
import org.eclipse.jetty.util.component.LifeCycle;
@ -61,141 +64,326 @@ public class ShutdownMonitor
return Holder.instance;
}
public static synchronized void register(LifeCycle... lifeCycles)
public static void register(LifeCycle... lifeCycles)
{
getInstance()._lifeCycles.addAll(Arrays.asList(lifeCycles));
getInstance().addLifeCycles(lifeCycles);
}
public static synchronized void deregister(LifeCycle lifeCycle)
public static void deregister(LifeCycle lifeCycle)
{
getInstance()._lifeCycles.remove(lifeCycle);
getInstance().removeLifeCycle(lifeCycle);
}
public static synchronized boolean isRegistered(LifeCycle lifeCycle)
public static boolean isRegistered(LifeCycle lifeCycle)
{
return getInstance()._lifeCycles.contains(lifeCycle);
return getInstance().containsLifeCycle(lifeCycle);
}
private final Set<LifeCycle> _lifeCycles = new LinkedHashSet<>();
private boolean debug;
private final String host;
private int port;
private String key;
private boolean exitVm;
private boolean alive;
/**
* Creates a ShutdownMonitor using configuration from the System properties.
* <p>
* <code>STOP.PORT</code> = the port to listen on (empty, null, or values less than 0 disable the stop ability)<br>
* <code>STOP.KEY</code> = the magic key/passphrase to allow the stop (defaults to "eclipse")<br>
* <p>
* Note: server socket will only listen on localhost, and a successful stop will issue a System.exit() call.
*/
private ShutdownMonitor()
{
this.debug = System.getProperty("DEBUG") != null;
this.host = System.getProperty("STOP.HOST", "127.0.0.1");
this.port = Integer.parseInt(System.getProperty("STOP.PORT", "-1"));
this.key = System.getProperty("STOP.KEY", null);
this.exitVm = true;
}
private void addLifeCycles(LifeCycle... lifeCycles)
{
synchronized (this)
{
_lifeCycles.addAll(Arrays.asList(lifeCycles));
}
}
private void removeLifeCycle(LifeCycle lifeCycle)
{
synchronized (this)
{
_lifeCycles.remove(lifeCycle);
}
}
private boolean containsLifeCycle(LifeCycle lifeCycle)
{
synchronized (this)
{
return _lifeCycles.contains(lifeCycle);
}
}
private void debug(String format, Object... args)
{
if (debug)
System.err.printf("[ShutdownMonitor] " + format + "%n", args);
}
private void debug(Throwable t)
{
if (debug)
t.printStackTrace(System.err);
}
public String getKey()
{
synchronized (this)
{
return key;
}
}
public int getPort()
{
synchronized (this)
{
return port;
}
}
public boolean isExitVm()
{
synchronized (this)
{
return exitVm;
}
}
public void setDebug(boolean flag)
{
this.debug = flag;
}
/**
* @param exitVm true to exit the VM on shutdown
*/
public void setExitVm(boolean exitVm)
{
synchronized (this)
{
if (alive)
throw new IllegalStateException("ShutdownMonitor already started");
this.exitVm = exitVm;
}
}
public void setKey(String key)
{
synchronized (this)
{
if (alive)
throw new IllegalStateException("ShutdownMonitor already started");
this.key = key;
}
}
public void setPort(int port)
{
synchronized (this)
{
if (alive)
throw new IllegalStateException("ShutdownMonitor already started");
this.port = port;
}
}
protected void start() throws Exception
{
synchronized (this)
{
if (alive)
{
debug("Already started");
return; // cannot start it again
}
ServerSocket serverSocket = listen();
if (serverSocket != null)
{
alive = true;
Thread thread = new Thread(new ShutdownMonitorRunnable(serverSocket));
thread.setDaemon(true);
thread.setName("ShutdownMonitor");
thread.start();
}
}
}
private void stop()
{
synchronized (this)
{
alive = false;
notifyAll();
}
}
// For test purposes only.
void await() throws InterruptedException
{
synchronized (this)
{
while (alive)
{
wait();
}
}
}
protected boolean isAlive()
{
synchronized (this)
{
return alive;
}
}
private ServerSocket listen()
{
int port = getPort();
if (port < 0)
{
debug("Not enabled (port < 0): %d", port);
return null;
}
String key = getKey();
try
{
ServerSocket serverSocket = new ServerSocket();
serverSocket.setReuseAddress(true);
serverSocket.bind(new InetSocketAddress(InetAddress.getByName(host), port));
if (port == 0)
{
port = serverSocket.getLocalPort();
System.out.printf("STOP.PORT=%d%n", port);
setPort(port);
}
if (key == null)
{
key = Long.toString((long)(Long.MAX_VALUE * Math.random() + this.hashCode() + System.currentTimeMillis()), 36);
System.out.printf("STOP.KEY=%s%n", key);
setKey(key);
}
return serverSocket;
}
catch (Throwable x)
{
debug(x);
System.err.println("Error binding ShutdownMonitor to port " + port + ": " + x.toString());
return null;
}
finally
{
// establish the port and key that are in use
debug("STOP.PORT=%d", port);
debug("STOP.KEY=%s", key);
}
}
@Override
public String toString()
{
return String.format("%s[port=%d,alive=%b]", this.getClass().getName(), getPort(), isAlive());
}
/**
* Thread for listening to STOP.PORT for command to stop Jetty.
* If ShowndownMonitor.exitVm is true, then Sytem.exit will also be
* If ShutdownMonitor.exitVm is true, then System.exit will also be
* called after the stop.
*/
private class ShutdownMonitorRunnable implements Runnable
{
public ShutdownMonitorRunnable()
private final ServerSocket serverSocket;
private ShutdownMonitorRunnable(ServerSocket serverSocket)
{
startListenSocket();
this.serverSocket = serverSocket;
}
@Override
public void run()
{
if (serverSocket == null)
{
return;
}
while (serverSocket != null)
{
Socket socket = null;
debug("Started");
try
{
socket = serverSocket.accept();
LineNumberReader lin = new LineNumberReader(new InputStreamReader(socket.getInputStream()));
String receivedKey = lin.readLine();
String key = getKey();
while (true)
{
try (Socket socket = serverSocket.accept())
{
LineNumberReader reader = new LineNumberReader(new InputStreamReader(socket.getInputStream()));
String receivedKey = reader.readLine();
if (!key.equals(receivedKey))
{
System.err.println("Ignoring command with incorrect key");
debug("Ignoring command with incorrect key: %s", receivedKey);
continue;
}
String cmd = reader.readLine();
debug("command=%s", cmd);
OutputStream out = socket.getOutputStream();
boolean exitVm = isExitVm();
String cmd = lin.readLine();
debug("command=%s",cmd);
if ("stop".equalsIgnoreCase(cmd)) //historic, for backward compatibility
{
//Stop the lifecycles, only if they are registered with the ShutdownThread, only destroying if vm is exiting
debug("Issuing stop...");
for (LifeCycle l:_lifeCycles)
{
try
{
if (l.isStarted() && ShutdownThread.isRegistered(l))
{
l.stop();
}
if ((l instanceof Destroyable) && exitVm)
((Destroyable)l).destroy();
}
catch (Exception e)
{
debug(e);
}
}
//Stop accepting any more commands
stopInput(socket);
debug("Performing stop command");
stopLifeCycles(ShutdownThread::isRegistered, exitVm);
// Reply to client
debug("Informing client that we are stopped.");
debug("Informing client that we are stopped");
informClient(out, "Stopped\r\n");
//Stop the output and close the monitor socket
stopOutput(socket);
if (!exitVm)
break;
if (exitVm)
{
// Kill JVM
debug("Killing JVM");
System.exit(0);
}
}
else if ("forcestop".equalsIgnoreCase(cmd))
{
debug("Issuing force stop...");
//Ensure that objects are stopped, destroyed only if vm is forcibly exiting
stopLifeCycles(exitVm);
//Stop accepting any more commands
stopInput(socket);
debug("Performing forced stop command");
stopLifeCycles(l -> true, exitVm);
// Reply to client
debug("Informing client that we are stopped.");
debug("Informing client that we are stopped");
informClient(out, "Stopped\r\n");
//Stop the output and close the monitor socket
stopOutput(socket);
if (!exitVm)
break;
//Honour any pre-setup config to stop the jvm when this command is given
if (exitVm)
{
// Kill JVM
debug("Killing JVM");
System.exit(0);
}
}
else if ("stopexit".equalsIgnoreCase(cmd))
{
debug("Issuing stop and exit...");
//Make sure that objects registered with the shutdown thread will be stopped
stopLifeCycles(true);
//Stop accepting any more input
stopInput(socket);
debug("Performing stop and exit commands");
stopLifeCycles(ShutdownThread::isRegistered, true);
// Reply to client
debug("Informing client that we are stopped.");
debug("Informing client that we are stopped");
informClient(out, "Stopped\r\n");
//Stop the output and close the monitor socket
stopOutput(socket);
debug("Killing JVM");
System.exit(0);
}
@ -210,309 +398,52 @@ public class ShutdownMonitor
informClient(out, "OK\r\n");
}
}
catch (Exception e)
catch (Throwable x)
{
debug(e);
System.err.println(e.toString());
debug(x);
}
}
}
catch (Throwable x)
{
debug(x);
}
finally
{
close(socket);
socket = null;
}
stop();
debug("Stopped");
}
}
public void stopInput (Socket socket)
{
//Stop accepting any more input
close(serverSocket);
serverSocket = null;
//Shutdown input from client
shutdownInput(socket);
}
public void stopOutput (Socket socket) throws IOException
{
socket.shutdownOutput();
close(socket);
socket = null;
debug("Shutting down monitor");
serverSocket = null;
}
public void informClient (OutputStream out, String message) throws IOException
private void informClient(OutputStream out, String message) throws IOException
{
out.write(message.getBytes(StandardCharsets.UTF_8));
out.flush();
}
/**
* Stop the registered lifecycles, optionally
* calling destroy on them.
*
* @param destroy true if {@link Destroyable}'s should also be destroyed.
*/
public void stopLifeCycles (boolean destroy)
private void stopLifeCycles(Predicate<LifeCycle> predicate, boolean destroy)
{
for (LifeCycle l:_lifeCycles)
List<LifeCycle> lifeCycles = new ArrayList<>();
synchronized (this)
{
lifeCycles.addAll(_lifeCycles);
}
for (LifeCycle l : lifeCycles)
{
try
{
if (l.isStarted())
{
if (l.isStarted() && predicate.test(l))
l.stop();
}
if ((l instanceof Destroyable) && destroy)
((Destroyable)l).destroy();
}
catch (Exception e)
catch (Throwable x)
{
debug(e);
debug(x);
}
}
}
public void startListenSocket()
{
if (port < 0)
{
if (DEBUG)
System.err.println("ShutdownMonitor not in use (port < 0): " + port);
return;
}
try
{
serverSocket = new ServerSocket();
serverSocket.setReuseAddress(true);
serverSocket.bind(new InetSocketAddress(InetAddress.getByName(host), port), 1);
if (port == 0)
{
// server assigned port in use
port = serverSocket.getLocalPort();
System.out.printf("STOP.PORT=%d%n",port);
}
if (key == null)
{
// create random key
key = Long.toString((long)(Long.MAX_VALUE * Math.random() + this.hashCode() + System.currentTimeMillis()),36);
System.out.printf("STOP.KEY=%s%n",key);
}
}
catch (Exception e)
{
debug(e);
System.err.println("Error binding monitor port " + port + ": " + e.toString());
serverSocket = null;
}
finally
{
// establish the port and key that are in use
debug("STOP.PORT=%d",port);
debug("STOP.KEY=%s",key);
debug("%s",serverSocket);
}
}
}
private final Set<LifeCycle> _lifeCycles = new CopyOnWriteArraySet<LifeCycle>();
private boolean DEBUG;
private String host;
private int port;
private String key;
private boolean exitVm;
private ServerSocket serverSocket;
private Thread thread;
/**
* Create a ShutdownMonitor using configuration from the System properties.
* <p>
* <code>STOP.PORT</code> = the port to listen on (empty, null, or values less than 0 disable the stop ability)<br>
* <code>STOP.KEY</code> = the magic key/passphrase to allow the stop (defaults to "eclipse")<br>
* <p>
* Note: server socket will only listen on localhost, and a successful stop will issue a System.exit() call.
*/
private ShutdownMonitor()
{
this.DEBUG = System.getProperty("DEBUG") != null;
// Use values passed thru via /jetty-start/
this.host = System.getProperty("STOP.HOST","127.0.0.1");
this.port = Integer.parseInt(System.getProperty("STOP.PORT","-1"));
this.key = System.getProperty("STOP.KEY",null);
this.exitVm = true;
}
private void close(ServerSocket server)
{
if (server == null)
{
return;
}
try
{
server.close();
}
catch (IOException ignore)
{
debug(ignore);
}
}
private void close(Socket socket)
{
if (socket == null)
{
return;
}
try
{
socket.close();
}
catch (IOException ignore)
{
debug(ignore);
}
}
private void shutdownInput(Socket socket)
{
if (socket == null)
return;
try
{
socket.shutdownInput();
}
catch (IOException ignore)
{
debug(ignore);
}
}
private void debug(String format, Object... args)
{
if (DEBUG)
{
System.err.printf("[ShutdownMonitor] " + format + "%n",args);
}
}
private void debug(Throwable t)
{
if (DEBUG)
{
t.printStackTrace(System.err);
}
}
public String getKey()
{
return key;
}
public int getPort()
{
return port;
}
public ServerSocket getServerSocket()
{
return serverSocket;
}
public boolean isExitVm()
{
return exitVm;
}
public void setDebug(boolean flag)
{
this.DEBUG = flag;
}
/**
* @param exitVm true to exit the VM on shutdown
*/
public void setExitVm(boolean exitVm)
{
synchronized (this)
{
if (thread != null && thread.isAlive())
{
throw new IllegalStateException("ShutdownMonitorThread already started");
}
this.exitVm = exitVm;
}
}
public void setKey(String key)
{
synchronized (this)
{
if (thread != null && thread.isAlive())
{
throw new IllegalStateException("ShutdownMonitorThread already started");
}
this.key = key;
}
}
public void setPort(int port)
{
synchronized (this)
{
if (thread != null && thread.isAlive())
{
throw new IllegalStateException("ShutdownMonitorThread already started");
}
this.port = port;
}
}
protected void start() throws Exception
{
Thread t = null;
synchronized (this)
{
if (thread != null && thread.isAlive())
{
if (DEBUG)
System.err.printf("ShutdownMonitorThread already started");
return; // cannot start it again
}
thread = new Thread(new ShutdownMonitorRunnable());
thread.setDaemon(true);
thread.setName("ShutdownMonitor");
t = thread;
}
if (t != null)
t.start();
}
protected boolean isAlive ()
{
boolean result = false;
synchronized (this)
{
result = (thread != null && thread.isAlive());
}
return result;
}
@Override
public String toString()
{
return String.format("%s[port=%d]",this.getClass().getName(),port);
}
}

View File

@ -18,89 +18,59 @@
package org.eclipse.jetty.server;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.Socket;
import java.util.concurrent.TimeUnit;
import org.eclipse.jetty.util.thread.ShutdownThread;
import org.junit.Test;
/**
* ShutdownMonitorTest
*/
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class ShutdownMonitorTest
{
public class TestableServer extends Server
{
boolean destroyed = false;
boolean stopped = false;
@Override
protected void doStop() throws Exception
{
stopped = true;
super.doStop();
}
@Override
public void destroy()
{
destroyed = true;
super.destroy();
}
@Override
protected void doStart() throws Exception
{
stopped = false;
destroyed = false;
super.doStart();
}
}
@Test
public void testShutdownMonitor() throws Exception
{
// test port and key assignment
ShutdownMonitor.getInstance().setPort(0);
ShutdownMonitor.getInstance().setExitVm(false);
ShutdownMonitor.getInstance().start();
String key = ShutdownMonitor.getInstance().getKey();
int port = ShutdownMonitor.getInstance().getPort();
ShutdownMonitor monitor = ShutdownMonitor.getInstance();
monitor.setDebug(true);
monitor.setPort(0);
monitor.setExitVm(false);
monitor.start();
String key = monitor.getKey();
int port = monitor.getPort();
// try starting a 2nd time (should be ignored)
ShutdownMonitor.getInstance().start();
monitor.start();
stop("stop", port,key,true);
assertTrue(!ShutdownMonitor.getInstance().isAlive());
stop("stop", port, key, true);
monitor.await();
assertTrue(!monitor.isAlive());
// should be able to change port and key because it is stopped
ShutdownMonitor.getInstance().setPort(0);
ShutdownMonitor.getInstance().setKey("foo");
ShutdownMonitor.getInstance().start();
monitor.setPort(0);
String newKey = "foo";
monitor.setKey(newKey);
monitor.start();
key = ShutdownMonitor.getInstance().getKey();
port = ShutdownMonitor.getInstance().getPort();
assertTrue(ShutdownMonitor.getInstance().isAlive());
key = monitor.getKey();
assertEquals(newKey, key);
port = monitor.getPort();
assertTrue(monitor.isAlive());
stop("stop", port,key,true);
assertTrue(!ShutdownMonitor.getInstance().isAlive());
stop("stop", port, key, true);
monitor.await();
assertTrue(!monitor.isAlive());
}
@Test
public void testForceStopCommand() throws Exception
{
//create a testable Server with stop(), destroy() overridden to instrument
//start server
//call "forcestop" and check that server stopped but not destroyed
// test port and key assignment
System.setProperty("DEBUG", "true");
ShutdownMonitor.getInstance().setPort(0);
ShutdownMonitor monitor = ShutdownMonitor.getInstance();
monitor.setPort(0);
TestableServer server = new TestableServer();
server.start();
@ -108,12 +78,13 @@ public class ShutdownMonitorTest
assertTrue(!ShutdownThread.isRegistered(server));
assertTrue(ShutdownMonitor.isRegistered(server));
String key = ShutdownMonitor.getInstance().getKey();
int port = ShutdownMonitor.getInstance().getPort();
String key = monitor.getKey();
int port = monitor.getPort();
stop("forcestop", port,key,true);
stop("forcestop", port, key, true);
monitor.await();
assertTrue(!ShutdownMonitor.getInstance().isAlive());
assertTrue(!monitor.isAlive());
assertTrue(server.stopped);
assertTrue(!server.destroyed);
assertTrue(!ShutdownThread.isRegistered(server));
@ -123,21 +94,10 @@ public class ShutdownMonitorTest
@Test
public void testOldStopCommandWithStopOnShutdownTrue() throws Exception
{
ShutdownMonitor monitor = ShutdownMonitor.getInstance();
monitor.setExitVm(false);
//create a testable Server with stop(), destroy() overridden to instrument
//call server.setStopAtShudown(true);
//start server
//call "stop" and check that server stopped but not destroyed
//stop server
//call server.setStopAtShutdown(false);
//start server
//call "stop" and check that the server is not stopped and not destroyed
System.setProperty("DEBUG", "true");
ShutdownMonitor.getInstance().setExitVm(false);
ShutdownMonitor.getInstance().setPort(0);
monitor.setPort(0);
TestableServer server = new TestableServer();
server.setStopAtShutdown(true);
server.start();
@ -146,11 +106,13 @@ public class ShutdownMonitorTest
assertTrue(ShutdownThread.isRegistered(server));
assertTrue(ShutdownMonitor.isRegistered(server));
String key = ShutdownMonitor.getInstance().getKey();
int port = ShutdownMonitor.getInstance().getPort();
String key = monitor.getKey();
int port = monitor.getPort();
stop("stop", port, key, true);
assertTrue(!ShutdownMonitor.getInstance().isAlive());
monitor.await();
assertTrue(!monitor.isAlive());
assertTrue(server.stopped);
assertTrue(!server.destroyed);
assertTrue(!ShutdownThread.isRegistered(server));
@ -160,10 +122,9 @@ public class ShutdownMonitorTest
@Test
public void testOldStopCommandWithStopOnShutdownFalse() throws Exception
{
//change so stopatshutdown is false, so stop does nothing in this case (as exitVm is false otherwise we couldn't run test)
ShutdownMonitor.getInstance().setExitVm(false);
System.setProperty("DEBUG", "true");
ShutdownMonitor.getInstance().setPort(0);
ShutdownMonitor monitor = ShutdownMonitor.getInstance();
monitor.setExitVm(false);
monitor.setPort(0);
TestableServer server = new TestableServer();
server.setStopAtShutdown(false);
server.start();
@ -171,43 +132,37 @@ public class ShutdownMonitorTest
assertTrue(!ShutdownThread.isRegistered(server));
assertTrue(ShutdownMonitor.isRegistered(server));
String key = ShutdownMonitor.getInstance().getKey();
int port = ShutdownMonitor.getInstance().getPort();
String key = monitor.getKey();
int port = monitor.getPort();
stop ("stop", port, key, true);
assertTrue(!ShutdownMonitor.getInstance().isAlive());
stop("stop", port, key, true);
monitor.await();
assertTrue(!monitor.isAlive());
assertTrue(!server.stopped);
assertTrue(!server.destroyed);
assertTrue(!ShutdownThread.isRegistered(server));
assertTrue(ShutdownMonitor.isRegistered(server));
}
public void stop(String command, int port, String key, boolean check) throws Exception
{
System.out.printf("Attempting to send "+command+" to localhost:%d (%b)%n",port,check);
try (Socket s = new Socket(InetAddress.getByName("127.0.0.1"),port))
System.out.printf("Attempting to send " + command + " to localhost:%d (%b)%n", port, check);
try (Socket s = new Socket(InetAddress.getByName("127.0.0.1"), port))
{
// send stop command
try (OutputStream out = s.getOutputStream())
{
out.write((key + "\r\n"+command+"\r\n").getBytes());
out.write((key + "\r\n" + command + "\r\n").getBytes());
out.flush();
if (check)
{
// wait a little
TimeUnit.MILLISECONDS.sleep(600);
// check for stop confirmation
LineNumberReader lin = new LineNumberReader(new InputStreamReader(s.getInputStream()));
String response;
if ((response = lin.readLine()) != null)
{
assertEquals("Stopped",response);
}
assertEquals("Stopped", response);
else
throw new IllegalStateException("No stop confirmation");
}
@ -215,4 +170,31 @@ public class ShutdownMonitorTest
}
}
public class TestableServer extends Server
{
boolean destroyed = false;
boolean stopped = false;
@Override
protected void doStop() throws Exception
{
stopped = true;
super.doStop();
}
@Override
public void destroy()
{
destroyed = true;
super.destroy();
}
@Override
protected void doStart() throws Exception
{
stopped = false;
destroyed = false;
super.doStart();
}
}
}