diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/ProtocolHandler.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/ProtocolHandler.java
index c12aa5a8d8..ed80fd395f 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/ProtocolHandler.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/ProtocolHandler.java
@@ -101,7 +101,7 @@ public class ProtocolHandler {
 
       private final boolean httpEnabled;
 
-      private ScheduledFuture timeoutFuture;
+      private ScheduledFuture<?> timeoutFuture;
 
       private int handshakeTimeout;
 
@@ -122,6 +122,15 @@ public class ProtocolHandler {
          }
       }
 
+      @Override
+      public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+         super.channelInactive(ctx);
+         if (handshakeTimeout > 0 && timeoutFuture != null) {
+            timeoutFuture.cancel(true);
+            timeoutFuture = null;
+         }
+      }
+
       @Override
       public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
          if (msg instanceof FullHttpRequest) {