diff --git a/packer/plugin/client.go b/packer/plugin/client.go
index d4d87088a..b8292c40e 100644
--- a/packer/plugin/client.go
+++ b/packer/plugin/client.go
@@ -73,6 +73,9 @@ type ClientConfig struct {
 //
 // This must only be called _once_.
 func CleanupClients() {
+	// Set the killed to true so that we don't get unexpected panics
+	Killed = true
+
 	// Kill all the managed clients in parallel and use a WaitGroup
 	// to wait for them all to finish up.
 	var wg sync.WaitGroup
diff --git a/signal.go b/signal.go
index 3d465ddbb..de1668dc9 100644
--- a/signal.go
+++ b/signal.go
@@ -28,7 +28,6 @@ func setupSignalHandlers(env packer.Environment) {
 
 		// Force kill all the plugins, but mark that we're killing them
 		// first so that we don't get panics everywhere.
-		plugin.Killed = true
 		plugin.CleanupClients()
 		os.Exit(1)
 	}()