diff --git a/packer/rpc/communicator.go b/packer/rpc/communicator.go
index c6971a9d7..dc2e570a3 100644
--- a/packer/rpc/communicator.go
+++ b/packer/rpc/communicator.go
@@ -78,7 +78,8 @@ func (c *communicator) Start(cmd *packer.RemoteCmd) (err error) {
 
 		conn, err := responseL.Accept()
 		if err != nil {
-			log.Panic(err)
+			cmd.SetExited(123)
+			return
 		}
 
 		defer conn.Close()
@@ -87,7 +88,8 @@ func (c *communicator) Start(cmd *packer.RemoteCmd) (err error) {
 
 		var finished CommandFinished
 		if err := decoder.Decode(&finished); err != nil {
-			log.Panic(err)
+			cmd.SetExited(123)
+			return
 		}
 
 		cmd.SetExited(finished.ExitStatus)