builder/amazonebs: retry SSH handshakes [GH-130]
This commit is contained in:
parent
27bada1ba9
commit
65d74f90c1
|
@ -22,6 +22,7 @@ BUG FIXES:
|
||||||
|
|
||||||
* core: Non-200 response codes on downloads now show proper errors.
|
* core: Non-200 response codes on downloads now show proper errors.
|
||||||
[GH-141]
|
[GH-141]
|
||||||
|
* amazon-ebs: SSH handshake is retried. [GH-130]
|
||||||
* vagrant: The `BuildName` template propery works properly in
|
* vagrant: The `BuildName` template propery works properly in
|
||||||
the output path.
|
the output path.
|
||||||
* vagrant: Properly configure the provider-specific post-processors so
|
* vagrant: Properly configure the provider-specific post-processors so
|
||||||
|
|
|
@ -14,10 +14,64 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type stepConnectSSH struct {
|
type stepConnectSSH struct {
|
||||||
|
cancel bool
|
||||||
conn net.Conn
|
conn net.Conn
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stepConnectSSH) Run(state map[string]interface{}) multistep.StepAction {
|
func (s *stepConnectSSH) Run(state map[string]interface{}) multistep.StepAction {
|
||||||
|
config := state["config"].(config)
|
||||||
|
ui := state["ui"].(packer.Ui)
|
||||||
|
|
||||||
|
var comm packer.Communicator
|
||||||
|
var err error
|
||||||
|
|
||||||
|
waitDone := make(chan bool, 1)
|
||||||
|
go func() {
|
||||||
|
comm, err = s.waitForSSH(state)
|
||||||
|
waitDone <- true
|
||||||
|
}()
|
||||||
|
|
||||||
|
log.Printf("Waiting for SSH, up to timeout: %s", config.SSHTimeout.String())
|
||||||
|
|
||||||
|
timeout := time.After(config.SSHTimeout)
|
||||||
|
WaitLoop:
|
||||||
|
for {
|
||||||
|
// Wait for either SSH to become available, a timeout to occur,
|
||||||
|
// or an interrupt to come through.
|
||||||
|
select {
|
||||||
|
case <-waitDone:
|
||||||
|
if err != nil {
|
||||||
|
ui.Error(fmt.Sprintf("Error waiting for SSH: %s", err))
|
||||||
|
return multistep.ActionHalt
|
||||||
|
}
|
||||||
|
|
||||||
|
state["communicator"] = comm
|
||||||
|
break WaitLoop
|
||||||
|
case <-timeout:
|
||||||
|
ui.Error("Timeout waiting for SSH.")
|
||||||
|
s.cancel = true
|
||||||
|
return multistep.ActionHalt
|
||||||
|
case <-time.After(1 * time.Second):
|
||||||
|
if _, ok := state[multistep.StateCancelled]; ok {
|
||||||
|
log.Println("Interrupt detected, quitting waiting for SSH.")
|
||||||
|
return multistep.ActionHalt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return multistep.ActionContinue
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stepConnectSSH) Cleanup(map[string]interface{}) {
|
||||||
|
if s.conn != nil {
|
||||||
|
s.conn.Close()
|
||||||
|
s.conn = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// This blocks until SSH becomes available, and sends the communicator
|
||||||
|
// on the given channel.
|
||||||
|
func (s *stepConnectSSH) waitForSSH(state map[string]interface{}) (packer.Communicator, error) {
|
||||||
config := state["config"].(config)
|
config := state["config"].(config)
|
||||||
instance := state["instance"].(*ec2.Instance)
|
instance := state["instance"].(*ec2.Instance)
|
||||||
privateKey := state["privateKey"].(string)
|
privateKey := state["privateKey"].(string)
|
||||||
|
@ -28,10 +82,33 @@ func (s *stepConnectSSH) Run(state map[string]interface{}) multistep.StepAction
|
||||||
keyring := &ssh.SimpleKeychain{}
|
keyring := &ssh.SimpleKeychain{}
|
||||||
err := keyring.AddPEMKey(privateKey)
|
err := keyring.AddPEMKey(privateKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err := fmt.Errorf("Error setting up SSH config: %s", err)
|
return nil, fmt.Errorf("Error setting up SSH config: %s", err)
|
||||||
state["error"] = err
|
}
|
||||||
ui.Error(err.Error())
|
|
||||||
return multistep.ActionHalt
|
ui.Say("Waiting for SSH to become available...")
|
||||||
|
var comm packer.Communicator
|
||||||
|
var nc net.Conn
|
||||||
|
for {
|
||||||
|
if nc != nil {
|
||||||
|
nc.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
|
||||||
|
if s.cancel {
|
||||||
|
log.Println("SSH wait cancelled. Exiting loop.")
|
||||||
|
return nil, errors.New("SSH wait cancelled")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt to connect to SSH port
|
||||||
|
log.Printf(
|
||||||
|
"Opening TCP conn for SSH to %s:%d",
|
||||||
|
instance.DNSName, config.SSHPort)
|
||||||
|
nc, err := net.Dial("tcp",
|
||||||
|
fmt.Sprintf("%s:%d", instance.DNSName, config.SSHPort))
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("TCP connection to SSH ip/port failed: %s", err)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build the actual SSH client configuration
|
// Build the actual SSH client configuration
|
||||||
|
@ -42,84 +119,33 @@ func (s *stepConnectSSH) Run(state map[string]interface{}) multistep.StepAction
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start trying to connect to SSH
|
sshConnectSuccess := make(chan bool, 1)
|
||||||
connected := make(chan bool, 1)
|
|
||||||
connectQuit := make(chan bool, 1)
|
|
||||||
defer func() {
|
|
||||||
connectQuit <- true
|
|
||||||
}()
|
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
var err error
|
comm, err = ssh.New(nc, sshConfig)
|
||||||
|
if err != nil {
|
||||||
ui.Say("Connecting to the instance via SSH...")
|
log.Printf("SSH connection fail: %s", err)
|
||||||
attempts := 0
|
sshConnectSuccess <- false
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-connectQuit:
|
|
||||||
return
|
return
|
||||||
default:
|
|
||||||
}
|
}
|
||||||
|
|
||||||
attempts += 1
|
sshConnectSuccess <- true
|
||||||
log.Printf(
|
}()
|
||||||
"Opening TCP conn for SSH to %s:%d (attempt %d)",
|
|
||||||
instance.DNSName, config.SSHPort, attempts)
|
select {
|
||||||
s.conn, err = net.Dial("tcp", fmt.Sprintf("%s:%d", instance.DNSName, config.SSHPort))
|
case success := <-sshConnectSuccess:
|
||||||
if err == nil {
|
if !success {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
log.Printf("SSH handshake timeout. Trying again.")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
ui.Say("Connected via SSH!")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
// A brief sleep so we're not being overly zealous attempting
|
// Store the connection so we can close it later
|
||||||
// to connect to the instance.
|
s.conn = nc
|
||||||
time.Sleep(500 * time.Millisecond)
|
return comm, nil
|
||||||
}
|
|
||||||
|
|
||||||
connected <- true
|
|
||||||
}()
|
|
||||||
|
|
||||||
log.Printf("Waiting up to %s for SSH connection", config.SSHTimeout)
|
|
||||||
timeout := time.After(config.SSHTimeout)
|
|
||||||
|
|
||||||
ConnectWaitLoop:
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-connected:
|
|
||||||
// We connected. Just break the loop.
|
|
||||||
break ConnectWaitLoop
|
|
||||||
case <-timeout:
|
|
||||||
err := errors.New("Timeout waiting for SSH to become available.")
|
|
||||||
state["error"] = err
|
|
||||||
ui.Error(err.Error())
|
|
||||||
return multistep.ActionHalt
|
|
||||||
case <-time.After(1 * time.Second):
|
|
||||||
if _, ok := state[multistep.StateCancelled]; ok {
|
|
||||||
log.Println("Interrupt detected, quitting waiting for SSH.")
|
|
||||||
return multistep.ActionHalt
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var comm packer.Communicator
|
|
||||||
if err == nil {
|
|
||||||
comm, err = ssh.New(s.conn, sshConfig)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
err := fmt.Errorf("Error connecting to SSH: %s", err)
|
|
||||||
state["error"] = err
|
|
||||||
ui.Error(err.Error())
|
|
||||||
return multistep.ActionHalt
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set the communicator on the state bag so it can be used later
|
|
||||||
state["communicator"] = comm
|
|
||||||
|
|
||||||
return multistep.ActionContinue
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *stepConnectSSH) Cleanup(map[string]interface{}) {
|
|
||||||
if s.conn != nil {
|
|
||||||
s.conn.Close()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue