builder/digitalocean: retry power off a number of times

See comment.

builder/digitalocean/step_droplet_info.go

@@ -16,7 +16,7 @@ func (s *stepDropletInfo) Run(state multistep.StateBag) multistep.StepAction {
 
 	ui.Say("Waiting for droplet to become active...")
 
-	err := waitForDropletState("active", dropletId, client, c)
+	err := waitForDropletState("active", dropletId, client, c.stateTimeout)
 	if err != nil {
 		err := fmt.Errorf("Error waiting for droplet to become active: %s", err)
 		state.Put("error", err)

builder/digitalocean/step_power_off.go

@@ -5,6 +5,7 @@ import (
 	"github.com/mitchellh/multistep"
 	"github.com/mitchellh/packer/packer"
 	"log"
+	"time"
 )
 
 type stepPowerOff struct{}
@@ -14,10 +15,27 @@ func (s *stepPowerOff) Run(state multistep.StateBag) multistep.StepAction {
 	ui := state.Get("ui").(packer.Ui)
 	dropletId := state.Get("droplet_id").(uint)
 
-	// Poweroff the droplet so it can be snapshot
-	err := client.PowerOffDroplet(dropletId)
+	// Gracefully power off the droplet. We have to retry this a number
+	// of times because sometimes it says it completed when it actually
+	// did absolutely nothing (*ALAKAZAM!* magic!). We give up after
+	// a pretty arbitrary amount of time.
+	var err error
+	ui.Say("Gracefully shutting down droplet...")
+	for attempts := 1; attempts <= 10; attempts++ {
+		log.Printf("PowerOffDroplet attempt #%d...", attempts)
+		err := client.PowerOffDroplet(dropletId)
+		if err != nil {
+			err := fmt.Errorf("Error powering off droplet: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		err = waitForDropletState("off", dropletId, client, 20*time.Second)
+	}
+
 	if err != nil {
-		err := fmt.Errorf("Error powering off droplet: %s", err)
+		err := fmt.Errorf("Error waiting for droplet to become 'off': %s", err)
 		state.Put("error", err)
 		ui.Error(err.Error())
 		return multistep.ActionHalt

builder/digitalocean/step_shutdown.go

@@ -22,8 +22,7 @@ func (s *stepShutdown) Run(state multistep.StateBag) multistep.StepAction {
 		return multistep.ActionHalt
 	}
 
-	ui.Say("Waiting for droplet to shutdown...")
-	err = waitForDropletState("off", dropletId, client, c)
+	err = waitForDropletState("off", dropletId, client, c.stateTimeout)
 	if err != nil {
 		err := fmt.Errorf("Error waiting for droplet to become 'off': %s", err)
 		state.Put("error", err)

builder/digitalocean/step_snapshot.go

@@ -26,7 +26,7 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction {
 	}
 
 	ui.Say("Waiting for snapshot to complete...")
-	err = waitForDropletState("active", dropletId, client, c)
+	err = waitForDropletState("active", dropletId, client, c.stateTimeout)
 	if err != nil {
 		err := fmt.Errorf("Error waiting for snapshot to complete: %s", err)
 		state.Put("error", err)

builder/digitalocean/wait.go

@@ -8,7 +8,7 @@ import (
 
 // waitForState simply blocks until the droplet is in
 // a state we expect, while eventually timing out.
-func waitForDropletState(desiredState string, dropletId uint, client *DigitalOceanClient, c config) error {
+func waitForDropletState(desiredState string, dropletId uint, client *DigitalOceanClient, timeout time.Duration) error {
 	result := make(chan error, 1)
 	go func() {
 		attempts := 0
@@ -32,11 +32,11 @@ func waitForDropletState(desiredState string, dropletId uint, client *DigitalOce
 		}
 	}()
 
-	log.Printf("Waiting for up to %s for droplet to become %s", c.RawStateTimeout, desiredState)
+	log.Printf("Waiting for up to %d seconds for droplet to become %s", timeout, desiredState)
 	select {
 	case err := <-result:
 		return err
-	case <-time.After(c.stateTimeout):
+	case <-time.After(timeout):
 		err := fmt.Errorf("Timeout while waiting to for droplet to become '%s'", desiredState)
 		return err
 	}