Merge pull request #5029 from hashicorp/5007_instance_stop

add exponential backoff retry for stopping instance in amazon

builder/amazon/common/step_stop_ebs_instance.go

@@ -3,7 +3,9 @@ package common
 import (
 	"fmt"
 
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/packer"
 	"github.com/mitchellh/multistep"
 )
@@ -28,15 +30,47 @@ func (s *StepStopEBSBackedInstance) Run(state multistep.StateBag) multistep.Step
 	if !s.DisableStopInstance {
 		// Stop the instance so we can create an AMI from it
 		ui.Say("Stopping the source instance...")
-		_, err = ec2conn.StopInstances(&ec2.StopInstancesInput{
-			InstanceIds: []*string{instance.InstanceId},
+
+		// Amazon EC2 API follows an eventual consistency model.
+
+		// This means that if you run a command to modify or describe a resource
+		// that you just created, its ID might not have propagated throughout
+		// the system, and you will get an error responding that the resource
+		// does not exist.
+
+		// Work around this by retrying a few times, up to about 5 minutes.
+		err := common.Retry(10, 60, 6, func(i uint) (bool, error) {
+			ui.Message(fmt.Sprintf("Stopping instance, attempt %d", i+1))
+
+			_, err = ec2conn.StopInstances(&ec2.StopInstancesInput{
+				InstanceIds: []*string{instance.InstanceId},
+			})
+
+			if err == nil {
+				// success
+				return true, nil
+			}
+
+			if awsErr, ok := err.(awserr.Error); ok {
+				if awsErr.Code() == "InvalidInstanceID.NotFound" {
+					ui.Message(fmt.Sprintf(
+						"Error stopping instance; will retry ..."+
+							"Error: %s", err))
+					// retry
+					return false, nil
+				}
+			}
+			// errored, but not in expected way. Don't want to retry
+			return true, err
 		})
+
 		if err != nil {
 			err := fmt.Errorf("Error stopping instance: %s", err)
 			state.Put("error", err)
 			ui.Error(err.Error())
 			return multistep.ActionHalt
 		}
+
 	} else {
 		ui.Say("Automatic instance stop disabled. Please stop instance manually.")
 	}
@@ -44,7 +78,7 @@ func (s *StepStopEBSBackedInstance) Run(state multistep.StateBag) multistep.Step
 	// Wait for the instance to actual stop
 	ui.Say("Waiting for the instance to stop...")
 	stateChange := StateChangeConf{
-		Pending:   []string{"running", "stopping"},
+		Pending:   []string{"running", "pending", "stopping"},
 		Target:    "stopped",
 		Refresh:   InstanceStateRefreshFunc(ec2conn, *instance.InstanceId),
 		StepState: state,