From d706147423f4585141aae29f95f376a5f0f876ed Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Tue, 13 Jun 2017 14:28:44 -0700 Subject: [PATCH 1/2] add exponential backoff retry for stopping instance in amazon retry only if the error is instancenotfound --- .../amazon/common/step_stop_ebs_instance.go | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/builder/amazon/common/step_stop_ebs_instance.go b/builder/amazon/common/step_stop_ebs_instance.go index 7a3f2bc15..42ce308e8 100644 --- a/builder/amazon/common/step_stop_ebs_instance.go +++ b/builder/amazon/common/step_stop_ebs_instance.go @@ -3,7 +3,9 @@ package common import ( "fmt" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/hashicorp/packer/common" "github.com/hashicorp/packer/packer" "github.com/mitchellh/multistep" ) @@ -28,15 +30,47 @@ func (s *StepStopEBSBackedInstance) Run(state multistep.StateBag) multistep.Step if !s.DisableStopInstance { // Stop the instance so we can create an AMI from it ui.Say("Stopping the source instance...") - _, err = ec2conn.StopInstances(&ec2.StopInstancesInput{ - InstanceIds: []*string{instance.InstanceId}, + + // Amazon EC2 API follows an eventual consistency model. + + // This means that if you run a command to modify or describe a resource + // that you just created, its ID might not have propagated throughout + // the system, and you will get an error responding that the resource + // does not exist. + + // Work around this by retrying a few times, up to about 5 minutes. + err := common.Retry(10, 60, 6, func(i uint) (bool, error) { + ui.Message(fmt.Sprintf("Stopping instance, attempt %d", i+1)) + + _, err = ec2conn.StopInstances(&ec2.StopInstancesInput{ + InstanceIds: []*string{instance.InstanceId}, + }) + + if err == nil { + // success + return true, nil + } + + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "InvalidInstanceID.NotFound" { + ui.Message(fmt.Sprintf( + "Error stopping instance; will retry ..."+ + "Error: %s", err)) + // retry + return false, nil + } + } + // errored, but not in expected way. Don't want to retry + return true, err }) + if err != nil { err := fmt.Errorf("Error stopping instance: %s", err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } + } else { ui.Say("Automatic instance stop disabled. Please stop instance manually.") } From f7a703dfb29fb9f45abc0ed861fc47ab1e54e1f3 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Tue, 20 Jun 2017 10:55:23 -0700 Subject: [PATCH 2/2] add pending to allowable states while waiting for ebs instance to stop --- builder/amazon/common/step_stop_ebs_instance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/common/step_stop_ebs_instance.go b/builder/amazon/common/step_stop_ebs_instance.go index 42ce308e8..b3fb72ee6 100644 --- a/builder/amazon/common/step_stop_ebs_instance.go +++ b/builder/amazon/common/step_stop_ebs_instance.go @@ -78,7 +78,7 @@ func (s *StepStopEBSBackedInstance) Run(state multistep.StateBag) multistep.Step // Wait for the instance to actual stop ui.Say("Waiting for the instance to stop...") stateChange := StateChangeConf{ - Pending: []string{"running", "stopping"}, + Pending: []string{"running", "pending", "stopping"}, Target: "stopped", Refresh: InstanceStateRefreshFunc(ec2conn, *instance.InstanceId), StepState: state,