diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 07abadebf..abb0ab0f1 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -199,14 +199,34 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa return multistep.ActionHalt } - r, err := ec2conn.DescribeInstances(describeInstance) + // there's a race condition that can happen because of AWS's eventual + // consistency where even though the wait is complete, the describe call + // will fail. Retry a couple of times to try to mitigate that race. + var r *ec2.DescribeInstancesOutput + err = retry.Config{ + Tries: 11, + ShouldRetry: func(err error) bool { + if awsErr, ok := err.(awserr.Error); ok { + switch awsErr.Code() { + case "InvalidInstanceID.NotFound": + return true + } + } + return false + }, + RetryDelay: (&retry.Backoff{InitialBackoff: 200 * time.Millisecond, MaxBackoff: 30 * time.Second, Multiplier: 2}).Linear, + }.Run(ctx, func(ctx context.Context) error { + r, err = ec2conn.DescribeInstances(describeInstance) + return err + }) if err != nil || len(r.Reservations) == 0 || len(r.Reservations[0].Instances) == 0 { err := fmt.Errorf("Error finding source instance.") state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } + instance := r.Reservations[0].Instances[0] if s.Debug { diff --git a/builder/amazon/common/step_run_spot_instance.go b/builder/amazon/common/step_run_spot_instance.go index d45db0208..ab15bdff3 100644 --- a/builder/amazon/common/step_run_spot_instance.go +++ b/builder/amazon/common/step_run_spot_instance.go @@ -286,6 +286,31 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) } instanceId = *createOutput.Instances[0].InstanceIds[0] + // Set the instance ID so that the cleanup works properly + s.instanceId = instanceId + + ui.Message(fmt.Sprintf("Instance ID: %s", instanceId)) + + // Get information about the created instance + var describeOutput *ec2.DescribeInstancesOutput + err = retry.Config{ + Tries: 11, + RetryDelay: (&retry.Backoff{InitialBackoff: 200 * time.Millisecond, MaxBackoff: 30 * time.Second, Multiplier: 2}).Linear, + }.Run(ctx, func(ctx context.Context) error { + describeOutput, err = ec2conn.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceId)}, + }) + return err + }) + if err != nil || len(describeOutput.Reservations) == 0 || len(describeOutput.Reservations[0].Instances) == 0 { + err := fmt.Errorf("Error finding source instance.") + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + instance := describeOutput.Reservations[0].Instances[0] + // Tag the spot instance request (not the eventual spot instance) spotTags, err := s.SpotTags.EC2Tags(s.Ctx, *ec2conn.Config.Region, state) if err != nil { @@ -297,30 +322,8 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) if len(spotTags) > 0 && s.SpotTags.IsSet() { spotTags.Report(ui) - // Use the instance ID to find out the SIR, so that we can tag the spot // request associated with this instance. - - err = retry.Config{ - Tries: 11, - ShouldRetry: func(error) bool { return true }, - RetryDelay: (&retry.Backoff{InitialBackoff: 200 * time.Millisecond, MaxBackoff: 30 * time.Second, Multiplier: 2}).Linear, - }.Run(ctx, func(ctx context.Context) error { - _, err := ec2conn.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: []*string{aws.String(instanceId)}, - }) - return err - }) - if err != nil { - err := fmt.Errorf("Error describing instance for spot request tags: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - describeOutput, err := ec2conn.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: []*string{aws.String(instanceId)}, - }) sir := describeOutput.Reservations[0].Instances[0].SpotInstanceRequestId // Apply tags to the spot request. @@ -343,22 +346,6 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) } } - // Set the instance ID so that the cleanup works properly - s.instanceId = instanceId - - ui.Message(fmt.Sprintf("Instance ID: %s", instanceId)) - - r, err := ec2conn.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: []*string{aws.String(instanceId)}, - }) - if err != nil || len(r.Reservations) == 0 || len(r.Reservations[0].Instances) == 0 { - err := fmt.Errorf("Error finding source instance.") - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - instance := r.Reservations[0].Instances[0] - // Retry creating tags for about 2.5 minutes err = retry.Config{ Tries: 11,