From 36e4eaff991eb0bcb9bb6bd5c2244ce670a8768d Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 11 Jun 2019 12:20:00 +0200 Subject: [PATCH 01/11] document retry.Backoff better --- common/retry/retry.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/common/retry/retry.go b/common/retry/retry.go index 2b1a4f231..3189c4c97 100644 --- a/common/retry/retry.go +++ b/common/retry/retry.go @@ -76,12 +76,23 @@ func (cfg Config) Run(ctx context.Context, fn func(context.Context) error) error } } +// Backoff is a self contained backoff time calculator. This struct should be +// passed around as a copy as it changes its own fields upon any Backoff call. +// Backoff is not thread safe. For now only a Linear backoff call is +// implemented and the Exponential call will be implemented when needed. type Backoff struct { + // Initial time to wait. A Backoff call will change this value. InitialBackoff time.Duration - MaxBackoff time.Duration - Multiplier float64 + // Maximum time returned. + MaxBackoff time.Duration + // For a Linear backoff, InitialBackoff will be multiplied by Multiplier + // after each call. + Multiplier float64 } +// Linear Backoff returns a linearly increasing Duration. +// n = n * Multiplier. +// the first value of n is InitialBackoff. n is maxed by MaxBackoff. func (lb *Backoff) Linear() time.Duration { wait := lb.InitialBackoff lb.InitialBackoff = time.Duration(lb.Multiplier * float64(lb.InitialBackoff)) @@ -90,3 +101,8 @@ func (lb *Backoff) Linear() time.Duration { } return wait } + +// Exponential backoff panics: not implemented, yet. +func (lb *Backoff) Exponential() time.Duration { + panic("not implemented, yet") +} From 98206d59d764025906589c3eb0ba644fd7688a56 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 11 Jun 2019 12:37:52 +0200 Subject: [PATCH 02/11] aws: step_create_tags make the max waiting time 30s and not 30ns --- builder/amazon/common/step_create_tags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/common/step_create_tags.go b/builder/amazon/common/step_create_tags.go index 337bc1daa..68adc9878 100644 --- a/builder/amazon/common/step_create_tags.go +++ b/builder/amazon/common/step_create_tags.go @@ -102,7 +102,7 @@ func (s *StepCreateTags) Run(ctx context.Context, state multistep.StateBag) mult } return false }, - RetryDelay: (&retry.Backoff{InitialBackoff: 200 * time.Millisecond, MaxBackoff: 30, Multiplier: 2}).Linear, + RetryDelay: (&retry.Backoff{InitialBackoff: 200 * time.Millisecond, MaxBackoff: 30 * time.Second, Multiplier: 2}).Linear, }.Run(ctx, func(ctx context.Context) error { // Tag images and snapshots From 39cfacd5fa6a7317a4c9f8658a1753812cc48bc0 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 11 Jun 2019 12:41:21 +0200 Subject: [PATCH 03/11] Backoff.Linear: panic when InitialBackoff > MaxBackoff this probably means there's a configuration issue. Since this struct is mainly set manually from code, I think it is okay to panic here. --- common/retry/retry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/retry/retry.go b/common/retry/retry.go index 3189c4c97..38e2ba88d 100644 --- a/common/retry/retry.go +++ b/common/retry/retry.go @@ -94,6 +94,9 @@ type Backoff struct { // n = n * Multiplier. // the first value of n is InitialBackoff. n is maxed by MaxBackoff. func (lb *Backoff) Linear() time.Duration { + if lb.InitialBackoff > lb.MaxBackoff { + panic("InitialBackoff > MaxBackoff, did you forgot setting the seconds ?") + } wait := lb.InitialBackoff lb.InitialBackoff = time.Duration(lb.Multiplier * float64(lb.InitialBackoff)) if lb.MaxBackoff != 0 && lb.InitialBackoff > lb.MaxBackoff { From ca33f8bc5c23aea3e5025001b0e179b88c8eccc7 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 11 Jun 2019 12:53:06 +0200 Subject: [PATCH 04/11] Revert "Backoff.Linear: panic when InitialBackoff > MaxBackoff" This reverts commit 39cfacd5fa6a7317a4c9f8658a1753812cc48bc0. --- common/retry/retry.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/common/retry/retry.go b/common/retry/retry.go index 38e2ba88d..3189c4c97 100644 --- a/common/retry/retry.go +++ b/common/retry/retry.go @@ -94,9 +94,6 @@ type Backoff struct { // n = n * Multiplier. // the first value of n is InitialBackoff. n is maxed by MaxBackoff. func (lb *Backoff) Linear() time.Duration { - if lb.InitialBackoff > lb.MaxBackoff { - panic("InitialBackoff > MaxBackoff, did you forgot setting the seconds ?") - } wait := lb.InitialBackoff lb.InitialBackoff = time.Duration(lb.Multiplier * float64(lb.InitialBackoff)) if lb.MaxBackoff != 0 && lb.InitialBackoff > lb.MaxBackoff { From df977926ba769fe4cc764042317ac0a123ed6e5a Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Wed, 12 Jun 2019 09:50:27 -0700 Subject: [PATCH 05/11] filter machine readable UI --- packer/ui.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packer/ui.go b/packer/ui.go index 0492ab314..4c0b0857a 100644 --- a/packer/ui.go +++ b/packer/ui.go @@ -336,6 +336,12 @@ func (u *MachineReadableUi) Machine(category string, args ...string) { args[i] = strings.Replace(v, ",", "%!(PACKER_COMMA)", -1) args[i] = strings.Replace(args[i], "\r", "\\r", -1) args[i] = strings.Replace(args[i], "\n", "\\n", -1) + // Use LogSecretFilter to scrub out sensitive variables + for s := range LogSecretFilter.s { + if s != "" { + args[i] = strings.Replace(args[i], s, "", -1) + } + } } argsString := strings.Join(args, ",") From 6a5db1e948a9b0b160119545e719cd6fd467eae9 Mon Sep 17 00:00:00 2001 From: "bozhi.ch" Date: Thu, 13 Jun 2019 11:17:39 +0800 Subject: [PATCH 06/11] cleanup image and snapshot if target image is still not available after timeout --- builder/alicloud/ecs/client.go | 14 ++++++++++---- builder/alicloud/ecs/step_create_image.go | 9 +++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/builder/alicloud/ecs/client.go b/builder/alicloud/ecs/client.go index ac599049f..c18a1585c 100644 --- a/builder/alicloud/ecs/client.go +++ b/builder/alicloud/ecs/client.go @@ -161,6 +161,7 @@ func (c *ClientWrapper) WaitForExpected(args *WaitForExpectArgs) (responses.AcsR timeoutPoint = time.Now().Add(args.RetryTimeout) } + var lastResponse responses.AcsResponse var lastError error for i := 0; ; i++ { @@ -173,6 +174,7 @@ func (c *ClientWrapper) WaitForExpected(args *WaitForExpectArgs) (responses.AcsR } response, err := args.RequestFunc() + lastResponse = response lastError = err evalResult := args.EvalFunc(response, err) @@ -180,17 +182,21 @@ func (c *ClientWrapper) WaitForExpected(args *WaitForExpectArgs) (responses.AcsR return response, nil } if evalResult.stopRetry { - return nil, err + return response, err } time.Sleep(args.RetryInterval) } - if args.RetryTimeout > 0 { - return nil, fmt.Errorf("evaluate failed after %d seconds timeout with %d seconds retry interval: %s", int(args.RetryTimeout.Seconds()), int(args.RetryInterval.Seconds()), lastError) + if lastError == nil { + lastError = fmt.Errorf("") } - return nil, fmt.Errorf("evaluate failed after %d times retry with %d seconds retry interval: %s", args.RetryTimes, int(args.RetryInterval.Seconds()), lastError) + if args.RetryTimeout > 0 { + return lastResponse, fmt.Errorf("evaluate failed after %d seconds timeout with %d seconds retry interval: %s", int(args.RetryTimeout.Seconds()), int(args.RetryInterval.Seconds()), lastError) + } + + return lastResponse, fmt.Errorf("evaluate failed after %d times retry with %d seconds retry interval: %s", args.RetryTimes, int(args.RetryInterval.Seconds()), lastError) } func (c *ClientWrapper) WaitForInstanceStatus(regionId string, instanceId string, expectedStatus string) (responses.AcsResponse, error) { diff --git a/builder/alicloud/ecs/step_create_image.go b/builder/alicloud/ecs/step_create_image.go index 6b460d35b..f69842733 100644 --- a/builder/alicloud/ecs/step_create_image.go +++ b/builder/alicloud/ecs/step_create_image.go @@ -52,17 +52,18 @@ func (s *stepCreateAlicloudImage) Run(ctx context.Context, state multistep.State imageId := createImageResponse.(*ecs.CreateImageResponse).ImageId imagesResponse, err := client.WaitForImageStatus(config.AlicloudRegion, imageId, ImageStatusAvailable, time.Duration(s.WaitSnapshotReadyTimeout)*time.Second) - if err != nil { - return halt(state, err, "Timeout waiting for image to be created") - } + // save image first for cleaning up if timeout images := imagesResponse.(*ecs.DescribeImagesResponse).Images.Image if len(images) == 0 { return halt(state, err, "Unable to find created image") } - s.image = &images[0] + if err != nil { + return halt(state, err, "Timeout waiting for image to be created") + } + var snapshotIds []string for _, device := range images[0].DiskDeviceMappings.DiskDeviceMapping { snapshotIds = append(snapshotIds, device.SnapshotId) From 6982ec796f0fb22554c06ab0eabbe792fa9d467d Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Wed, 12 Jun 2019 13:18:57 -0700 Subject: [PATCH 07/11] remove redundant error check --- provisioner/shell-local/provisioner.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/provisioner/shell-local/provisioner.go b/provisioner/shell-local/provisioner.go index 861e98427..9768f85b8 100644 --- a/provisioner/shell-local/provisioner.go +++ b/provisioner/shell-local/provisioner.go @@ -27,11 +27,8 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { func (p *Provisioner) Provision(ctx context.Context, ui packer.Ui, _ packer.Communicator) error { _, retErr := sl.Run(ctx, ui, &p.config) - if retErr != nil { - return retErr - } - return nil + return retErr } func (p *Provisioner) Cancel() { From 88e5a21170d0aae3454291e34c24d941b1af561c Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Thu, 13 Jun 2019 10:48:13 -0700 Subject: [PATCH 08/11] make sure machine readable logs print what's come through the UI into the logs --- packer/ui.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packer/ui.go b/packer/ui.go index 9ba44518b..cc4dd0a7e 100644 --- a/packer/ui.go +++ b/packer/ui.go @@ -350,6 +350,7 @@ func (u *MachineReadableUi) Machine(category string, args ...string) { panic(err) } } + log.Printf("%d,%s,%s,%s\n", now.Unix(), target, category, argsString) } // TimestampedUi is a UI that wraps another UI implementation and From b3277698f6cfbe5d78743bfc0ecb81c00f9ae4b7 Mon Sep 17 00:00:00 2001 From: "bozhi.ch" Date: Fri, 14 Jun 2019 11:49:42 +0800 Subject: [PATCH 09/11] let product API determine the default value of io_optimized --- builder/alicloud/ecs/run_config.go | 2 +- builder/alicloud/ecs/step_create_instance.go | 12 +++++++----- website/source/docs/builders/alicloud-ecs.html.md | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/builder/alicloud/ecs/run_config.go b/builder/alicloud/ecs/run_config.go index e36a33854..ce4eda5b0 100644 --- a/builder/alicloud/ecs/run_config.go +++ b/builder/alicloud/ecs/run_config.go @@ -14,7 +14,7 @@ import ( type RunConfig struct { AssociatePublicIpAddress bool `mapstructure:"associate_public_ip_address"` ZoneId string `mapstructure:"zone_id"` - IOOptimized bool `mapstructure:"io_optimized"` + IOOptimized *bool `mapstructure:"io_optimized"` InstanceType string `mapstructure:"instance_type"` Description string `mapstructure:"description"` AlicloudSourceImage string `mapstructure:"source_image"` diff --git a/builder/alicloud/ecs/step_create_instance.go b/builder/alicloud/ecs/step_create_instance.go index 512ea5c98..ad3854751 100644 --- a/builder/alicloud/ecs/step_create_instance.go +++ b/builder/alicloud/ecs/step_create_instance.go @@ -17,7 +17,7 @@ import ( ) type stepCreateAlicloudInstance struct { - IOOptimized bool + IOOptimized *bool InstanceType string UserData string UserDataFile string @@ -142,11 +142,13 @@ func (s *stepCreateAlicloudInstance) buildCreateInstanceRequest(state multistep. request.InternetChargeType = s.InternetChargeType request.InternetMaxBandwidthOut = requests.Integer(convertNumber(s.InternetMaxBandwidthOut)) - ioOptimized := IOOptimizedNone - if s.IOOptimized { - ioOptimized = IOOptimizedOptimized + if s.IOOptimized != nil { + if *s.IOOptimized { + request.IoOptimized = IOOptimizedOptimized + } else { + request.IoOptimized = IOOptimizedNone + } } - request.IoOptimized = ioOptimized config := state.Get("config").(*Config) password := config.Comm.SSHPassword diff --git a/website/source/docs/builders/alicloud-ecs.html.md b/website/source/docs/builders/alicloud-ecs.html.md index 22839d032..ef113f487 100644 --- a/website/source/docs/builders/alicloud-ecs.html.md +++ b/website/source/docs/builders/alicloud-ecs.html.md @@ -218,7 +218,8 @@ builder. error is returned. - `io_optimized` (boolean) - Whether an ECS instance is I/O optimized or not. - The default value is `false`. + If this option is not provided, the value will be determined by product API + according to what `instance_type` is used. - `security_group_id` (string) - ID of the security group to which a newly created instance belongs. Mutual access is allowed between instances in one From d8b91cd7a5c4eea27015c09988ff7c807b44b83b Mon Sep 17 00:00:00 2001 From: wrolisonaflac <47217958+wrolisonaflac@users.noreply.github.com> Date: Fri, 14 Jun 2019 07:54:00 -0400 Subject: [PATCH 10/11] Update ncloud.html.md Fixed blog link at cloudwindows --- website/source/docs/builders/ncloud.html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/builders/ncloud.html.md b/website/source/docs/builders/ncloud.html.md index 32199cf6e..154fc1c8d 100644 --- a/website/source/docs/builders/ncloud.html.md +++ b/website/source/docs/builders/ncloud.html.md @@ -91,7 +91,7 @@ Here is a basic example for windows server. } -> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: -https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ +https://cloudywindows.io/post/winrm-for-provisioning-close-the-door-on-the-way-out-eh/ Here is a basic example for linux server. From 065fee86a8d2cea7c809dfcee5f263a8d59a367a Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Fri, 14 Jun 2019 15:08:59 +0200 Subject: [PATCH 11/11] fix more urls --- website/source/community-tools.html.md | 2 +- website/source/docs/builders/hyperv-iso.html.md.erb | 2 +- website/source/docs/builders/hyperv-vmcx.html.md.erb | 2 +- website/source/docs/provisioners/ansible.html.md.erb | 2 +- website/source/intro/getting-started/build-image.html.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/website/source/community-tools.html.md b/website/source/community-tools.html.md index 644775cd8..027ad6d26 100644 --- a/website/source/community-tools.html.md +++ b/website/source/community-tools.html.md @@ -54,4 +54,4 @@ power of Packer templates. ## Other - [suitcase](https://github.com/tmclaugh/suitcase) - Packer based build system for CentOS OS images -- [Undo-WinRMConfig](https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/) - Open source automation to stage WinRM reset to pristine state at next shtudown +- [Undo-WinRMConfig](https://cloudywindows.io/post/winrm-for-provisioning-close-the-door-on-the-way-out-eh/) - Open source automation to stage WinRM reset to pristine state at next shtudown diff --git a/website/source/docs/builders/hyperv-iso.html.md.erb b/website/source/docs/builders/hyperv-iso.html.md.erb index 72693ce7a..716285d5c 100644 --- a/website/source/docs/builders/hyperv-iso.html.md.erb +++ b/website/source/docs/builders/hyperv-iso.html.md.erb @@ -877,7 +877,7 @@ Finish proxy after sysprep --> ``` -> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: -https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ +https://cloudywindows.io/post/winrm-for-provisioning-close-the-door-on-the-way-out-eh/ ## Example For Ubuntu Vivid Generation 2 diff --git a/website/source/docs/builders/hyperv-vmcx.html.md.erb b/website/source/docs/builders/hyperv-vmcx.html.md.erb index 34121076c..493a78109 100644 --- a/website/source/docs/builders/hyperv-vmcx.html.md.erb +++ b/website/source/docs/builders/hyperv-vmcx.html.md.erb @@ -894,7 +894,7 @@ Finish proxy after sysprep --> ``` -> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: -https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ +https://cloudywindows.io/post/winrm-for-provisioning-close-the-door-on-the-way-out-eh/ ## Example For Ubuntu Vivid Generation 2 diff --git a/website/source/docs/provisioners/ansible.html.md.erb b/website/source/docs/provisioners/ansible.html.md.erb index be33fbf13..0e98a7601 100644 --- a/website/source/docs/provisioners/ansible.html.md.erb +++ b/website/source/docs/provisioners/ansible.html.md.erb @@ -325,7 +325,7 @@ Platform: ``` -> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: -https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ +https://cloudywindows.io/post/winrm-for-provisioning-close-the-door-on-the-way-out-eh/ ### Post i/o timeout errors diff --git a/website/source/intro/getting-started/build-image.html.md b/website/source/intro/getting-started/build-image.html.md index 672fe8dac..272d30ef8 100644 --- a/website/source/intro/getting-started/build-image.html.md +++ b/website/source/intro/getting-started/build-image.html.md @@ -405,7 +405,7 @@ Start-Service -Name WinRM ``` -> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: -https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ +https://cloudywindows.io/post/winrm-for-provisioning-close-the-door-on-the-way-out-eh/ Save the above code in a file named `bootstrap_win.txt`.