From 3c2c13a02107999736cbf24ddb99c7f586e8488c Mon Sep 17 00:00:00 2001 From: "bozhi.ch" Date: Mon, 19 Nov 2018 15:25:12 +0800 Subject: [PATCH 1/2] support creating image without data disks --- builder/alicloud/ecs/builder.go | 10 +- builder/alicloud/ecs/builder_test.go | 43 +++++++++ builder/alicloud/ecs/image_config.go | 1 + builder/alicloud/ecs/packer_helper.go | 9 ++ builder/alicloud/ecs/step_create_image.go | 27 ++++-- builder/alicloud/ecs/step_create_snapshot.go | 92 +++++++++++++++++++ .../source/docs/builders/alicloud-ecs.html.md | 5 + 7 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 builder/alicloud/ecs/step_create_snapshot.go diff --git a/builder/alicloud/ecs/builder.go b/builder/alicloud/ecs/builder.go index ea317c819..af339c718 100644 --- a/builder/alicloud/ecs/builder.go +++ b/builder/alicloud/ecs/builder.go @@ -169,8 +169,16 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe AlicloudImageForceDeleteSnapshots: b.config.AlicloudImageForceDeleteSnapshots, AlicloudImageForceDelete: b.config.AlicloudImageForceDelete, AlicloudImageName: b.config.AlicloudImageName, + }) + + if b.config.AlicloudImageIgnoreDataDisks { + steps = append(steps, &stepCreateAlicloudSnapshot{}) + } + + steps = append(steps, + &stepCreateAlicloudImage{ + AlicloudImageIgnoreDataDisks: b.config.AlicloudImageIgnoreDataDisks, }, - &stepCreateAlicloudImage{}, &stepCreateTags{ Tags: b.config.AlicloudImageTags, }, diff --git a/builder/alicloud/ecs/builder_test.go b/builder/alicloud/ecs/builder_test.go index 4dab95fd5..153291f73 100644 --- a/builder/alicloud/ecs/builder_test.go +++ b/builder/alicloud/ecs/builder_test.go @@ -152,3 +152,46 @@ func TestBuilderPrepare_Devices(t *testing.T) { t.Fatalf("data disks are not set properly, actual: %#v", b.config.ECSImagesDiskMappings) } } + +func TestBuilderPrepare_IgnoreDataDisks(t *testing.T) { + var b Builder + config := testBuilderConfig() + + warnings, err := b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + if b.config.AlicloudImageIgnoreDataDisks != false { + t.Fatalf("image_ignore_data_disks is not set properly, expect: %t, actual: %t", false, b.config.AlicloudImageIgnoreDataDisks) + } + + config["image_ignore_data_disks"] = "false" + warnings, err = b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + if b.config.AlicloudImageIgnoreDataDisks != false { + t.Fatalf("image_ignore_data_disks is not set properly, expect: %t, actual: %t", false, b.config.AlicloudImageIgnoreDataDisks) + } + + config["image_ignore_data_disks"] = "true" + warnings, err = b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + if b.config.AlicloudImageIgnoreDataDisks != true { + t.Fatalf("image_ignore_data_disks is not set properly, expect: %t, actual: %t", true, b.config.AlicloudImageIgnoreDataDisks) + } +} diff --git a/builder/alicloud/ecs/image_config.go b/builder/alicloud/ecs/image_config.go index ae5db8abc..c9fa7d5b1 100644 --- a/builder/alicloud/ecs/image_config.go +++ b/builder/alicloud/ecs/image_config.go @@ -36,6 +36,7 @@ type AlicloudImageConfig struct { AlicloudImageForceDelete bool `mapstructure:"image_force_delete"` AlicloudImageForceDeleteSnapshots bool `mapstructure:"image_force_delete_snapshots"` AlicloudImageForceDeleteInstances bool `mapstructure:"image_force_delete_instances"` + AlicloudImageIgnoreDataDisks bool `mapstructure:"image_ignore_data_disks"` AlicloudImageSkipRegionValidation bool `mapstructure:"skip_region_validation"` AlicloudImageTags map[string]string `mapstructure:"tags"` AlicloudDiskDevices `mapstructure:",squash"` diff --git a/builder/alicloud/ecs/packer_helper.go b/builder/alicloud/ecs/packer_helper.go index 15f257381..4e20d40ac 100644 --- a/builder/alicloud/ecs/packer_helper.go +++ b/builder/alicloud/ecs/packer_helper.go @@ -20,3 +20,12 @@ func message(state multistep.StateBag, module string) { } } + +func halt(state multistep.StateBag, err error, prefix string) multistep.StepAction { + ui := state.Get("ui").(packer.Ui) + + err = fmt.Errorf("%s: %s", prefix, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt +} diff --git a/builder/alicloud/ecs/step_create_image.go b/builder/alicloud/ecs/step_create_image.go index fedbfb272..fafb5819d 100644 --- a/builder/alicloud/ecs/step_create_image.go +++ b/builder/alicloud/ecs/step_create_image.go @@ -11,7 +11,8 @@ import ( ) type stepCreateAlicloudImage struct { - image *ecs.ImageType + AlicloudImageIgnoreDataDisks bool + image *ecs.ImageType } func (s *stepCreateAlicloudImage) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { @@ -24,13 +25,23 @@ func (s *stepCreateAlicloudImage) Run(_ context.Context, state multistep.StateBa var imageId string var err error - instance := state.Get("instance").(*ecs.InstanceAttributesType) - imageId, err = client.CreateImage(&ecs.CreateImageArgs{ - RegionId: common.Region(config.AlicloudRegion), - InstanceId: instance.InstanceId, - ImageName: config.AlicloudImageName, - ImageVersion: config.AlicloudImageVersion, - Description: config.AlicloudImageDescription}) + if s.AlicloudImageIgnoreDataDisks { + snapshotId := state.Get("alicloudsnapshot").(string) + imageId, err = client.CreateImage(&ecs.CreateImageArgs{ + RegionId: common.Region(config.AlicloudRegion), + SnapshotId: snapshotId, + ImageName: config.AlicloudImageName, + ImageVersion: config.AlicloudImageVersion, + Description: config.AlicloudImageDescription}) + } else { + instance := state.Get("instance").(*ecs.InstanceAttributesType) + imageId, err = client.CreateImage(&ecs.CreateImageArgs{ + RegionId: common.Region(config.AlicloudRegion), + InstanceId: instance.InstanceId, + ImageName: config.AlicloudImageName, + ImageVersion: config.AlicloudImageVersion, + Description: config.AlicloudImageDescription}) + } if err != nil { err := fmt.Errorf("Error creating image: %s", err) diff --git a/builder/alicloud/ecs/step_create_snapshot.go b/builder/alicloud/ecs/step_create_snapshot.go new file mode 100644 index 000000000..875db37b8 --- /dev/null +++ b/builder/alicloud/ecs/step_create_snapshot.go @@ -0,0 +1,92 @@ +package ecs + +import ( + "context" + "fmt" + + "github.com/denverdino/aliyungo/common" + "github.com/denverdino/aliyungo/ecs" + "github.com/hashicorp/packer/helper/multistep" + "github.com/hashicorp/packer/packer" +) + +type stepCreateAlicloudSnapshot struct { + snapshot *ecs.SnapshotType +} + +func (s *stepCreateAlicloudSnapshot) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(*Config) + client := state.Get("client").(*ecs.Client) + ui := state.Get("ui").(packer.Ui) + + instance := state.Get("instance").(*ecs.InstanceAttributesType) + disks, _, err := client.DescribeDisks(&ecs.DescribeDisksArgs{ + RegionId: common.Region(config.AlicloudRegion), + InstanceId: instance.InstanceId, + DiskType: ecs.DiskTypeAllSystem, + }) + + if err != nil { + return halt(state, err, "Error describe disks") + } + if len(disks) == 0 { + return halt(state, err, "Unable to find system disk of instance") + } + + // Create the alicloud snapshot + ui.Say(fmt.Sprintf("Creating snapshot from system disk: %s", disks[0].DiskId)) + + snapshotId, err := client.CreateSnapshot(&ecs.CreateSnapshotArgs{ + DiskId: disks[0].DiskId, + }) + + if err != nil { + return halt(state, err, "Error creating snapshot") + } + + err = client.WaitForSnapShotReady(common.Region(config.AlicloudRegion), + snapshotId, ALICLOUD_DEFAULT_LONG_TIMEOUT) + if err != nil { + return halt(state, err, "Timeout waiting for snapshot to be created") + } + + snapshots, _, err := client.DescribeSnapshots(&ecs.DescribeSnapshotsArgs{ + RegionId: common.Region(config.AlicloudRegion), + SnapshotIds: []string{snapshotId}, + }) + + if err != nil { + return halt(state, err, "Error querying created snapshot") + } + if len(snapshots) == 0 { + return halt(state, err, "Unable to find created snapshot") + } + s.snapshot = &snapshots[0] + + state.Put("alicloudsnapshot", snapshotId) + alicloudSnapshots := make(map[string]string) + alicloudSnapshots[config.AlicloudRegion] = snapshotId + state.Put("alicloudsnapshots", alicloudSnapshots) + + return multistep.ActionContinue +} + +func (s *stepCreateAlicloudSnapshot) Cleanup(state multistep.StateBag) { + if s.snapshot == nil { + return + } + _, cancelled := state.GetOk(multistep.StateCancelled) + _, halted := state.GetOk(multistep.StateHalted) + if !cancelled && !halted { + return + } + + client := state.Get("client").(*ecs.Client) + ui := state.Get("ui").(packer.Ui) + + ui.Say("Deleting the snapshot because of cancellation or error...") + if err := client.DeleteSnapshot(s.snapshot.SnapshotId); err != nil { + ui.Error(fmt.Sprintf("Error deleting snapshot, it may still be around: %s", err)) + return + } +} diff --git a/website/source/docs/builders/alicloud-ecs.html.md b/website/source/docs/builders/alicloud-ecs.html.md index 97846a048..10766bb76 100644 --- a/website/source/docs/builders/alicloud-ecs.html.md +++ b/website/source/docs/builders/alicloud-ecs.html.md @@ -147,6 +147,11 @@ builder. Snapshots from on or before July 15, 2013 cannot be used to create a disk. + +- `image_ignore_data_disks`(boolean) - If this value is true, the image created + will not include any snapshot of data disks. This option would be useful for + any circumstance that default data disks with instance types are not concerned. + The default value is false. - `image_force_delete` (boolean) - If this value is true, when the target image name is duplicated with an existing image, it will delete the From 3c1395eb04eaadd6d6152a0452efc235c084b112 Mon Sep 17 00:00:00 2001 From: "bozhi.ch" Date: Sun, 25 Nov 2018 15:46:03 +0800 Subject: [PATCH 2/2] support wait_snapshot_ready_timeout for much bigger disk --- builder/alicloud/ecs/builder.go | 13 ++++++- builder/alicloud/ecs/builder_test.go | 37 +++++++++++++++++++ builder/alicloud/ecs/run_config.go | 1 + builder/alicloud/ecs/step_create_image.go | 4 +- builder/alicloud/ecs/step_create_snapshot.go | 6 +-- .../source/docs/builders/alicloud-ecs.html.md | 4 ++ 6 files changed, 59 insertions(+), 6 deletions(-) diff --git a/builder/alicloud/ecs/builder.go b/builder/alicloud/ecs/builder.go index af339c718..46e6889fc 100644 --- a/builder/alicloud/ecs/builder.go +++ b/builder/alicloud/ecs/builder.go @@ -172,12 +172,15 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }) if b.config.AlicloudImageIgnoreDataDisks { - steps = append(steps, &stepCreateAlicloudSnapshot{}) + steps = append(steps, &stepCreateAlicloudSnapshot{ + WaitSnapshotReadyTimeout: b.getSnapshotReadyTimeout(), + }) } steps = append(steps, &stepCreateAlicloudImage{ AlicloudImageIgnoreDataDisks: b.config.AlicloudImageIgnoreDataDisks, + WaitSnapshotReadyTimeout: b.getSnapshotReadyTimeout(), }, &stepCreateTags{ Tags: b.config.AlicloudImageTags, @@ -253,3 +256,11 @@ func (b *Builder) isUserDataNeeded() bool { func (b *Builder) isKeyPairNeeded() bool { return b.config.Comm.SSHKeyPairName != "" || b.config.Comm.SSHTemporaryKeyPairName != "" } + +func (b *Builder) getSnapshotReadyTimeout() int { + if b.config.WaitSnapshotReadyTimeout > 0 { + return b.config.WaitSnapshotReadyTimeout + } + + return ALICLOUD_DEFAULT_LONG_TIMEOUT +} diff --git a/builder/alicloud/ecs/builder_test.go b/builder/alicloud/ecs/builder_test.go index 153291f73..c6e54d793 100644 --- a/builder/alicloud/ecs/builder_test.go +++ b/builder/alicloud/ecs/builder_test.go @@ -195,3 +195,40 @@ func TestBuilderPrepare_IgnoreDataDisks(t *testing.T) { t.Fatalf("image_ignore_data_disks is not set properly, expect: %t, actual: %t", true, b.config.AlicloudImageIgnoreDataDisks) } } + +func TestBuilderPrepare_WaitSnapshotReadyTimeout(t *testing.T) { + var b Builder + config := testBuilderConfig() + + warnings, err := b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + if b.config.WaitSnapshotReadyTimeout != 0 { + t.Fatalf("wait_snapshot_ready_timeout is not set properly, expect: %d, actual: %d", 0, b.config.WaitSnapshotReadyTimeout) + } + if b.getSnapshotReadyTimeout() != ALICLOUD_DEFAULT_LONG_TIMEOUT { + t.Fatalf("default timeout is not set properly, expect: %d, actual: %d", ALICLOUD_DEFAULT_LONG_TIMEOUT, b.getSnapshotReadyTimeout()) + } + + config["wait_snapshot_ready_timeout"] = ALICLOUD_DEFAULT_TIMEOUT + warnings, err = b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + if b.config.WaitSnapshotReadyTimeout != ALICLOUD_DEFAULT_TIMEOUT { + t.Fatalf("wait_snapshot_ready_timeout is not set properly, expect: %d, actual: %d", ALICLOUD_DEFAULT_TIMEOUT, b.config.WaitSnapshotReadyTimeout) + } + + if b.getSnapshotReadyTimeout() != ALICLOUD_DEFAULT_TIMEOUT { + t.Fatalf("default timeout is not set properly, expect: %d, actual: %d", ALICLOUD_DEFAULT_TIMEOUT, b.getSnapshotReadyTimeout()) + } +} diff --git a/builder/alicloud/ecs/run_config.go b/builder/alicloud/ecs/run_config.go index aaab9f785..e36a33854 100644 --- a/builder/alicloud/ecs/run_config.go +++ b/builder/alicloud/ecs/run_config.go @@ -32,6 +32,7 @@ type RunConfig struct { InstanceName string `mapstructure:"instance_name"` InternetChargeType string `mapstructure:"internet_charge_type"` InternetMaxBandwidthOut int `mapstructure:"internet_max_bandwidth_out"` + WaitSnapshotReadyTimeout int `mapstructure:"wait_snapshot_ready_timeout"` // Communicator settings Comm communicator.Config `mapstructure:",squash"` diff --git a/builder/alicloud/ecs/step_create_image.go b/builder/alicloud/ecs/step_create_image.go index fafb5819d..d80dca8ea 100644 --- a/builder/alicloud/ecs/step_create_image.go +++ b/builder/alicloud/ecs/step_create_image.go @@ -12,6 +12,7 @@ import ( type stepCreateAlicloudImage struct { AlicloudImageIgnoreDataDisks bool + WaitSnapshotReadyTimeout int image *ecs.ImageType } @@ -49,8 +50,7 @@ func (s *stepCreateAlicloudImage) Run(_ context.Context, state multistep.StateBa ui.Error(err.Error()) return multistep.ActionHalt } - err = client.WaitForImageReady(common.Region(config.AlicloudRegion), - imageId, ALICLOUD_DEFAULT_LONG_TIMEOUT) + err = client.WaitForImageReady(common.Region(config.AlicloudRegion), imageId, s.WaitSnapshotReadyTimeout) if err != nil { err := fmt.Errorf("Timeout waiting for image to be created: %s", err) state.Put("error", err) diff --git a/builder/alicloud/ecs/step_create_snapshot.go b/builder/alicloud/ecs/step_create_snapshot.go index 875db37b8..a9f5ae590 100644 --- a/builder/alicloud/ecs/step_create_snapshot.go +++ b/builder/alicloud/ecs/step_create_snapshot.go @@ -11,7 +11,8 @@ import ( ) type stepCreateAlicloudSnapshot struct { - snapshot *ecs.SnapshotType + snapshot *ecs.SnapshotType + WaitSnapshotReadyTimeout int } func (s *stepCreateAlicloudSnapshot) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { @@ -44,8 +45,7 @@ func (s *stepCreateAlicloudSnapshot) Run(_ context.Context, state multistep.Stat return halt(state, err, "Error creating snapshot") } - err = client.WaitForSnapShotReady(common.Region(config.AlicloudRegion), - snapshotId, ALICLOUD_DEFAULT_LONG_TIMEOUT) + err = client.WaitForSnapShotReady(common.Region(config.AlicloudRegion), snapshotId, s.WaitSnapshotReadyTimeout) if err != nil { return halt(state, err, "Timeout waiting for snapshot to be created") } diff --git a/website/source/docs/builders/alicloud-ecs.html.md b/website/source/docs/builders/alicloud-ecs.html.md index 10766bb76..0c743ca4b 100644 --- a/website/source/docs/builders/alicloud-ecs.html.md +++ b/website/source/docs/builders/alicloud-ecs.html.md @@ -153,6 +153,10 @@ builder. any circumstance that default data disks with instance types are not concerned. The default value is false. +- `wait_snapshot_ready_timeout`(number) - Timeout of creating snapshot(s). The + default timeout is 3600 seconds if this option is not set or is set to 0. For + those disks containing lots of data, it may require a higher timeout value. + - `image_force_delete` (boolean) - If this value is true, when the target image name is duplicated with an existing image, it will delete the existing image and then create the target image, otherwise, the creation