diff --git a/builder/amazon/common/access_config_test.go b/builder/amazon/common/access_config_test.go index 0e2d4b4fa..e1eeaa05f 100644 --- a/builder/amazon/common/access_config_test.go +++ b/builder/amazon/common/access_config_test.go @@ -5,20 +5,10 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" ) -func testAccessConfig() *AccessConfig { - return &AccessConfig{ - getEC2Connection: func() ec2iface.EC2API { - return &mockEC2Client{} - }, - PollingConfig: new(AWSPollingConfig), - } -} - func TestAccessConfigPrepare_Region(t *testing.T) { - c := testAccessConfig() + c := FakeAccessConfig() c.RawRegion = "us-east-12" err := c.ValidateRegion(c.RawRegion) @@ -40,7 +30,7 @@ func TestAccessConfigPrepare_Region(t *testing.T) { } func TestAccessConfigPrepare_RegionRestricted(t *testing.T) { - c := testAccessConfig() + c := FakeAccessConfig() // Create a Session with a custom region c.session = session.Must(session.NewSession(&aws.Config{ diff --git a/builder/amazon/common/ami_config.go b/builder/amazon/common/ami_config.go index d9596eb22..36b9ab571 100644 --- a/builder/amazon/common/ami_config.go +++ b/builder/amazon/common/ami_config.go @@ -135,26 +135,8 @@ type AMIConfig struct { // the intermediary AMI into any regions provided in `ami_regions`, then // delete the intermediary AMI. Default `false`. AMISkipBuildRegion bool `mapstructure:"skip_save_build_region"` - // Key/value pair tags to apply to snapshot. They will override AMI tags if - // already applied to snapshot. This is a [template - // engine](/docs/templates/legacy_json_templates/engine), see [Build template - // data](#build-template-data) for more information. - SnapshotTags map[string]string `mapstructure:"snapshot_tags" required:"false"` - // Same as [`snapshot_tags`](#snapshot_tags) but defined as a singular - // repeatable block containing a `key` and a `value` field. In HCL2 mode the - // [`dynamic_block`](/docs/templates/hcl_templates/expressions#dynamic-blocks) - // will allow you to create those programatically. - SnapshotTag config.KeyValues `mapstructure:"snapshot_tag" required:"false"` - // A list of account IDs that have - // access to create volumes from the snapshot(s). By default no additional - // users other than the user creating the AMI has permissions to create - // volumes from the backing snapshot(s). - SnapshotUsers []string `mapstructure:"snapshot_users" required:"false"` - // A list of groups that have access to - // create volumes from the snapshot(s). By default no groups have permission - // to create volumes from the snapshot(s). all will make the snapshot - // publicly accessible. - SnapshotGroups []string `mapstructure:"snapshot_groups" required:"false"` + + SnapshotConfig `mapstructure:",squash"` } func stringInSlice(s []string, searchstr string) bool { diff --git a/builder/amazon/common/ami_config_test.go b/builder/amazon/common/ami_config_test.go index ca2815589..1a3435868 100644 --- a/builder/amazon/common/ami_config_test.go +++ b/builder/amazon/common/ami_config_test.go @@ -7,7 +7,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/hashicorp/packer-plugin-sdk/template/config" ) @@ -18,14 +17,14 @@ func testAMIConfig() *AMIConfig { } func getFakeAccessConfig(region string) *AccessConfig { - c := testAccessConfig() + c := FakeAccessConfig() c.RawRegion = region return c } func TestAMIConfigPrepare_name(t *testing.T) { c := testAMIConfig() - accessConf := testAccessConfig() + accessConf := FakeAccessConfig() if err := c.Prepare(accessConf, nil); err != nil { t.Fatalf("shouldn't have err: %s", err) } @@ -36,10 +35,6 @@ func TestAMIConfigPrepare_name(t *testing.T) { } } -type mockEC2Client struct { - ec2iface.EC2API -} - func (m *mockEC2Client) DescribeRegions(*ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) { return &ec2.DescribeRegionsOutput{ Regions: []*ec2.Region{ @@ -56,7 +51,7 @@ func TestAMIConfigPrepare_regions(t *testing.T) { var errs []error var err error - accessConf := testAccessConfig() + accessConf := FakeAccessConfig() mockConn := &mockEC2Client{} if errs = c.prepareRegions(accessConf); len(errs) > 0 { t.Fatalf("shouldn't have err: %#v", errs) @@ -163,7 +158,7 @@ func TestAMIConfigPrepare_Share_EncryptedBoot(t *testing.T) { c.AMIUsers = []string{"testAccountID"} c.AMIEncryptBootVolume = config.TriTrue - accessConf := testAccessConfig() + accessConf := FakeAccessConfig() c.AMIKmsKeyId = "" if err := c.Prepare(accessConf, nil); err == nil { @@ -179,7 +174,7 @@ func TestAMIConfigPrepare_ValidateKmsKey(t *testing.T) { c := testAMIConfig() c.AMIEncryptBootVolume = config.TriTrue - accessConf := testAccessConfig() + accessConf := FakeAccessConfig() validCases := []string{ "abcd1234-e567-890f-a12b-a123b4cd56ef", @@ -215,7 +210,7 @@ func TestAMIConfigPrepare_ValidateKmsKey(t *testing.T) { func TestAMINameValidation(t *testing.T) { c := testAMIConfig() - accessConf := testAccessConfig() + accessConf := FakeAccessConfig() c.AMIName = "aa" if err := c.Prepare(accessConf, nil); err == nil { diff --git a/builder/amazon/common/snapshot_config.go b/builder/amazon/common/snapshot_config.go new file mode 100644 index 000000000..9e8065e39 --- /dev/null +++ b/builder/amazon/common/snapshot_config.go @@ -0,0 +1,29 @@ +//go:generate struct-markdown + +package common + +import "github.com/hashicorp/packer-plugin-sdk/template/config" + +// SnapshotConfig is for common configuration related to creating AMIs. +type SnapshotConfig struct { + // Key/value pair tags to apply to snapshot. They will override AMI tags if + // already applied to snapshot. This is a [template + // engine](/docs/templates/legacy_json_templates/engine), see [Build template + // data](#build-template-data) for more information. + SnapshotTags map[string]string `mapstructure:"snapshot_tags" required:"false"` + // Same as [`snapshot_tags`](#snapshot_tags) but defined as a singular + // repeatable block containing a `key` and a `value` field. In HCL2 mode the + // [`dynamic_block`](/docs/templates/hcl_templates/expressions#dynamic-blocks) + // will allow you to create those programatically. + SnapshotTag config.KeyValues `mapstructure:"snapshot_tag" required:"false"` + // A list of account IDs that have + // access to create volumes from the snapshot(s). By default no additional + // users other than the user creating the AMI has permissions to create + // volumes from the backing snapshot(s). + SnapshotUsers []string `mapstructure:"snapshot_users" required:"false"` + // A list of groups that have access to + // create volumes from the snapshot(s). By default no groups have permission + // to create volumes from the snapshot(s). all will make the snapshot + // publicly accessible. + SnapshotGroups []string `mapstructure:"snapshot_groups" required:"false"` +} diff --git a/builder/amazon/common/state.go b/builder/amazon/common/state.go index 1ceacef84..74010658e 100644 --- a/builder/amazon/common/state.go +++ b/builder/amazon/common/state.go @@ -149,15 +149,22 @@ func (w *AWSPollingConfig) WaitUntilVolumeAvailable(ctx aws.Context, conn *ec2.E return err } -func (w *AWSPollingConfig) WaitUntilSnapshotDone(ctx aws.Context, conn *ec2.EC2, snapshotID string) error { +func (w *AWSPollingConfig) WaitUntilSnapshotDone(ctx aws.Context, conn ec2iface.EC2API, snapshotID string) error { snapInput := ec2.DescribeSnapshotsInput{ SnapshotIds: []*string{&snapshotID}, } + waitOpts := w.getWaiterOptions() + if len(waitOpts) == 0 { + // Bump this default to 30 minutes. + // Large snapshots can take a long time for the copy to s3 + waitOpts = append(waitOpts, request.WithWaiterMaxAttempts(120)) + } + err := conn.WaitUntilSnapshotCompletedWithContext( ctx, &snapInput, - w.getWaiterOptions()...) + waitOpts...) return err } diff --git a/builder/amazon/common/step_ami_region_copy_test.go b/builder/amazon/common/step_ami_region_copy_test.go index 9f5e3fe5d..4c809b665 100644 --- a/builder/amazon/common/step_ami_region_copy_test.go +++ b/builder/amazon/common/step_ami_region_copy_test.go @@ -105,7 +105,7 @@ func TestStepAMIRegionCopy_duplicates(t *testing.T) { // ------------------------------------------------------------------------ stepAMIRegionCopy := StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: []string{"us-east-1"}, AMIKmsKeyId: "12345", // Original region key in regionkeyids is different than in amikmskeyid @@ -131,7 +131,7 @@ func TestStepAMIRegionCopy_duplicates(t *testing.T) { // the ami is only copied once. stepAMIRegionCopy = StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: []string{"us-east-1"}, Name: "fake-ami-name", OriginalRegion: "us-east-1", @@ -152,7 +152,7 @@ func TestStepAMIRegionCopy_duplicates(t *testing.T) { // the ami is only copied once. stepAMIRegionCopy = StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: []string{"us-east-1"}, EncryptBootVolume: config.TriFalse, Name: "fake-ami-name", @@ -174,7 +174,7 @@ func TestStepAMIRegionCopy_duplicates(t *testing.T) { // ------------------------------------------------------------------------ stepAMIRegionCopy = StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), // Many duplicates for only 3 actual values Regions: []string{"us-east-1", "us-west-2", "us-west-2", "ap-east-1", "ap-east-1", "ap-east-1"}, AMIKmsKeyId: "IlikePancakes", @@ -203,7 +203,7 @@ func TestStepAMIRegionCopy_duplicates(t *testing.T) { // ------------------------------------------------------------------------ stepAMIRegionCopy = StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), // Many duplicates for only 3 actual values Regions: []string{"us-east-1", "us-west-2", "us-west-2", "ap-east-1", "ap-east-1", "ap-east-1"}, Name: "fake-ami-name", @@ -223,7 +223,7 @@ func TestStepAMIRegionCopy_duplicates(t *testing.T) { func TestStepAmiRegionCopy_nil_encryption(t *testing.T) { // create step stepAMIRegionCopy := StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: make([]string, 0), AMIKmsKeyId: "", RegionKeyIds: make(map[string]string), @@ -249,7 +249,7 @@ func TestStepAmiRegionCopy_nil_encryption(t *testing.T) { func TestStepAmiRegionCopy_true_encryption(t *testing.T) { // create step stepAMIRegionCopy := StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: make([]string, 0), AMIKmsKeyId: "", RegionKeyIds: make(map[string]string), @@ -275,7 +275,7 @@ func TestStepAmiRegionCopy_true_encryption(t *testing.T) { func TestStepAmiRegionCopy_nil_intermediary(t *testing.T) { // create step stepAMIRegionCopy := StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: make([]string, 0), AMIKmsKeyId: "", RegionKeyIds: make(map[string]string), @@ -303,7 +303,7 @@ func TestStepAmiRegionCopy_AMISkipBuildRegion(t *testing.T) { // ------------------------------------------------------------------------ stepAMIRegionCopy := StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: []string{"us-west-1"}, AMIKmsKeyId: "", RegionKeyIds: map[string]string{"us-west-1": "abcde"}, @@ -329,7 +329,7 @@ func TestStepAmiRegionCopy_AMISkipBuildRegion(t *testing.T) { // skip build region is false. // ------------------------------------------------------------------------ stepAMIRegionCopy = StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: []string{"us-west-1"}, AMIKmsKeyId: "", RegionKeyIds: make(map[string]string), @@ -354,7 +354,7 @@ func TestStepAmiRegionCopy_AMISkipBuildRegion(t *testing.T) { // skip build region is false, but encrypt is true // ------------------------------------------------------------------------ stepAMIRegionCopy = StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: []string{"us-west-1"}, AMIKmsKeyId: "", RegionKeyIds: map[string]string{"us-west-1": "abcde"}, @@ -380,7 +380,7 @@ func TestStepAmiRegionCopy_AMISkipBuildRegion(t *testing.T) { // skip build region is true, and encrypt is true // ------------------------------------------------------------------------ stepAMIRegionCopy = StepAMIRegionCopy{ - AccessConfig: testAccessConfig(), + AccessConfig: FakeAccessConfig(), Regions: []string{"us-west-1"}, AMIKmsKeyId: "", RegionKeyIds: map[string]string{"us-west-1": "abcde"}, diff --git a/builder/amazon/common/test_helper_funcs.go b/builder/amazon/common/test_helper_funcs.go new file mode 100644 index 000000000..c951ba855 --- /dev/null +++ b/builder/amazon/common/test_helper_funcs.go @@ -0,0 +1,25 @@ +package common + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" +) + +type mockEC2Client struct { + ec2iface.EC2API +} + +func FakeAccessConfig() *AccessConfig { + accessConfig := AccessConfig{ + getEC2Connection: func() ec2iface.EC2API { + return &mockEC2Client{} + }, + PollingConfig: new(AWSPollingConfig), + } + accessConfig.session = session.Must(session.NewSession(&aws.Config{ + Region: aws.String("us-west-1"), + })) + + return &accessConfig +} diff --git a/builder/amazon/ebsvolume/artifact.go b/builder/amazon/ebsvolume/artifact.go index 683bcbbd3..a6e56060c 100644 --- a/builder/amazon/ebsvolume/artifact.go +++ b/builder/amazon/ebsvolume/artifact.go @@ -13,11 +13,15 @@ import ( // map of region to list of volume IDs type EbsVolumes map[string][]string +// map of region to list of snapshot IDs +type EbsSnapshots map[string][]string + // Artifact is an artifact implementation that contains built AMIs. type Artifact struct { // A map of regions to EBS Volume IDs. Volumes EbsVolumes - + // A map of regions to EBS Snapshot IDs. + Snapshots EbsSnapshots // BuilderId is the unique ID for the builder that created this AMI BuilderIdValue string @@ -40,13 +44,21 @@ func (*Artifact) Files() []string { // returns a sorted list of region:ID pairs func (a *Artifact) idList() []string { - parts := make([]string, 0, len(a.Volumes)) + + parts := make([]string, 0, len(a.Volumes)+len(a.Snapshots)) + for region, volumeIDs := range a.Volumes { for _, volumeID := range volumeIDs { parts = append(parts, fmt.Sprintf("%s:%s", region, volumeID)) } } + for region, snapshotIDs := range a.Snapshots { + for _, snapshotID := range snapshotIDs { + parts = append(parts, fmt.Sprintf("%s:%s", region, snapshotID)) + } + } + sort.Strings(parts) return parts } diff --git a/builder/amazon/ebsvolume/block_device.go b/builder/amazon/ebsvolume/block_device.go index 49d1067c8..042af8cf7 100644 --- a/builder/amazon/ebsvolume/block_device.go +++ b/builder/amazon/ebsvolume/block_device.go @@ -20,6 +20,11 @@ type BlockDevice struct { // [`dynamic_block`](/docs/templates/hcl_templates/expressions#dynamic-blocks) // will allow you to create those programatically. Tag config.KeyValues `mapstructure:"tag" required:"false"` + + // Create a Snapshot of this Volume. + SnapshotVolume bool `mapstructure:"snapshot_volume" required:"false"` + + awscommon.SnapshotConfig `mapstructure:",squash"` } type BlockDevices []BlockDevice @@ -38,6 +43,7 @@ func (bds BlockDevices) Prepare(ctx *interpolate.Context) (errs []error) { for _, block := range bds { errs = append(errs, block.Tag.CopyOn(&block.Tags)...) + errs = append(errs, block.SnapshotTag.CopyOn(&block.SnapshotTags)...) if err := block.Prepare(ctx); err != nil { errs = append(errs, err) diff --git a/builder/amazon/ebsvolume/builder.go b/builder/amazon/ebsvolume/builder.go index 0afe9eb71..25a24fb39 100644 --- a/builder/amazon/ebsvolume/builder.go +++ b/builder/amazon/ebsvolume/builder.go @@ -123,16 +123,12 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) { // Accumulate any errors var errs *packersdk.MultiError var warns []string + errs = packersdk.MultiErrorAppend(errs, b.config.VolumeRunTag.CopyOn(&b.config.VolumeRunTags)...) errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) errs = packersdk.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...) errs = packersdk.MultiErrorAppend(errs, b.config.launchBlockDevices.Prepare(&b.config.ctx)...) - - for _, d := range b.config.VolumeMappings { - if err := d.Prepare(&b.config.ctx); err != nil { - errs = packersdk.MultiErrorAppend(errs, fmt.Errorf("AMIMapping: %s", err.Error())) - } - } + errs = packersdk.MultiErrorAppend(errs, b.config.VolumeMappings.Prepare(&b.config.ctx)...) b.config.launchBlockDevices = b.config.VolumeMappings if err != nil { @@ -318,6 +314,12 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook) EnableAMISriovNetSupport: b.config.AMISriovNetSupport, EnableAMIENASupport: b.config.AMIENASupport, }, + &stepSnapshotEBSVolumes{ + PollingConfig: b.config.PollingConfig, + VolumeMapping: b.config.VolumeMappings, + AccessConfig: &b.config.AccessConfig, + Ctx: b.config.ctx, + }, } // Run! @@ -332,6 +334,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook) // Build the artifact and return it artifact := &Artifact{ Volumes: state.Get("ebsvolumes").(EbsVolumes), + Snapshots: state.Get("ebssnapshots").(EbsSnapshots), BuilderIdValue: BuilderId, Conn: ec2conn, StateData: map[string]interface{}{"generated_data": state.Get("generated_data")}, diff --git a/builder/amazon/ebsvolume/builder.hcl2spec.go b/builder/amazon/ebsvolume/builder.hcl2spec.go index 241b45fd9..7ff6a2a82 100644 --- a/builder/amazon/ebsvolume/builder.hcl2spec.go +++ b/builder/amazon/ebsvolume/builder.hcl2spec.go @@ -25,6 +25,11 @@ type FlatBlockDevice struct { KmsKeyId *string `mapstructure:"kms_key_id" required:"false" cty:"kms_key_id" hcl:"kms_key_id"` Tags map[string]string `mapstructure:"tags" required:"false" cty:"tags" hcl:"tags"` Tag []config.FlatKeyValue `mapstructure:"tag" required:"false" cty:"tag" hcl:"tag"` + SnapshotVolume *bool `mapstructure:"snapshot_volume" required:"false" cty:"snapshot_volume" hcl:"snapshot_volume"` + SnapshotTags map[string]string `mapstructure:"snapshot_tags" required:"false" cty:"snapshot_tags" hcl:"snapshot_tags"` + SnapshotTag []config.FlatKeyValue `mapstructure:"snapshot_tag" required:"false" cty:"snapshot_tag" hcl:"snapshot_tag"` + SnapshotUsers []string `mapstructure:"snapshot_users" required:"false" cty:"snapshot_users" hcl:"snapshot_users"` + SnapshotGroups []string `mapstructure:"snapshot_groups" required:"false" cty:"snapshot_groups" hcl:"snapshot_groups"` } // FlatMapstructure returns a new FlatBlockDevice. @@ -52,6 +57,11 @@ func (*FlatBlockDevice) HCL2Spec() map[string]hcldec.Spec { "kms_key_id": &hcldec.AttrSpec{Name: "kms_key_id", Type: cty.String, Required: false}, "tags": &hcldec.AttrSpec{Name: "tags", Type: cty.Map(cty.String), Required: false}, "tag": &hcldec.BlockListSpec{TypeName: "tag", Nested: hcldec.ObjectSpec((*config.FlatKeyValue)(nil).HCL2Spec())}, + "snapshot_volume": &hcldec.AttrSpec{Name: "snapshot_volume", Type: cty.Bool, Required: false}, + "snapshot_tags": &hcldec.AttrSpec{Name: "snapshot_tags", Type: cty.Map(cty.String), Required: false}, + "snapshot_tag": &hcldec.BlockListSpec{TypeName: "snapshot_tag", Nested: hcldec.ObjectSpec((*config.FlatKeyValue)(nil).HCL2Spec())}, + "snapshot_users": &hcldec.AttrSpec{Name: "snapshot_users", Type: cty.List(cty.String), Required: false}, + "snapshot_groups": &hcldec.AttrSpec{Name: "snapshot_groups", Type: cty.List(cty.String), Required: false}, } return s } diff --git a/builder/amazon/ebsvolume/builder_test.go b/builder/amazon/ebsvolume/builder_test.go index f7110f30e..9db0a189b 100644 --- a/builder/amazon/ebsvolume/builder_test.go +++ b/builder/amazon/ebsvolume/builder_test.go @@ -104,9 +104,6 @@ func TestBuilderPrepare_ReturnGeneratedData(t *testing.T) { if len(generatedData) == 0 { t.Fatalf("Generated data should not be empty") } - if len(generatedData) == 0 { - t.Fatalf("Generated data should not be empty") - } if generatedData[0] != "SourceAMIName" { t.Fatalf("Generated data should contain SourceAMIName") } @@ -126,3 +123,44 @@ func TestBuilderPrepare_ReturnGeneratedData(t *testing.T) { t.Fatalf("Generated data should contain SourceAMIOwnerName") } } + +func TestBuidler_ConfigBlockdevicemapping(t *testing.T) { + var b Builder + config := testConfig() + + //Set some snapshot settings + config["ebs_volumes"] = []map[string]interface{}{ + { + "device_name": "/dev/xvdb", + "volume_size": "32", + "delete_on_termination": true, + }, + { + "device_name": "/dev/xvdc", + "volume_size": "32", + "delete_on_termination": true, + "snapshot_tags": map[string]string{ + "Test_Tag": "tag_value", + "another tag": "another value", + }, + "snapshot_users": []string{ + "123", "456", + }, + }, + } + + generatedData, warnings, err := b.Prepare(config) + + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + if len(generatedData) == 0 { + t.Fatalf("Generated data should not be empty") + } + + t.Logf("Test gen %+v", b.config.VolumeMappings) + +} diff --git a/builder/amazon/ebsvolume/step_snapshot_ebs_volumes.go b/builder/amazon/ebsvolume/step_snapshot_ebs_volumes.go new file mode 100644 index 000000000..4b3408241 --- /dev/null +++ b/builder/amazon/ebsvolume/step_snapshot_ebs_volumes.go @@ -0,0 +1,167 @@ +package ebsvolume + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/hashicorp/packer-plugin-sdk/multistep" + "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer-plugin-sdk/template/interpolate" + awscommon "github.com/hashicorp/packer/builder/amazon/common" +) + +type stepSnapshotEBSVolumes struct { + PollingConfig *awscommon.AWSPollingConfig + AccessConfig *awscommon.AccessConfig + VolumeMapping []BlockDevice + //Map of SnapshotID: BlockDevice, Where *BlockDevice is in VolumeMapping + snapshotMap map[string]*BlockDevice + Ctx interpolate.Context +} + +func (s *stepSnapshotEBSVolumes) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { + ec2conn := state.Get("ec2").(ec2iface.EC2API) + instance := state.Get("instance").(*ec2.Instance) + ui := state.Get("ui").(packer.Ui) + + s.snapshotMap = make(map[string]*BlockDevice) + + for _, instanceBlockDevice := range instance.BlockDeviceMappings { + for _, configVolumeMapping := range s.VolumeMapping { + //Find the config entry for the instance blockDevice + if configVolumeMapping.DeviceName == *instanceBlockDevice.DeviceName { + //Skip Volumes that are not set to create snapshot + if configVolumeMapping.SnapshotVolume != true { + continue + } + + ui.Message(fmt.Sprintf("Compiling list of tags to apply to snapshot from Volume %s...", *instanceBlockDevice.DeviceName)) + tags, err := awscommon.TagMap(configVolumeMapping.SnapshotTags).EC2Tags(s.Ctx, s.AccessConfig.SessionRegion(), state) + if err != nil { + err := fmt.Errorf("Error generating tags for snapshot %s: %s", *instanceBlockDevice.DeviceName, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + tags.Report(ui) + + tagSpec := &ec2.TagSpecification{ + ResourceType: aws.String("snapshot"), + Tags: tags, + } + + input := &ec2.CreateSnapshotInput{ + VolumeId: aws.String(*instanceBlockDevice.Ebs.VolumeId), + TagSpecifications: []*ec2.TagSpecification{tagSpec}, + } + + //Dont try to set an empty tag spec + if len(tags) == 0 { + input.TagSpecifications = nil + } + + ui.Message(fmt.Sprintf("Requesting snapshot of volume: %s...", *instanceBlockDevice.Ebs.VolumeId)) + snapshot, err := ec2conn.CreateSnapshot(input) + if err != nil || snapshot == nil { + err := fmt.Errorf("Error generating snapsot for volume %s: %s", *instanceBlockDevice.Ebs.VolumeId, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + ui.Message(fmt.Sprintf("Requested Snapshot of Volume %s: %s", *instanceBlockDevice.Ebs.VolumeId, *snapshot.SnapshotId)) + s.snapshotMap[*snapshot.SnapshotId] = &configVolumeMapping + } + } + } + + ui.Say("Waiting for Snapshots to become ready...") + for snapID := range s.snapshotMap { + ui.Message(fmt.Sprintf("Waiting for %s to be ready.", snapID)) + err := s.PollingConfig.WaitUntilSnapshotDone(ctx, ec2conn, snapID) + if err != nil { + err = fmt.Errorf("Error waiting for snapsot to become ready %s", err) + state.Put("error", err) + ui.Error(err.Error()) + ui.Message("Failed to wait") + return multistep.ActionHalt + } + ui.Message(fmt.Sprintf("Snapshot Ready: %s", snapID)) + } + + //Attach User and Group permissions to snapshots + ui.Say("Setting User/Group Permissions for Snapshots...") + for snapID, bd := range s.snapshotMap { + snapshotOptions := make(map[string]*ec2.ModifySnapshotAttributeInput) + + if len(bd.SnapshotGroups) > 0 { + groups := make([]*string, len(bd.SnapshotGroups)) + addsSnapshot := make([]*ec2.CreateVolumePermission, len(bd.SnapshotGroups)) + + addSnapshotGroups := &ec2.ModifySnapshotAttributeInput{ + CreateVolumePermission: &ec2.CreateVolumePermissionModifications{}, + } + + for i, g := range bd.SnapshotGroups { + groups[i] = aws.String(g) + addsSnapshot[i] = &ec2.CreateVolumePermission{ + Group: aws.String(g), + } + } + + addSnapshotGroups.GroupNames = groups + addSnapshotGroups.CreateVolumePermission.Add = addsSnapshot + snapshotOptions["groups"] = addSnapshotGroups + + } + + if len(bd.SnapshotUsers) > 0 { + users := make([]*string, len(bd.SnapshotUsers)) + addsSnapshot := make([]*ec2.CreateVolumePermission, len(bd.SnapshotUsers)) + for i, u := range bd.SnapshotUsers { + users[i] = aws.String(u) + addsSnapshot[i] = &ec2.CreateVolumePermission{UserId: aws.String(u)} + } + + snapshotOptions["users"] = &ec2.ModifySnapshotAttributeInput{ + UserIds: users, + CreateVolumePermission: &ec2.CreateVolumePermissionModifications{ + Add: addsSnapshot, + }, + } + } + + //Todo: Copy to other regions and repeat this block in all regions. + for name, input := range snapshotOptions { + ui.Message(fmt.Sprintf("Modifying: %s", name)) + input.SnapshotId = &snapID + _, err := ec2conn.ModifySnapshotAttribute(input) + if err != nil { + err := fmt.Errorf("Error modify snapshot attributes: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + } + } + + //Record all snapshots in current Region. + snapshots := make(EbsSnapshots) + currentregion := s.AccessConfig.SessionRegion() + + for snapID := range s.snapshotMap { + snapshots[currentregion] = append( + snapshots[currentregion], + snapID) + } + //Records artifacts + state.Put("ebssnapshots", snapshots) + + return multistep.ActionContinue +} + +func (s *stepSnapshotEBSVolumes) Cleanup(state multistep.StateBag) { + // No cleanup... +} diff --git a/builder/amazon/ebsvolume/step_snapshot_ebs_volumes_test.go b/builder/amazon/ebsvolume/step_snapshot_ebs_volumes_test.go new file mode 100644 index 000000000..fa946efc7 --- /dev/null +++ b/builder/amazon/ebsvolume/step_snapshot_ebs_volumes_test.go @@ -0,0 +1,170 @@ +package ebsvolume + +import ( + "bytes" + "context" + "fmt" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + + //"github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/hashicorp/packer-plugin-sdk/multistep" + "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer/builder/amazon/common" +) + +// Define a mock struct to be used in unit tests for common aws steps. +type mockEC2Conn struct { + ec2iface.EC2API + Config *aws.Config +} + +func (m *mockEC2Conn) CreateSnapshot(input *ec2.CreateSnapshotInput) (*ec2.Snapshot, error) { + snap := &ec2.Snapshot{ + // This isn't typical amazon format, but injecting the volume id into + // this field lets us verify that the right volume was snapshotted with + // a simple string comparison + SnapshotId: aws.String(fmt.Sprintf("snap-of-%s", *input.VolumeId)), + } + + return snap, nil +} + +func (m *mockEC2Conn) WaitUntilSnapshotCompletedWithContext(aws.Context, *ec2.DescribeSnapshotsInput, ...request.WaiterOption) error { + return nil +} + +func getMockConn(config *common.AccessConfig, target string) (ec2iface.EC2API, error) { + mockConn := &mockEC2Conn{ + Config: aws.NewConfig(), + } + return mockConn, nil +} + +// Create statebag for running test +func tState(t *testing.T) multistep.StateBag { + state := new(multistep.BasicStateBag) + state.Put("ui", &packer.BasicUi{ + Reader: new(bytes.Buffer), + Writer: new(bytes.Buffer), + }) + // state.Put("amis", map[string]string{"us-east-1": "ami-12345"}) + // state.Put("snapshots", map[string][]string{"us-east-1": {"snap-0012345"}}) + conn, _ := getMockConn(&common.AccessConfig{}, "us-east-2") + + state.Put("ec2", conn) + // Store a fake instance that contains a block device that matches the + // volumes defined in the config above + state.Put("instance", &ec2.Instance{ + InstanceId: aws.String("instance-id"), + BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ + { + DeviceName: aws.String("/dev/xvda"), + Ebs: &ec2.EbsInstanceBlockDevice{ + VolumeId: aws.String("vol-1234"), + }, + }, + { + DeviceName: aws.String("/dev/xvdb"), + Ebs: &ec2.EbsInstanceBlockDevice{ + VolumeId: aws.String("vol-5678"), + }, + }, + }, + }) + return state +} + +func TestStepSnapshot_run_simple(t *testing.T) { + var b Builder + config := testConfig() //from builder_test + + //Set some snapshot settings + config["ebs_volumes"] = []map[string]interface{}{ + { + "device_name": "/dev/xvdb", + "volume_size": "32", + "delete_on_termination": true, + "snapshot_volume": true, + }, + } + + generatedData, warnings, err := b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + if len(generatedData) == 0 { + t.Fatalf("Generated data should not be empty") + } + + state := tState(t) + + accessConfig := common.FakeAccessConfig() + + step := stepSnapshotEBSVolumes{ + PollingConfig: new(common.AWSPollingConfig), + AccessConfig: accessConfig, + VolumeMapping: b.config.VolumeMappings, + Ctx: b.config.ctx, + } + + step.Run(context.Background(), state) + + if len(step.snapshotMap) != 1 { + t.Fatalf("Missing Snapshot from step") + } + + if volmapping := step.snapshotMap["snap-of-vol-5678"]; volmapping == nil { + t.Fatalf("Didn't snapshot correct volume: Map is %#v", step.snapshotMap) + } +} + +func TestStepSnapshot_run_no_snaps(t *testing.T) { + var b Builder + config := testConfig() //from builder_test + + //Set some snapshot settings + config["ebs_volumes"] = []map[string]interface{}{ + { + "device_name": "/dev/xvdb", + "volume_size": "32", + "delete_on_termination": true, + "snapshot_volume": false, + }, + } + + generatedData, warnings, err := b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + if len(generatedData) == 0 { + t.Fatalf("Generated data should not be empty") + } + + state := tState(t) + + accessConfig := common.FakeAccessConfig() + + step := stepSnapshotEBSVolumes{ + PollingConfig: new(common.AWSPollingConfig), + AccessConfig: accessConfig, + VolumeMapping: b.config.VolumeMappings, + Ctx: b.config.ctx, + } + + step.Run(context.Background(), state) + + if len(step.snapshotMap) != 0 { + t.Fatalf("Shouldn't have snapshotted any volumes") + } +} diff --git a/website/content/docs/builders/amazon/chroot.mdx b/website/content/docs/builders/amazon/chroot.mdx index f11f038ab..077e62474 100644 --- a/website/content/docs/builders/amazon/chroot.mdx +++ b/website/content/docs/builders/amazon/chroot.mdx @@ -85,6 +85,8 @@ builders. @include 'builder/amazon/common/AMIConfig-not-required.mdx' +@include 'builder/amazon/common/SnapshotConfig-not-required.mdx' + ### Block Devices Configuration Block devices can be nested in the diff --git a/website/content/docs/builders/amazon/ebs.mdx b/website/content/docs/builders/amazon/ebs.mdx index 432bf8667..e7034621f 100644 --- a/website/content/docs/builders/amazon/ebs.mdx +++ b/website/content/docs/builders/amazon/ebs.mdx @@ -59,6 +59,8 @@ necessary for this build to succeed and can be found further down the page. @include 'builder/amazon/common/AMIConfig-not-required.mdx' +@include 'builder/amazon/common/SnapshotConfig-not-required.mdx' + ### Access Configuration #### Required: diff --git a/website/content/docs/builders/amazon/ebssurrogate.mdx b/website/content/docs/builders/amazon/ebssurrogate.mdx index 9e19279cb..a253fb507 100644 --- a/website/content/docs/builders/amazon/ebssurrogate.mdx +++ b/website/content/docs/builders/amazon/ebssurrogate.mdx @@ -55,6 +55,8 @@ necessary for this build to succeed and can be found further down the page. @include 'builder/amazon/common/AMIConfig-not-required.mdx' +@include 'builder/amazon/common/SnapshotConfig-not-required.mdx' + ### Access Configuration #### Required: diff --git a/website/content/docs/builders/amazon/ebsvolume.mdx b/website/content/docs/builders/amazon/ebsvolume.mdx index 52bba564e..814449764 100644 --- a/website/content/docs/builders/amazon/ebsvolume.mdx +++ b/website/content/docs/builders/amazon/ebsvolume.mdx @@ -69,20 +69,6 @@ necessary for this build to succeed and can be found further down the page. @include 'builder/amazon/common/AWSPollingConfig-not-required.mdx' -### AMI Configuration - -#### Optional: - -- `snapshot_groups` (array of strings) - A list of groups that have access to - create volumes from the snapshot(s). By default no groups have permission - to create volumes from the snapshot(s). `all` will make the snapshot - publicly accessible. - -- `snapshot_users` (array of strings) - A list of account IDs that have - access to create volumes from the snapshot(s). By default no additional - users other than the user creating the AMI has permissions to create - volumes from the backing snapshot(s). - ### Block Devices Configuration Block devices can be nested in the @@ -96,6 +82,8 @@ Block devices can be nested in the @include 'builder/amazon/ebsvolume/BlockDevice-not-required.mdx' +@include 'builder/amazon/common/SnapshotConfig-not-required.mdx' + ### Run Configuration #### Required: diff --git a/website/content/docs/builders/amazon/index.mdx b/website/content/docs/builders/amazon/index.mdx index d5e34da1a..f439d4439 100644 --- a/website/content/docs/builders/amazon/index.mdx +++ b/website/content/docs/builders/amazon/index.mdx @@ -44,7 +44,7 @@ filesystem and data. - [amazon-ebsvolume](/docs/builders/amazon/ebsvolume) - Create EBS volumes by launching a source AMI with block devices mapped. Provision the - instance, then destroy it, retaining the EBS volumes. + instance, then destroy it, retaining the EBS volumes and or Snapshot. diff --git a/website/content/partials/builder/amazon/common/AMIConfig-not-required.mdx b/website/content/partials/builder/amazon/common/AMIConfig-not-required.mdx index 72d84bf6c..c07f415c9 100644 --- a/website/content/partials/builder/amazon/common/AMIConfig-not-required.mdx +++ b/website/content/partials/builder/amazon/common/AMIConfig-not-required.mdx @@ -116,23 +116,3 @@ which it will not convert to an AMI in the build region. It will copy the intermediary AMI into any regions provided in `ami_regions`, then delete the intermediary AMI. Default `false`. - -- `snapshot_tags` (map[string]string) - Key/value pair tags to apply to snapshot. They will override AMI tags if - already applied to snapshot. This is a [template - engine](/docs/templates/legacy_json_templates/engine), see [Build template - data](#build-template-data) for more information. - -- `snapshot_tag` ([]{key string, value string}) - Same as [`snapshot_tags`](#snapshot_tags) but defined as a singular - repeatable block containing a `key` and a `value` field. In HCL2 mode the - [`dynamic_block`](/docs/templates/hcl_templates/expressions#dynamic-blocks) - will allow you to create those programatically. - -- `snapshot_users` ([]string) - A list of account IDs that have - access to create volumes from the snapshot(s). By default no additional - users other than the user creating the AMI has permissions to create - volumes from the backing snapshot(s). - -- `snapshot_groups` ([]string) - A list of groups that have access to - create volumes from the snapshot(s). By default no groups have permission - to create volumes from the snapshot(s). all will make the snapshot - publicly accessible. diff --git a/website/content/partials/builder/amazon/common/SnapshotConfig-not-required.mdx b/website/content/partials/builder/amazon/common/SnapshotConfig-not-required.mdx new file mode 100644 index 000000000..ebfd16e78 --- /dev/null +++ b/website/content/partials/builder/amazon/common/SnapshotConfig-not-required.mdx @@ -0,0 +1,21 @@ + + +- `snapshot_tags` (map[string]string) - Key/value pair tags to apply to snapshot. They will override AMI tags if + already applied to snapshot. This is a [template + engine](/docs/templates/legacy_json_templates/engine), see [Build template + data](#build-template-data) for more information. + +- `snapshot_tag` ([]{key string, value string}) - Same as [`snapshot_tags`](#snapshot_tags) but defined as a singular + repeatable block containing a `key` and a `value` field. In HCL2 mode the + [`dynamic_block`](/docs/templates/hcl_templates/expressions#dynamic-blocks) + will allow you to create those programatically. + +- `snapshot_users` ([]string) - A list of account IDs that have + access to create volumes from the snapshot(s). By default no additional + users other than the user creating the AMI has permissions to create + volumes from the backing snapshot(s). + +- `snapshot_groups` ([]string) - A list of groups that have access to + create volumes from the snapshot(s). By default no groups have permission + to create volumes from the snapshot(s). all will make the snapshot + publicly accessible. diff --git a/website/content/partials/builder/amazon/common/SnapshotConfig.mdx b/website/content/partials/builder/amazon/common/SnapshotConfig.mdx new file mode 100644 index 000000000..9ed3d99da --- /dev/null +++ b/website/content/partials/builder/amazon/common/SnapshotConfig.mdx @@ -0,0 +1,3 @@ + + +SnapshotConfig is for common configuration related to creating AMIs. diff --git a/website/content/partials/builder/amazon/ebsvolume/BlockDevice-not-required.mdx b/website/content/partials/builder/amazon/ebsvolume/BlockDevice-not-required.mdx index 9bccc2818..fbeffd139 100644 --- a/website/content/partials/builder/amazon/ebsvolume/BlockDevice-not-required.mdx +++ b/website/content/partials/builder/amazon/ebsvolume/BlockDevice-not-required.mdx @@ -8,3 +8,5 @@ containing a `key` and a `value` field. In HCL2 mode the [`dynamic_block`](/docs/templates/hcl_templates/expressions#dynamic-blocks) will allow you to create those programatically. + +- `snapshot_volume` (bool) - Create a Snapshot of this Volume.