From 80dadaf3f156af775cb406027379627c6c8afed7 Mon Sep 17 00:00:00 2001 From: Marin Salinas Date: Thu, 7 Mar 2019 11:10:16 -0600 Subject: [PATCH] feature: add bsuvolume builder and acceptance test --- builder/osc/bsuvolume/artifact.go | 87 ++++++++ builder/osc/bsuvolume/block_device.go | 29 +++ builder/osc/bsuvolume/builder.go | 204 ++++++++++++++++++ builder/osc/bsuvolume/builder_acc_test.go | 86 ++++++++ builder/osc/bsuvolume/builder_test.go | 92 ++++++++ builder/osc/bsuvolume/step_tag_bsu_volumes.go | 81 +++++++ builder/osc/common/block_device.go | 12 +- command/plugin.go | 2 + 8 files changed, 588 insertions(+), 5 deletions(-) create mode 100644 builder/osc/bsuvolume/artifact.go create mode 100644 builder/osc/bsuvolume/block_device.go create mode 100644 builder/osc/bsuvolume/builder.go create mode 100644 builder/osc/bsuvolume/builder_acc_test.go create mode 100644 builder/osc/bsuvolume/builder_test.go create mode 100644 builder/osc/bsuvolume/step_tag_bsu_volumes.go diff --git a/builder/osc/bsuvolume/artifact.go b/builder/osc/bsuvolume/artifact.go new file mode 100644 index 000000000..61d92b6a2 --- /dev/null +++ b/builder/osc/bsuvolume/artifact.go @@ -0,0 +1,87 @@ +package bsuvolume + +import ( + "fmt" + "log" + "sort" + "strings" + + "github.com/hashicorp/packer/packer" + "github.com/outscale/osc-go/oapi" +) + +// map of region to list of volume IDs +type BsuVolumes map[string][]string + +// Artifact is an artifact implementation that contains built AMIs. +type Artifact struct { + // A map of regions to EBS Volume IDs. + Volumes BsuVolumes + + // BuilderId is the unique ID for the builder that created this AMI + BuilderIdValue string + + // Client connection for performing API stuff. + Conn *oapi.Client +} + +func (a *Artifact) BuilderId() string { + return a.BuilderIdValue +} + +func (*Artifact) Files() []string { + // We have no files + return nil +} + +// returns a sorted list of region:ID pairs +func (a *Artifact) idList() []string { + parts := make([]string, 0, len(a.Volumes)) + for region, volumeIDs := range a.Volumes { + for _, volumeID := range volumeIDs { + parts = append(parts, fmt.Sprintf("%s:%s", region, volumeID)) + } + } + + sort.Strings(parts) + return parts +} + +func (a *Artifact) Id() string { + return strings.Join(a.idList(), ",") +} + +func (a *Artifact) String() string { + return fmt.Sprintf("EBS Volumes were created:\n\n%s", strings.Join(a.idList(), "\n")) +} + +func (a *Artifact) State(name string) interface{} { + return nil +} + +func (a *Artifact) Destroy() error { + errors := make([]error, 0) + + for region, volumeIDs := range a.Volumes { + for _, volumeID := range volumeIDs { + log.Printf("Deregistering Volume ID (%s) from region (%s)", volumeID, region) + + input := oapi.DeleteVolumeRequest{ + VolumeId: volumeID, + } + if _, err := a.Conn.POST_DeleteVolume(input); err != nil { + errors = append(errors, err) + } + } + } + + if len(errors) > 0 { + if len(errors) == 1 { + return errors[0] + } else { + return &packer.MultiError{Errors: errors} + } + } + + return nil +} diff --git a/builder/osc/bsuvolume/block_device.go b/builder/osc/bsuvolume/block_device.go new file mode 100644 index 000000000..3217fef7f --- /dev/null +++ b/builder/osc/bsuvolume/block_device.go @@ -0,0 +1,29 @@ +package bsuvolume + +import ( + osccommon "github.com/hashicorp/packer/builder/osc/common" + "github.com/hashicorp/packer/template/interpolate" +) + +type BlockDevice struct { + osccommon.BlockDevice `mapstructure:"-,squash"` + Tags osccommon.TagMap `mapstructure:"tags"` +} + +func commonBlockDevices(mappings []BlockDevice, ctx *interpolate.Context) (osccommon.BlockDevices, error) { + result := make([]osccommon.BlockDevice, len(mappings)) + + for i, mapping := range mappings { + interpolateBlockDev, err := interpolate.RenderInterface(&mapping.BlockDevice, ctx) + if err != nil { + return osccommon.BlockDevices{}, err + } + result[i] = *interpolateBlockDev.(*osccommon.BlockDevice) + } + + return osccommon.BlockDevices{ + LaunchBlockDevices: osccommon.LaunchBlockDevices{ + LaunchMappings: result, + }, + }, nil +} diff --git a/builder/osc/bsuvolume/builder.go b/builder/osc/bsuvolume/builder.go new file mode 100644 index 000000000..363e50faf --- /dev/null +++ b/builder/osc/bsuvolume/builder.go @@ -0,0 +1,204 @@ +// The ebsvolume package contains a packer.Builder implementation that +// builds EBS volumes for Outscale using an ephemeral instance, +package bsuvolume + +import ( + "crypto/tls" + "fmt" + "log" + "net/http" + + osccommon "github.com/hashicorp/packer/builder/osc/common" + "github.com/hashicorp/packer/common" + "github.com/hashicorp/packer/helper/communicator" + "github.com/hashicorp/packer/helper/config" + "github.com/hashicorp/packer/helper/multistep" + "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/template/interpolate" + "github.com/outscale/osc-go/oapi" +) + +const BuilderId = "oapi.outscale.bsuvolume" + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + osccommon.AccessConfig `mapstructure:",squash"` + osccommon.RunConfig `mapstructure:",squash"` + + VolumeMappings []BlockDevice `mapstructure:"ebs_volumes"` + + launchBlockDevices osccommon.BlockDevices + ctx interpolate.Context +} + +type Builder struct { + config Config + runner multistep.Runner +} + +type EngineVarsTemplate struct { + BuildRegion string + SourceOMI string +} + +func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { + b.config.ctx.Funcs = osccommon.TemplateFuncs + // Create passthrough for {{ .BuildRegion }} and {{ .SourceOMI }} variables + // so we can fill them in later + b.config.ctx.Data = &EngineVarsTemplate{ + BuildRegion: `{{ .BuildRegion }}`, + SourceOMI: `{{ .SourceOMI }} `, + } + err := config.Decode(&b.config, &config.DecodeOpts{ + Interpolate: true, + InterpolateContext: &b.config.ctx, + }, raws...) + if err != nil { + return nil, err + } + + // Accumulate any errors + var errs *packer.MultiError + errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) + errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...) + errs = packer.MultiErrorAppend(errs, b.config.launchBlockDevices.Prepare(&b.config.ctx)...) + + for _, d := range b.config.VolumeMappings { + if err := d.Prepare(&b.config.ctx); err != nil { + errs = packer.MultiErrorAppend(errs, fmt.Errorf("OMIMapping: %s", err.Error())) + } + } + + b.config.launchBlockDevices, err = commonBlockDevices(b.config.VolumeMappings, &b.config.ctx) + if err != nil { + errs = packer.MultiErrorAppend(errs, err) + } + + if errs != nil && len(errs.Errors) > 0 { + return nil, errs + } + + packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token) + return nil, nil +} + +func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { + clientConfig, err := b.config.Config() + if err != nil { + return nil, err + } + + skipClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + + oapiconn := oapi.NewClient(clientConfig, skipClient) + + // Setup the state bag and initial state for the steps + state := new(multistep.BasicStateBag) + state.Put("config", &b.config) + state.Put("oapi", oapiconn) + state.Put("hook", hook) + state.Put("ui", ui) + + log.Printf("[DEBUG] launch block devices %#v", b.config.launchBlockDevices) + + instanceStep := &osccommon.StepRunSourceVm{ + AssociatePublicIpAddress: b.config.AssociatePublicIpAddress, + BlockDevices: b.config.launchBlockDevices, + Comm: &b.config.RunConfig.Comm, + Ctx: b.config.ctx, + Debug: b.config.PackerDebug, + BsuOptimized: b.config.BsuOptimized, + EnableT2Unlimited: b.config.EnableT2Unlimited, + ExpectedRootDevice: "ebs", + IamVmProfile: b.config.IamVmProfile, + VmInitiatedShutdownBehavior: b.config.VmInitiatedShutdownBehavior, + VmType: b.config.VmType, + SourceOMI: b.config.SourceOmi, + Tags: b.config.RunTags, + UserData: b.config.UserData, + UserDataFile: b.config.UserDataFile, + } + + // Build the steps + steps := []multistep.Step{ + &osccommon.StepSourceOMIInfo{ + SourceOmi: b.config.SourceOmi, + OmiFilters: b.config.SourceOmiFilter, + }, + &osccommon.StepNetworkInfo{ + NetId: b.config.NetId, + NetFilter: b.config.NetFilter, + SecurityGroupIds: b.config.SecurityGroupIds, + SecurityGroupFilter: b.config.SecurityGroupFilter, + SubnetId: b.config.SubnetId, + SubnetFilter: b.config.SubnetFilter, + SubregionName: b.config.Subregion, + }, + &osccommon.StepKeyPair{ + Debug: b.config.PackerDebug, + Comm: &b.config.RunConfig.Comm, + DebugKeyPath: fmt.Sprintf("oapi_%s.pem", b.config.PackerBuildName), + }, + &osccommon.StepSecurityGroup{ + SecurityGroupFilter: b.config.SecurityGroupFilter, + SecurityGroupIds: b.config.SecurityGroupIds, + CommConfig: &b.config.RunConfig.Comm, + TemporarySGSourceCidr: b.config.TemporarySGSourceCidr, + }, + instanceStep, + &stepTagBSUVolumes{ + VolumeMapping: b.config.VolumeMappings, + Ctx: b.config.ctx, + }, + &osccommon.StepGetPassword{ + Debug: b.config.PackerDebug, + Comm: &b.config.RunConfig.Comm, + Timeout: b.config.WindowsPasswordTimeout, + BuildName: b.config.PackerBuildName, + }, + &communicator.StepConnect{ + Config: &b.config.RunConfig.Comm, + Host: osccommon.SSHHost( + oapiconn, + b.config.Comm.SSHInterface), + SSHConfig: b.config.RunConfig.Comm.SSHConfigFunc(), + }, + &common.StepProvision{}, + &common.StepCleanupTempKeys{ + Comm: &b.config.RunConfig.Comm, + }, + &osccommon.StepStopBSUBackedVm{ + Skip: b.config.IsSpotVm(), + DisableStopVm: b.config.DisableStopVm, + }, + } + + // Run! + b.runner = common.NewRunner(steps, b.config.PackerConfig, ui) + b.runner.Run(state) + + // If there was an error, return that + if rawErr, ok := state.GetOk("error"); ok { + return nil, rawErr.(error) + } + + // Build the artifact and return it + artifact := &Artifact{ + Volumes: state.Get("bsuvolumes").(BsuVolumes), + BuilderIdValue: BuilderId, + Conn: oapiconn, + } + ui.Say(fmt.Sprintf("Created Volumes: %s", artifact)) + return artifact, nil +} + +func (b *Builder) Cancel() { + if b.runner != nil { + log.Println("Cancelling the step runner...") + b.runner.Cancel() + } +} diff --git a/builder/osc/bsuvolume/builder_acc_test.go b/builder/osc/bsuvolume/builder_acc_test.go new file mode 100644 index 000000000..ef4e46cde --- /dev/null +++ b/builder/osc/bsuvolume/builder_acc_test.go @@ -0,0 +1,86 @@ +//TODO: explain how to delete the image. +package bsuvolume + +import ( + "crypto/tls" + "net/http" + "testing" + + "github.com/hashicorp/packer/builder/osc/common" + builderT "github.com/hashicorp/packer/helper/builder/testing" + "github.com/outscale/osc-go/oapi" +) + +func TestBuilderAcc_basic(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Builder: &Builder{}, + Template: testBuilderAccBasic, + SkipArtifactTeardown: true, + }) +} + +func testAccPreCheck(t *testing.T) { +} + +func testOAPIConn() (*oapi.Client, error) { + access := &common.AccessConfig{RawRegion: "us-east-1"} + clientConfig, err := access.Config() + if err != nil { + return nil, err + } + + skipClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + + return oapi.NewClient(clientConfig, skipClient), nil +} + +const testBuilderAccBasic = ` +{ + "builders": [ + { + "type": "test", + "region": "eu-west-2", + "vm_type": "t2.micro", + "source_omi": "ami-65efcc11", + "ssh_username": "outscale", + "ebs_volumes": [ + { + "volume_type": "gp2", + "device_name": "/dev/xvdf", + "delete_on_vm_deletion": false, + "tags": { + "zpool": "data", + "Name": "Data1" + }, + "volume_size": 10 + }, + { + "volume_type": "gp2", + "device_name": "/dev/xvdg", + "tags": { + "zpool": "data", + "Name": "Data2" + }, + "delete_on_vm_deletion": false, + "volume_size": 10 + }, + { + "volume_size": 10, + "tags": { + "Name": "Data3", + "zpool": "data" + }, + "delete_on_vm_deletion": false, + "device_name": "/dev/xvdh", + "volume_type": "gp2" + } + ] + } + ] +} +` diff --git a/builder/osc/bsuvolume/builder_test.go b/builder/osc/bsuvolume/builder_test.go new file mode 100644 index 000000000..956fba60f --- /dev/null +++ b/builder/osc/bsuvolume/builder_test.go @@ -0,0 +1,92 @@ +package bsuvolume + +import ( + "testing" + + "github.com/hashicorp/packer/packer" +) + +func testConfig() map[string]interface{} { + return map[string]interface{}{ + "access_key": "foo", + "secret_key": "bar", + "source_omi": "foo", + "vm_type": "foo", + "region": "us-east-1", + "ssh_username": "root", + } +} + +func TestBuilder_ImplementsBuilder(t *testing.T) { + var raw interface{} + raw = &Builder{} + if _, ok := raw.(packer.Builder); !ok { + t.Fatalf("Builder should be a builder") + } +} + +func TestBuilder_Prepare_BadType(t *testing.T) { + b := &Builder{} + c := map[string]interface{}{ + "access_key": []string{}, + } + + warnings, err := b.Prepare(c) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err == nil { + t.Fatalf("prepare should fail") + } +} + +func TestBuilderPrepare_InvalidKey(t *testing.T) { + var b Builder + config := testConfig() + + // Add a random key + config["i_should_not_be_valid"] = true + warnings, err := b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err == nil { + t.Fatal("should have error") + } +} + +func TestBuilderPrepare_InvalidShutdownBehavior(t *testing.T) { + var b Builder + config := testConfig() + + // Test good + config["shutdown_behavior"] = "terminate" + config["skip_region_validation"] = true + warnings, err := b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + // Test good + config["shutdown_behavior"] = "stop" + warnings, err = b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + // Test bad + config["shutdown_behavior"] = "foobar" + warnings, err = b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err == nil { + t.Fatal("should have error") + } +} diff --git a/builder/osc/bsuvolume/step_tag_bsu_volumes.go b/builder/osc/bsuvolume/step_tag_bsu_volumes.go new file mode 100644 index 000000000..c02f77112 --- /dev/null +++ b/builder/osc/bsuvolume/step_tag_bsu_volumes.go @@ -0,0 +1,81 @@ +package bsuvolume + +import ( + "context" + "fmt" + + "github.com/hashicorp/packer/helper/multistep" + "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/template/interpolate" + "github.com/outscale/osc-go/oapi" +) + +type stepTagBSUVolumes struct { + VolumeMapping []BlockDevice + Ctx interpolate.Context +} + +func (s *stepTagBSUVolumes) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { + oapiconn := state.Get("oapi").(*oapi.Client) + vm := state.Get("vm").(oapi.Vm) + ui := state.Get("ui").(packer.Ui) + + volumes := make(BsuVolumes) + for _, instanceBlockDevices := range vm.BlockDeviceMappings { + for _, configVolumeMapping := range s.VolumeMapping { + if configVolumeMapping.DeviceName == instanceBlockDevices.DeviceName { + volumes[oapiconn.GetConfig().Region] = append( + volumes[oapiconn.GetConfig().Region], + instanceBlockDevices.Bsu.VolumeId) + } + } + } + state.Put("bsuvolumes", volumes) + + if len(s.VolumeMapping) > 0 { + ui.Say("Tagging BSU volumes...") + + toTag := map[string][]oapi.ResourceTag{} + for _, mapping := range s.VolumeMapping { + if len(mapping.Tags) == 0 { + ui.Say(fmt.Sprintf("No tags specified for volume on %s...", mapping.DeviceName)) + continue + } + + tags, err := mapping.Tags.OAPITags(s.Ctx, oapiconn.GetConfig().Region, state) + if err != nil { + err := fmt.Errorf("Error tagging device %s with %s", mapping.DeviceName, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + tags.Report(ui) + + for _, v := range vm.BlockDeviceMappings { + if v.DeviceName == mapping.DeviceName { + toTag[v.Bsu.VolumeId] = tags + } + } + } + + for volumeId, tags := range toTag { + _, err := oapiconn.POST_CreateTags(oapi.CreateTagsRequest{ + ResourceIds: []string{volumeId}, + Tags: tags, + }) + if err != nil { + err := fmt.Errorf("Error tagging BSU Volume %s on %s: %s", volumeId, vm.VmId, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + } + } + + return multistep.ActionContinue +} + +func (s *stepTagBSUVolumes) Cleanup(state multistep.StateBag) { + // No cleanup... +} diff --git a/builder/osc/common/block_device.go b/builder/osc/common/block_device.go index 420d112da..24da540ce 100644 --- a/builder/osc/common/block_device.go +++ b/builder/osc/common/block_device.go @@ -2,6 +2,7 @@ package common import ( "fmt" + "log" "strings" "github.com/hashicorp/packer/template/interpolate" @@ -50,9 +51,8 @@ func buildBlockDevices(b []BlockDevice) []*oapi.BlockDeviceMapping { mapping.VirtualDeviceName = blockDevice.VirtualName } } else { - bsu := oapi.Bsu{ - DeleteOnVmDeletion: blockDevice.DeleteOnVmDeletion, - } + bsu := oapi.Bsu{} + bsu.DeleteOnVmDeletion = &blockDevice.DeleteOnVmDeletion if blockDevice.VolumeType != "" { bsu.VolumeType = blockDevice.VolumeType @@ -97,7 +97,7 @@ func buildBlockDevicesImage(b []BlockDevice) []oapi.BlockDeviceMappingImage { } } else { bsu := oapi.BsuToCreate{ - DeleteOnVmDeletion: blockDevice.DeleteOnVmDeletion, + DeleteOnVmDeletion: &blockDevice.DeleteOnVmDeletion, } if blockDevice.VolumeType != "" { @@ -126,6 +126,8 @@ func buildBlockDevicesImage(b []BlockDevice) []oapi.BlockDeviceMappingImage { } func buildBlockDevicesVmCreation(b []BlockDevice) []oapi.BlockDeviceMappingVmCreation { + log.Printf("[DEBUG] Launch Block Device %#v", b) + var blockDevices []oapi.BlockDeviceMappingVmCreation for _, blockDevice := range b { @@ -141,7 +143,7 @@ func buildBlockDevicesVmCreation(b []BlockDevice) []oapi.BlockDeviceMappingVmCre } } else { bsu := oapi.BsuToCreate{ - DeleteOnVmDeletion: blockDevice.DeleteOnVmDeletion, + DeleteOnVmDeletion: &blockDevice.DeleteOnVmDeletion, } if blockDevice.VolumeType != "" { diff --git a/command/plugin.go b/command/plugin.go index cfe9c417d..9ebd2a046 100644 --- a/command/plugin.go +++ b/command/plugin.go @@ -40,6 +40,7 @@ import ( oracleocibuilder "github.com/hashicorp/packer/builder/oracle/oci" oscbsubuilder "github.com/hashicorp/packer/builder/osc/bsu" oscbsusurrogatebuilder "github.com/hashicorp/packer/builder/osc/bsusurrogate" + oscbsuvolumebuilder "github.com/hashicorp/packer/builder/osc/bsuvolume" oscchrootbuilder "github.com/hashicorp/packer/builder/osc/chroot" parallelsisobuilder "github.com/hashicorp/packer/builder/parallels/iso" parallelspvmbuilder "github.com/hashicorp/packer/builder/parallels/pvm" @@ -126,6 +127,7 @@ var Builders = map[string]packer.Builder{ "oracle-oci": new(oracleocibuilder.Builder), "osc-bsu": new(oscbsubuilder.Builder), "osc-bsusurrogate": new(oscbsusurrogatebuilder.Builder), + "osc-bsuvolume": new(oscbsuvolumebuilder.Builder), "osc-chroot": new(oscchrootbuilder.Builder), "parallels-iso": new(parallelsisobuilder.Builder), "parallels-pvm": new(parallelspvmbuilder.Builder),