diff --git a/builder/openstack/access_config.go b/builder/openstack/access_config.go index a5ab91b60..1fcefa659 100644 --- a/builder/openstack/access_config.go +++ b/builder/openstack/access_config.go @@ -194,6 +194,13 @@ func (c *AccessConfig) imageV2Client() (*gophercloud.ServiceClient, error) { }) } +func (c *AccessConfig) blockStorageV3Client() (*gophercloud.ServiceClient, error) { + return openstack.NewBlockStorageV3(c.osClient, gophercloud.EndpointOpts{ + Region: c.Region, + Availability: c.getEndpointType(), + }) +} + func (c *AccessConfig) getEndpointType() gophercloud.Availability { if c.EndpointType == "internal" || c.EndpointType == "internalURL" { return gophercloud.AvailabilityInternal diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index 8ebd65d3c..26105a9fd 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -86,18 +86,26 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe PrivateKeyFile: b.config.RunConfig.Comm.SSHPrivateKey, SSHAgentAuth: b.config.RunConfig.Comm.SSHAgentAuth, }, + &StepCreateVolume{ + UseBlockStorageVolume: b.config.UseBlockStorageVolume, + SourceImage: b.config.SourceImage, + VolumeName: b.config.VolumeName, + VolumeType: b.config.VolumeType, + VolumeAvailabilityZone: b.config.VolumeAvailabilityZone, + }, &StepRunSourceServer{ - Name: b.config.InstanceName, - SourceImage: b.config.SourceImage, - SourceImageName: b.config.SourceImageName, - SecurityGroups: b.config.SecurityGroups, - Networks: b.config.Networks, - Ports: b.config.Ports, - AvailabilityZone: b.config.AvailabilityZone, - UserData: b.config.UserData, - UserDataFile: b.config.UserDataFile, - ConfigDrive: b.config.ConfigDrive, - InstanceMetadata: b.config.InstanceMetadata, + Name: b.config.InstanceName, + SourceImage: b.config.SourceImage, + SourceImageName: b.config.SourceImageName, + SecurityGroups: b.config.SecurityGroups, + Networks: b.config.Networks, + Ports: b.config.Ports, + AvailabilityZone: b.config.AvailabilityZone, + UserData: b.config.UserData, + UserDataFile: b.config.UserDataFile, + ConfigDrive: b.config.ConfigDrive, + InstanceMetadata: b.config.InstanceMetadata, + UseBlockStorageVolume: b.config.UseBlockStorageVolume, }, &StepGetPassword{ Debug: b.config.PackerDebug, diff --git a/builder/openstack/run_config.go b/builder/openstack/run_config.go index 2562e0c8d..dbe9a2624 100644 --- a/builder/openstack/run_config.go +++ b/builder/openstack/run_config.go @@ -36,6 +36,11 @@ type RunConfig struct { ConfigDrive bool `mapstructure:"config_drive"` + UseBlockStorageVolume bool `mapstructure:"use_blockstorage_volume"` + VolumeName string `mapstructure:"volume_name"` + VolumeType string `mapstructure:"volume_type"` + VolumeAvailabilityZone string `mapstructure:"volume_availability_zone"` + // Not really used, but here for BC OpenstackProvider string `mapstructure:"openstack_provider"` UseFloatingIp bool `mapstructure:"use_floating_ip"` @@ -90,5 +95,22 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { } } + if c.UseBlockStorageVolume { + if c.VolumeType == "" { + errs = append(errs, errors.New("A volume_type must be provided when use_blockstorage_volume is set to true")) + } + + // Use Compute instance availability zone for the Block Storage volume if + // it's not provided. + if c.VolumeAvailabilityZone == "" { + c.VolumeAvailabilityZone = c.AvailabilityZone + } + + // Use random name for the Block Storage volume if it's not provided. + if c.VolumeName == "" { + c.VolumeName = fmt.Sprintf("packer_%s", uuid.TimeOrderedUUID()) + } + } + return errs } diff --git a/builder/openstack/run_config_test.go b/builder/openstack/run_config_test.go index 151dcd99d..b20e832d8 100644 --- a/builder/openstack/run_config_test.go +++ b/builder/openstack/run_config_test.go @@ -70,3 +70,42 @@ func TestRunConfigPrepare_SSHPort(t *testing.T) { t.Fatalf("invalid value: %d", c.Comm.SSHPort) } } + +func TestRunConfigPrepare_BlockStorage(t *testing.T) { + c := testRunConfig() + c.UseBlockStorageVolume = true + c.VolumeType = "fast" + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + if c.VolumeType != "fast" { + t.Fatalf("invalid value: %s", c.VolumeType) + } + + c.AvailabilityZone = "RegionTwo" + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.VolumeAvailabilityZone != "RegionTwo" { + t.Fatalf("invalid value: %s", c.VolumeAvailabilityZone) + } + + c.VolumeAvailabilityZone = "RegionOne" + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.VolumeAvailabilityZone != "RegionOne" { + t.Fatalf("invalid value: %s", c.VolumeAvailabilityZone) + } + + c.VolumeName = "PackerVolume" + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.VolumeName != "PackerVolume" { + t.Fatalf("invalid value: %s", c.VolumeName) + } +} diff --git a/builder/openstack/step_create_volume.go b/builder/openstack/step_create_volume.go new file mode 100644 index 000000000..6ad569c0b --- /dev/null +++ b/builder/openstack/step_create_volume.go @@ -0,0 +1,173 @@ +package openstack + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes" + "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" + "github.com/hashicorp/packer/helper/multistep" + "github.com/hashicorp/packer/packer" +) + +type StepCreateVolume struct { + UseBlockStorageVolume bool + SourceImage string + VolumeName string + VolumeType string + VolumeAvailabilityZone string + volumeID string + doCleanup bool +} + +func (s *StepCreateVolume) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { + // Proceed only if block storage volume is required. + if !s.UseBlockStorageVolume { + return multistep.ActionContinue + } + + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We will need Block Storage and Image services clients. + blockStorageClient, err := config.blockStorageV3Client() + if err != nil { + err = fmt.Errorf("Error initializing block storage client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + imageClient, err := config.imageV2Client() + if err != nil { + err = fmt.Errorf("Error initializing image client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + // Get needed volume size from the source image. + volumeSize, err := GetVolumeSize(imageClient, s.SourceImage) + if err != nil { + err := fmt.Errorf("Error creating volume: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + ui.Say("Creating volume...") + volumeOpts := volumes.CreateOpts{ + Size: volumeSize, + VolumeType: s.VolumeType, + AvailabilityZone: s.VolumeAvailabilityZone, + Name: s.VolumeName, + ImageID: s.SourceImage, + } + + volume, err := volumes.Create(blockStorageClient, volumeOpts).Extract() + if err != nil { + err := fmt.Errorf("Error creating volume: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Wait for the volume to become ready. + ui.Say(fmt.Sprintf("Waiting for volume %s (volume id: %s) to become ready...", config.VolumeName, volume.ID)) + if err := WaitForVolume(blockStorageClient, volume.ID); err != nil { + err := fmt.Errorf("Error waiting for volume: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Volume was created, so remember to clean it up. + s.doCleanup = true + + // Set the Volume ID in the state. + ui.Message(fmt.Sprintf("Volume ID: %s", volume.ID)) + state.Put("volume_id", volume.ID) + s.volumeID = volume.ID + + return multistep.ActionContinue +} + +// GetVolumeSize returns volume size in gigabytes based on the image min disk +// value if it's not empty. +// Or it calculates needed gigabytes size from the image bytes size. +func GetVolumeSize(imageClient *gophercloud.ServiceClient, imageID string) (int, error) { + sourceImage, err := images.Get(imageClient, imageID).Extract() + if err != nil { + return 0, err + } + + if sourceImage.MinDiskGigabytes != 0 { + return sourceImage.MinDiskGigabytes, nil + } + + volumeSizeMB := sourceImage.SizeBytes / 1024 / 1024 + volumeSizeGB := int(sourceImage.SizeBytes / 1024 / 1024 / 1024) + + // Increment gigabytes size if the initial size can't be divided without + // remainder. + if volumeSizeMB%1024 > 0 { + volumeSizeGB++ + } + + return volumeSizeGB, nil +} + +// WaitForVolume waits for the given volume to become ready. +func WaitForVolume(blockStorageClient *gophercloud.ServiceClient, volumeID string) error { + maxNumErrors := 10 + numErrors := 0 + + for { + volume, err := volumes.Get(blockStorageClient, volumeID).Extract() + if err != nil { + errCode, ok := err.(*gophercloud.ErrUnexpectedResponseCode) + if ok && (errCode.Actual == 500 || errCode.Actual == 404) { + numErrors++ + if numErrors >= maxNumErrors { + log.Printf("[ERROR] Maximum number of errors (%d) reached; failing with: %s", numErrors, err) + return err + } + log.Printf("[ERROR] %d error received, will ignore and retry: %s", errCode.Actual, err) + time.Sleep(2 * time.Second) + continue + } + + return err + } + + if volume.Status == "available" { + return nil + } + + log.Printf("Waiting for volume creation status: %s", volume.Status) + time.Sleep(2 * time.Second) + } +} + +func (s *StepCreateVolume) Cleanup(state multistep.StateBag) { + if !s.doCleanup { + return + } + + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + blockStorageClient, err := config.blockStorageV3Client() + if err != nil { + ui.Error(fmt.Sprintf( + "Error cleaning up volume. Please delete the volume manually: %s", s.volumeID)) + return + } + + ui.Say(fmt.Sprintf("Deleting volume: %s ...", s.volumeID)) + err = volumes.Delete(blockStorageClient, s.volumeID).ExtractErr() + if err != nil { + ui.Error(fmt.Sprintf( + "Error cleaning up volume. Please delete the volume manually: %s", s.volumeID)) + } +} diff --git a/builder/openstack/step_run_source_server.go b/builder/openstack/step_run_source_server.go index 28d50ed93..8b9b312bc 100644 --- a/builder/openstack/step_run_source_server.go +++ b/builder/openstack/step_run_source_server.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "log" + "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/bootfromvolume" "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/keypairs" "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" "github.com/hashicorp/packer/helper/multistep" @@ -13,18 +14,19 @@ import ( ) type StepRunSourceServer struct { - Name string - SourceImage string - SourceImageName string - SecurityGroups []string - Networks []string - Ports []string - AvailabilityZone string - UserData string - UserDataFile string - ConfigDrive bool - InstanceMetadata map[string]string - server *servers.Server + Name string + SourceImage string + SourceImageName string + SecurityGroups []string + Networks []string + Ports []string + AvailabilityZone string + UserData string + UserDataFile string + ConfigDrive bool + InstanceMetadata map[string]string + UseBlockStorageVolume bool + server *servers.Server } func (s *StepRunSourceServer) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { @@ -74,18 +76,40 @@ func (s *StepRunSourceServer) Run(_ context.Context, state multistep.StateBag) m ServiceClient: computeClient, Metadata: s.InstanceMetadata, } - var serverOptsExt servers.CreateOptsBuilder - keyName, hasKey := state.GetOk("keyPair") - if hasKey { - serverOptsExt = keypairs.CreateOptsExt{ + + // Create root volume in the Block Storage service if required. + // Add block device mapping v2 to the server create options if required. + if s.UseBlockStorageVolume { + volume := state.Get("volume_id").(string) + blockDeviceMappingV2 := []bootfromvolume.BlockDevice{ + bootfromvolume.BlockDevice{ + BootIndex: 0, + DestinationType: bootfromvolume.DestinationVolume, + SourceType: bootfromvolume.SourceVolume, + UUID: volume, + }, + } + // ImageRef and block device mapping is an invalid options combination. + serverOpts.ImageRef = "" + serverOptsExt = bootfromvolume.CreateOptsExt{ CreateOptsBuilder: serverOpts, - KeyName: keyName.(string), + BlockDevice: blockDeviceMappingV2, } } else { serverOptsExt = serverOpts } + // Add keypair to the server create options. + keyName, hasKey := state.GetOk("keyPair") + if hasKey { + serverOptsExt = keypairs.CreateOptsExt{ + CreateOptsBuilder: serverOptsExt, + KeyName: keyName.(string), + } + } + + ui.Say("Launching server...") s.server, err = servers.Create(computeClient, serverOptsExt).Extract() if err != nil { err := fmt.Errorf("Error launching source server: %s", err)