OpenStack builder: add Block Storage volumes

This commit allows user to use the Block Storage v3 volume as the
Compute instance root volume.
Also it adds new volume-related parameters to the builder.

builder/openstack/access_config.go

@@ -194,6 +194,13 @@ func (c *AccessConfig) imageV2Client() (*gophercloud.ServiceClient, error) {
 	})
 }
 
+func (c *AccessConfig) blockStorageV3Client() (*gophercloud.ServiceClient, error) {
+	return openstack.NewBlockStorageV3(c.osClient, gophercloud.EndpointOpts{
+		Region:       c.Region,
+		Availability: c.getEndpointType(),
+	})
+}
+
 func (c *AccessConfig) getEndpointType() gophercloud.Availability {
 	if c.EndpointType == "internal" || c.EndpointType == "internalURL" {
 		return gophercloud.AvailabilityInternal

builder/openstack/builder.go

@@ -86,18 +86,26 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			PrivateKeyFile:       b.config.RunConfig.Comm.SSHPrivateKey,
 			SSHAgentAuth:         b.config.RunConfig.Comm.SSHAgentAuth,
 		},
+		&StepCreateVolume{
+			UseBlockStorageVolume:  b.config.UseBlockStorageVolume,
+			SourceImage:            b.config.SourceImage,
+			VolumeName:             b.config.VolumeName,
+			VolumeType:             b.config.VolumeType,
+			VolumeAvailabilityZone: b.config.VolumeAvailabilityZone,
+		},
 		&StepRunSourceServer{
-			Name:             b.config.InstanceName,
-			SourceImage:      b.config.SourceImage,
-			SourceImageName:  b.config.SourceImageName,
-			SecurityGroups:   b.config.SecurityGroups,
-			Networks:         b.config.Networks,
-			Ports:            b.config.Ports,
-			AvailabilityZone: b.config.AvailabilityZone,
-			UserData:         b.config.UserData,
-			UserDataFile:     b.config.UserDataFile,
-			ConfigDrive:      b.config.ConfigDrive,
-			InstanceMetadata: b.config.InstanceMetadata,
+			Name:                  b.config.InstanceName,
+			SourceImage:           b.config.SourceImage,
+			SourceImageName:       b.config.SourceImageName,
+			SecurityGroups:        b.config.SecurityGroups,
+			Networks:              b.config.Networks,
+			Ports:                 b.config.Ports,
+			AvailabilityZone:      b.config.AvailabilityZone,
+			UserData:              b.config.UserData,
+			UserDataFile:          b.config.UserDataFile,
+			ConfigDrive:           b.config.ConfigDrive,
+			InstanceMetadata:      b.config.InstanceMetadata,
+			UseBlockStorageVolume: b.config.UseBlockStorageVolume,
 		},
 		&StepGetPassword{
 			Debug: b.config.PackerDebug,

builder/openstack/run_config.go

@@ -36,6 +36,11 @@ type RunConfig struct {
 
 	ConfigDrive bool `mapstructure:"config_drive"`
 
+	UseBlockStorageVolume  bool   `mapstructure:"use_blockstorage_volume"`
+	VolumeName             string `mapstructure:"volume_name"`
+	VolumeType             string `mapstructure:"volume_type"`
+	VolumeAvailabilityZone string `mapstructure:"volume_availability_zone"`
+
 	// Not really used, but here for BC
 	OpenstackProvider string `mapstructure:"openstack_provider"`
 	UseFloatingIp     bool   `mapstructure:"use_floating_ip"`
@@ -90,5 +95,22 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error {
 		}
 	}
 
+	if c.UseBlockStorageVolume {
+		if c.VolumeType == "" {
+			errs = append(errs, errors.New("A volume_type must be provided when use_blockstorage_volume is set to true"))
+		}
+
+		// Use Compute instance availability zone for the Block Storage volume if
+		// it's not provided.
+		if c.VolumeAvailabilityZone == "" {
+			c.VolumeAvailabilityZone = c.AvailabilityZone
+		}
+
+		// Use random name for the Block Storage volume if it's not provided.
+		if c.VolumeName == "" {
+			c.VolumeName = fmt.Sprintf("packer_%s", uuid.TimeOrderedUUID())
+		}
+	}
+
 	return errs
 }

builder/openstack/run_config_test.go

@@ -70,3 +70,42 @@ func TestRunConfigPrepare_SSHPort(t *testing.T) {
 		t.Fatalf("invalid value: %d", c.Comm.SSHPort)
 	}
 }
+
+func TestRunConfigPrepare_BlockStorage(t *testing.T) {
+	c := testRunConfig()
+	c.UseBlockStorageVolume = true
+	c.VolumeType = "fast"
+	if err := c.Prepare(nil); len(err) != 0 {
+		t.Fatalf("err: %s", err)
+	}
+	if c.VolumeType != "fast" {
+		t.Fatalf("invalid value: %s", c.VolumeType)
+	}
+
+	c.AvailabilityZone = "RegionTwo"
+	if err := c.Prepare(nil); len(err) != 0 {
+		t.Fatalf("err: %s", err)
+	}
+
+	if c.VolumeAvailabilityZone != "RegionTwo" {
+		t.Fatalf("invalid value: %s", c.VolumeAvailabilityZone)
+	}
+
+	c.VolumeAvailabilityZone = "RegionOne"
+	if err := c.Prepare(nil); len(err) != 0 {
+		t.Fatalf("err: %s", err)
+	}
+
+	if c.VolumeAvailabilityZone != "RegionOne" {
+		t.Fatalf("invalid value: %s", c.VolumeAvailabilityZone)
+	}
+
+	c.VolumeName = "PackerVolume"
+	if err := c.Prepare(nil); len(err) != 0 {
+		t.Fatalf("err: %s", err)
+	}
+
+	if c.VolumeName != "PackerVolume" {
+		t.Fatalf("invalid value: %s", c.VolumeName)
+	}
+}

builder/openstack/step_create_volume.go

@@ -0,0 +1,173 @@
+package openstack
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
+	"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+type StepCreateVolume struct {
+	UseBlockStorageVolume  bool
+	SourceImage            string
+	VolumeName             string
+	VolumeType             string
+	VolumeAvailabilityZone string
+	volumeID               string
+	doCleanup              bool
+}
+
+func (s *StepCreateVolume) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
+	// Proceed only if block storage volume is required.
+	if !s.UseBlockStorageVolume {
+		return multistep.ActionContinue
+	}
+
+	config := state.Get("config").(Config)
+	ui := state.Get("ui").(packer.Ui)
+
+	// We will need Block Storage and Image services clients.
+	blockStorageClient, err := config.blockStorageV3Client()
+	if err != nil {
+		err = fmt.Errorf("Error initializing block storage client: %s", err)
+		state.Put("error", err)
+		return multistep.ActionHalt
+	}
+	imageClient, err := config.imageV2Client()
+	if err != nil {
+		err = fmt.Errorf("Error initializing image client: %s", err)
+		state.Put("error", err)
+		return multistep.ActionHalt
+	}
+
+	// Get needed volume size from the source image.
+	volumeSize, err := GetVolumeSize(imageClient, s.SourceImage)
+	if err != nil {
+		err := fmt.Errorf("Error creating volume: %s", err)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	ui.Say("Creating volume...")
+	volumeOpts := volumes.CreateOpts{
+		Size:             volumeSize,
+		VolumeType:       s.VolumeType,
+		AvailabilityZone: s.VolumeAvailabilityZone,
+		Name:             s.VolumeName,
+		ImageID:          s.SourceImage,
+	}
+
+	volume, err := volumes.Create(blockStorageClient, volumeOpts).Extract()
+	if err != nil {
+		err := fmt.Errorf("Error creating volume: %s", err)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	// Wait for the volume to become ready.
+	ui.Say(fmt.Sprintf("Waiting for volume %s (volume id: %s) to become ready...", config.VolumeName, volume.ID))
+	if err := WaitForVolume(blockStorageClient, volume.ID); err != nil {
+		err := fmt.Errorf("Error waiting for volume: %s", err)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	// Volume was created, so remember to clean it up.
+	s.doCleanup = true
+
+	// Set the Volume ID in the state.
+	ui.Message(fmt.Sprintf("Volume ID: %s", volume.ID))
+	state.Put("volume_id", volume.ID)
+	s.volumeID = volume.ID
+
+	return multistep.ActionContinue
+}
+
+// GetVolumeSize returns volume size in gigabytes based on the image min disk
+// value if it's not empty.
+// Or it calculates needed gigabytes size from the image bytes size.
+func GetVolumeSize(imageClient *gophercloud.ServiceClient, imageID string) (int, error) {
+	sourceImage, err := images.Get(imageClient, imageID).Extract()
+	if err != nil {
+		return 0, err
+	}
+
+	if sourceImage.MinDiskGigabytes != 0 {
+		return sourceImage.MinDiskGigabytes, nil
+	}
+
+	volumeSizeMB := sourceImage.SizeBytes / 1024 / 1024
+	volumeSizeGB := int(sourceImage.SizeBytes / 1024 / 1024 / 1024)
+
+	// Increment gigabytes size if the initial size can't be divided without
+	// remainder.
+	if volumeSizeMB%1024 > 0 {
+		volumeSizeGB++
+	}
+
+	return volumeSizeGB, nil
+}
+
+// WaitForVolume waits for the given volume to become ready.
+func WaitForVolume(blockStorageClient *gophercloud.ServiceClient, volumeID string) error {
+	maxNumErrors := 10
+	numErrors := 0
+
+	for {
+		volume, err := volumes.Get(blockStorageClient, volumeID).Extract()
+		if err != nil {
+			errCode, ok := err.(*gophercloud.ErrUnexpectedResponseCode)
+			if ok && (errCode.Actual == 500 || errCode.Actual == 404) {
+				numErrors++
+				if numErrors >= maxNumErrors {
+					log.Printf("[ERROR] Maximum number of errors (%d) reached; failing with: %s", numErrors, err)
+					return err
+				}
+				log.Printf("[ERROR] %d error received, will ignore and retry: %s", errCode.Actual, err)
+				time.Sleep(2 * time.Second)
+				continue
+			}
+
+			return err
+		}
+
+		if volume.Status == "available" {
+			return nil
+		}
+
+		log.Printf("Waiting for volume creation status: %s", volume.Status)
+		time.Sleep(2 * time.Second)
+	}
+}
+
+func (s *StepCreateVolume) Cleanup(state multistep.StateBag) {
+	if !s.doCleanup {
+		return
+	}
+
+	config := state.Get("config").(Config)
+	ui := state.Get("ui").(packer.Ui)
+
+	blockStorageClient, err := config.blockStorageV3Client()
+	if err != nil {
+		ui.Error(fmt.Sprintf(
+			"Error cleaning up volume. Please delete the volume manually: %s", s.volumeID))
+		return
+	}
+
+	ui.Say(fmt.Sprintf("Deleting volume: %s ...", s.volumeID))
+	err = volumes.Delete(blockStorageClient, s.volumeID).ExtractErr()
+	if err != nil {
+		ui.Error(fmt.Sprintf(
+			"Error cleaning up volume. Please delete the volume manually: %s", s.volumeID))
+	}
+}

builder/openstack/step_run_source_server.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"log"
 
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/bootfromvolume"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/keypairs"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
 	"github.com/hashicorp/packer/helper/multistep"
@@ -13,18 +14,19 @@ import (
 )
 
 type StepRunSourceServer struct {
-	Name             string
-	SourceImage      string
-	SourceImageName  string
-	SecurityGroups   []string
-	Networks         []string
-	Ports            []string
-	AvailabilityZone string
-	UserData         string
-	UserDataFile     string
-	ConfigDrive      bool
-	InstanceMetadata map[string]string
-	server           *servers.Server
+	Name                  string
+	SourceImage           string
+	SourceImageName       string
+	SecurityGroups        []string
+	Networks              []string
+	Ports                 []string
+	AvailabilityZone      string
+	UserData              string
+	UserDataFile          string
+	ConfigDrive           bool
+	InstanceMetadata      map[string]string
+	UseBlockStorageVolume bool
+	server                *servers.Server
 }
 
 func (s *StepRunSourceServer) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
@@ -74,18 +76,40 @@ func (s *StepRunSourceServer) Run(_ context.Context, state multistep.StateBag) m
 		ServiceClient:    computeClient,
 		Metadata:         s.InstanceMetadata,
 	}
-
 	var serverOptsExt servers.CreateOptsBuilder
-	keyName, hasKey := state.GetOk("keyPair")
-	if hasKey {
-		serverOptsExt = keypairs.CreateOptsExt{
+
+	// Create root volume in the Block Storage service if required.
+	// Add block device mapping v2 to the server create options if required.
+	if s.UseBlockStorageVolume {
+		volume := state.Get("volume_id").(string)
+		blockDeviceMappingV2 := []bootfromvolume.BlockDevice{
+			bootfromvolume.BlockDevice{
+				BootIndex:       0,
+				DestinationType: bootfromvolume.DestinationVolume,
+				SourceType:      bootfromvolume.SourceVolume,
+				UUID:            volume,
+			},
+		}
+		// ImageRef and block device mapping is an invalid options combination.
+		serverOpts.ImageRef = ""
+		serverOptsExt = bootfromvolume.CreateOptsExt{
 			CreateOptsBuilder: serverOpts,
-			KeyName:           keyName.(string),
+			BlockDevice:       blockDeviceMappingV2,
 		}
 	} else {
 		serverOptsExt = serverOpts
 	}
 
+	// Add keypair to the server create options.
+	keyName, hasKey := state.GetOk("keyPair")
+	if hasKey {
+		serverOptsExt = keypairs.CreateOptsExt{
+			CreateOptsBuilder: serverOptsExt,
+			KeyName:           keyName.(string),
+		}
+	}
+
+	ui.Say("Launching server...")
 	s.server, err = servers.Create(computeClient, serverOptsExt).Extract()
 	if err != nil {
 		err := fmt.Errorf("Error launching source server: %s", err)