Add support for external source image url (#9992)

Signed-off-by: Niels Pardon <par@zurich.ibm.com>

builder/openstack/builder.go

@@ -104,6 +104,8 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
 		&StepSourceImageInfo{
 			SourceImage:               b.config.RunConfig.SourceImage,
 			SourceImageName:           b.config.RunConfig.SourceImageName,
+			ExternalSourceImageURL:    b.config.RunConfig.ExternalSourceImageURL,
+			ExternalSourceImageFormat: b.config.RunConfig.ExternalSourceImageFormat,
 			SourceImageOpts:           b.config.RunConfig.sourceImageOpts,
 			SourceMostRecent:          b.config.SourceImageFilters.MostRecent,
 			SourceProperties:          b.config.SourceImageFilters.Filters.Properties,

builder/openstack/builder.hcl2spec.go

@@ -95,6 +95,8 @@ type FlatConfig struct {
 	SSHIPVersion                *string                 `mapstructure:"ssh_ip_version" required:"false" cty:"ssh_ip_version" hcl:"ssh_ip_version"`
 	SourceImage                 *string                 `mapstructure:"source_image" required:"true" cty:"source_image" hcl:"source_image"`
 	SourceImageName             *string                 `mapstructure:"source_image_name" required:"true" cty:"source_image_name" hcl:"source_image_name"`
+	ExternalSourceImageURL      *string                 `mapstructure:"external_source_image_url" required:"true" cty:"external_source_image_url" hcl:"external_source_image_url"`
+	ExternalSourceImageFormat   *string                 `mapstructure:"external_source_image_format" required:"false" cty:"external_source_image_format" hcl:"external_source_image_format"`
 	SourceImageFilters          *FlatImageFilter        `mapstructure:"source_image_filter" required:"true" cty:"source_image_filter" hcl:"source_image_filter"`
 	Flavor                      *string                 `mapstructure:"flavor" required:"true" cty:"flavor" hcl:"flavor"`
 	AvailabilityZone            *string                 `mapstructure:"availability_zone" required:"false" cty:"availability_zone" hcl:"availability_zone"`
@@ -220,6 +222,8 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"ssh_ip_version":                &hcldec.AttrSpec{Name: "ssh_ip_version", Type: cty.String, Required: false},
 		"source_image":                  &hcldec.AttrSpec{Name: "source_image", Type: cty.String, Required: false},
 		"source_image_name":             &hcldec.AttrSpec{Name: "source_image_name", Type: cty.String, Required: false},
+		"external_source_image_url":     &hcldec.AttrSpec{Name: "external_source_image_url", Type: cty.String, Required: false},
+		"external_source_image_format":  &hcldec.AttrSpec{Name: "external_source_image_format", Type: cty.String, Required: false},
 		"source_image_filter":           &hcldec.BlockSpec{TypeName: "source_image_filter", Nested: hcldec.ObjectSpec((*FlatImageFilter)(nil).HCL2Spec())},
 		"flavor":                        &hcldec.AttrSpec{Name: "flavor", Type: cty.String, Required: false},
 		"availability_zone":             &hcldec.AttrSpec{Name: "availability_zone", Type: cty.String, Required: false},

builder/openstack/run_config.go

@@ -33,6 +33,11 @@ type RunConfig struct {
 	// The name of the base image to use. This is an alternative way of
 	// providing source_image and only either of them can be specified.
 	SourceImageName string `mapstructure:"source_image_name" required:"true"`
+	// The URL of an external base image to use. This is an alternative way of
+	// providing source_image and only either of them can be specified.
+	ExternalSourceImageURL string `mapstructure:"external_source_image_url" required:"true"`
+	// The format of the external source image to use, e.g. qcow2, raw.
+	ExternalSourceImageFormat string `mapstructure:"external_source_image_format" required:"false"`
 	// Filters used to populate filter options. Example:
 	//
 	// ```json
@@ -247,10 +252,19 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error {
 		}
 	}
 
-	if c.SourceImage == "" && c.SourceImageName == "" && c.SourceImageFilters.Filters.Empty() {
+	hasOnlySourceImage := len(c.SourceImage) > 0 && len(c.SourceImageName) == 0 && len(c.ExternalSourceImageURL) == 0
-		errs = append(errs, errors.New("Either a source_image, a source_image_name, or source_image_filter must be specified"))
+	hasOnlySourceImageName := len(c.SourceImageName) > 0 && len(c.SourceImage) == 0 && len(c.ExternalSourceImageURL) == 0
-	} else if len(c.SourceImage) > 0 && len(c.SourceImageName) > 0 {
+	hasOnlyExternalSourceImageURL := len(c.ExternalSourceImageURL) > 0 && len(c.SourceImage) == 0 && len(c.SourceImageName) == 0
-		errs = append(errs, errors.New("Only a source_image or a source_image_name can be specified, not both."))
+
+	if c.SourceImage == "" && c.SourceImageName == "" && c.ExternalSourceImageURL == "" && c.SourceImageFilters.Filters.Empty() {
+		errs = append(errs, errors.New("Either a source_image, a source_image_name, an external_source_image_url or source_image_filter must be specified"))
+	} else if !(hasOnlySourceImage || hasOnlySourceImageName || hasOnlyExternalSourceImageURL) {
+		errs = append(errs, errors.New("Only a source_image, a source_image_name or an external_source_image_url can be specified, not multiple."))
+	}
+
+	// if external_source_image_format is not set use qcow2 as default
+	if c.ExternalSourceImageFormat == "" {
+		c.ExternalSourceImageFormat = "qcow2"
 	}
 
 	if c.Flavor == "" {
@@ -283,9 +297,9 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error {
 		}
 	}
 
-	// if neither ID or image name is provided outside the filter, build the
+	// if neither ID, image name or external image URL is provided outside the filter,
-	// filter
+	// build the filter
-	if len(c.SourceImage) == 0 && len(c.SourceImageName) == 0 {
+	if len(c.SourceImage) == 0 && len(c.SourceImageName) == 0 && len(c.ExternalSourceImageURL) == 0 {
 
 		listOpts, filterErr := c.SourceImageFilters.Filters.Build()
 
@@ -295,6 +309,11 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error {
 		c.sourceImageOpts = *listOpts
 	}
 
+	// if c.ExternalSourceImageURL is set use a generated source image name
+	if c.ExternalSourceImageURL != "" {
+		c.SourceImageName = fmt.Sprintf("packer_%s", uuid.TimeOrderedUUID())
+	}
+
 	return errs
 }
 

builder/openstack/run_config_test.go

@@ -2,6 +2,7 @@ package openstack
 
 import (
 	"os"
+	"regexp"
 	"testing"
 
 	"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
@@ -132,6 +133,49 @@ func TestRunConfigPrepare_FloatingIPPoolCompat(t *testing.T) {
 	}
 }
 
+func TestRunConfigPrepare_ExternalSourceImageURL(t *testing.T) {
+	c := testRunConfig()
+	// test setting both ExternalSourceImageURL and SourceImage causes an error
+	c.ExternalSourceImageURL = "http://example.com/image.qcow2"
+	if err := c.Prepare(nil); len(err) != 1 {
+		t.Fatalf("err: %s", err)
+	}
+
+	// test setting both ExternalSourceImageURL and SourceImageName causes an error
+	c.SourceImage = ""
+	c.SourceImageName = "abcd"
+	c.ExternalSourceImageURL = "http://example.com/image.qcow2"
+	if err := c.Prepare(nil); len(err) != 1 {
+		t.Fatalf("err: %s", err)
+	}
+
+	// test neither setting SourceImage, SourceImageName or ExternalSourceImageURL causes an error
+	c.SourceImage = ""
+	c.SourceImageName = ""
+	c.ExternalSourceImageURL = ""
+	if err := c.Prepare(nil); len(err) != 1 {
+		t.Fatalf("err: %s", err)
+	}
+
+	// test setting only ExternalSourceImageURL passes
+	c.SourceImage = ""
+	c.SourceImageName = ""
+	c.ExternalSourceImageURL = "http://example.com/image.qcow2"
+	if err := c.Prepare(nil); len(err) != 0 {
+		t.Fatalf("err: %s", err)
+	}
+
+	// test default values
+	if c.ExternalSourceImageFormat != "qcow2" {
+		t.Fatalf("ExternalSourceImageFormat should have been set to default: qcow2")
+	}
+
+	p := `packer_[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`
+	if matches, _ := regexp.MatchString(p, c.SourceImageName); !matches {
+		t.Fatalf("invalid format for SourceImageName: %s", c.SourceImageName)
+	}
+}
+
 // This test case confirms that only allowed fields will be set to values
 // The checked values are non-nil for their target type
 func TestBuildImageFilter(t *testing.T) {

builder/openstack/step_source_image_info.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"time"
 
+	"github.com/gophercloud/gophercloud/openstack/imageservice/v2/imageimport"
 	"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
 	"github.com/gophercloud/gophercloud/pagination"
 	"github.com/hashicorp/packer/helper/multistep"
@@ -14,6 +16,8 @@ import (
 type StepSourceImageInfo struct {
 	SourceImage               string
 	SourceImageName           string
+	ExternalSourceImageURL    string
+	ExternalSourceImageFormat string
 	SourceImageOpts           images.ListOpts
 	SourceMostRecent          bool
 	SourceProperties          map[string]string
@@ -33,12 +37,6 @@ func (s *StepSourceImageInfo) Run(ctx context.Context, state multistep.StateBag)
 	config := state.Get("config").(*Config)
 	ui := state.Get("ui").(packer.Ui)
 
-	if s.SourceImage != "" {
-		state.Put("source_image", s.SourceImage)
-
-		return multistep.ActionContinue
-	}
-
 	client, err := config.imageV2Client()
 	if err != nil {
 		err := fmt.Errorf("error creating image client: %s", err)
@@ -47,6 +45,70 @@ func (s *StepSourceImageInfo) Run(ctx context.Context, state multistep.StateBag)
 		return multistep.ActionHalt
 	}
 
+	if s.ExternalSourceImageURL != "" {
+		createOpts := images.CreateOpts{
+			Name:            s.SourceImageName,
+			ContainerFormat: "bare",
+			DiskFormat:      s.ExternalSourceImageFormat,
+			Properties: map[string]string{
+				"packer_external_source_image_url":    s.ExternalSourceImageURL,
+				"packer_external_source_image_format": s.ExternalSourceImageFormat,
+			},
+		}
+
+		ui.Say("Creating image using external source image with name " + s.SourceImageName)
+		ui.Say("Using disk format " + s.ExternalSourceImageFormat)
+		image, err := images.Create(client, createOpts).Extract()
+
+		if err != nil {
+			err := fmt.Errorf("Error creating source image: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		ui.Say("Created image with ID " + image.ID)
+
+		importOpts := imageimport.CreateOpts{
+			Name: imageimport.WebDownloadMethod,
+			URI:  s.ExternalSourceImageURL,
+		}
+
+		ui.Say("Importing External Source Image from URL " + s.ExternalSourceImageURL)
+		err = imageimport.Create(client, image.ID, importOpts).ExtractErr()
+
+		if err != nil {
+			err := fmt.Errorf("Error importing source image: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		for image.Status != images.ImageStatusActive {
+			ui.Message("Image not Active, retrying in 10 seconds")
+			time.Sleep(10 * time.Second)
+
+			img, err := images.Get(client, image.ID).Extract()
+
+			if err != nil {
+				err := fmt.Errorf("Error querying image: %s", err)
+				state.Put("error", err)
+				ui.Error(err.Error())
+				return multistep.ActionHalt
+			}
+
+			image = img
+		}
+
+		s.SourceImage = image.ID
+	}
+
+	if s.SourceImage != "" {
+		state.Put("source_image", s.SourceImage)
+
+		return multistep.ActionContinue
+	}
+
 	if s.SourceImageName != "" {
 		s.SourceImageOpts = images.ListOpts{
 			Name: s.SourceImageName,
@@ -117,5 +179,25 @@ func (s *StepSourceImageInfo) Run(ctx context.Context, state multistep.StateBag)
 }
 
 func (s *StepSourceImageInfo) Cleanup(state multistep.StateBag) {
-	// No cleanup required for backout
+	if s.ExternalSourceImageURL != "" {
+		config := state.Get("config").(*Config)
+		ui := state.Get("ui").(packer.Ui)
+
+		client, err := config.imageV2Client()
+		if err != nil {
+			err := fmt.Errorf("error creating image client: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return
+		}
+
+		ui.Say(fmt.Sprintf("Deleting temporary external source image: %s ...", s.SourceImageName))
+		err = images.Delete(client, s.SourceImage).ExtractErr()
+		if err != nil {
+			err := fmt.Errorf("error cleaning up external source image: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return
+		}
+	}
 }

vendor/github.com/gophercloud/gophercloud/openstack/imageservice/v2/imageimport/doc.go

@@ -0,0 +1,27 @@
+/*
+Package imageimport enables management of images import and retrieval of the
+Imageservice Import API information.
+
+Example to Get an information about the Import API
+
+  importInfo, err := imageimport.Get(imagesClient).Extract()
+  if err != nil {
+    panic(err)
+  }
+
+  fmt.Printf("%+v\n", importInfo)
+
+Example to Create a new image import
+
+  createOpts := imageimport.CreateOpts{
+    Name: imageimport.WebDownloadMethod,
+    URI:  "http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img",
+  }
+  imageID := "da3b75d9-3f4a-40e7-8a2c-bfab23927dea"
+
+  err := imageimport.Create(imagesClient, imageID, createOpts).ExtractErr()
+  if err != nil {
+    panic(err)
+  }
+*/
+package imageimport

vendor/github.com/gophercloud/gophercloud/openstack/imageservice/v2/imageimport/requests.go

@@ -0,0 +1,55 @@
+package imageimport
+
+import "github.com/gophercloud/gophercloud"
+
+// ImportMethod represents valid Import API method.
+type ImportMethod string
+
+const (
+	// GlanceDirectMethod represents glance-direct Import API method.
+	GlanceDirectMethod ImportMethod = "glance-direct"
+
+	// WebDownloadMethod represents web-download Import API method.
+	WebDownloadMethod ImportMethod = "web-download"
+)
+
+// Get retrieves Import API information data.
+func Get(c *gophercloud.ServiceClient) (r GetResult) {
+	resp, err := c.Get(infoURL(c), &r.Body, nil)
+	_, r.Header, r.Err = gophercloud.ParseResponse(resp, err)
+	return
+}
+
+// CreateOptsBuilder allows to add additional parameters to the Create request.
+type CreateOptsBuilder interface {
+	ToImportCreateMap() (map[string]interface{}, error)
+}
+
+// CreateOpts specifies parameters of a new image import.
+type CreateOpts struct {
+	Name ImportMethod `json:"name"`
+	URI  string       `json:"uri"`
+}
+
+// ToImportCreateMap constructs a request body from CreateOpts.
+func (opts CreateOpts) ToImportCreateMap() (map[string]interface{}, error) {
+	b, err := gophercloud.BuildRequestBody(opts, "")
+	if err != nil {
+		return nil, err
+	}
+	return map[string]interface{}{"method": b}, nil
+}
+
+// Create requests the creation of a new image import on the server.
+func Create(client *gophercloud.ServiceClient, imageID string, opts CreateOptsBuilder) (r CreateResult) {
+	b, err := opts.ToImportCreateMap()
+	if err != nil {
+		r.Err = err
+		return
+	}
+	resp, err := client.Post(importURL(client, imageID), b, nil, &gophercloud.RequestOpts{
+		OkCodes: []int{202},
+	})
+	_, r.Header, r.Err = gophercloud.ParseResponse(resp, err)
+	return
+}

vendor/github.com/gophercloud/gophercloud/openstack/imageservice/v2/imageimport/results.go

@@ -0,0 +1,38 @@
+package imageimport
+
+import "github.com/gophercloud/gophercloud"
+
+type commonResult struct {
+	gophercloud.Result
+}
+
+// GetResult represents the result of a get operation. Call its Extract method
+// to interpret it as ImportInfo.
+type GetResult struct {
+	commonResult
+}
+
+// CreateResult is the result of import Create operation. Call its ExtractErr
+// method to determine if the request succeeded or failed.
+type CreateResult struct {
+	gophercloud.ErrResult
+}
+
+// ImportInfo represents information data for the Import API.
+type ImportInfo struct {
+	ImportMethods ImportMethods `json:"import-methods"`
+}
+
+// ImportMethods contains information about available Import API methods.
+type ImportMethods struct {
+	Description string   `json:"description"`
+	Type        string   `json:"type"`
+	Value       []string `json:"value"`
+}
+
+// Extract is a function that accepts a result and extracts ImportInfo.
+func (r commonResult) Extract() (*ImportInfo, error) {
+	var s *ImportInfo
+	err := r.ExtractInto(&s)
+	return s, err
+}

vendor/github.com/gophercloud/gophercloud/openstack/imageservice/v2/imageimport/urls.go

@@ -0,0 +1,17 @@
+package imageimport
+
+import "github.com/gophercloud/gophercloud"
+
+const (
+	rootPath     = "images"
+	infoPath     = "info"
+	resourcePath = "import"
+)
+
+func infoURL(c *gophercloud.ServiceClient) string {
+	return c.ServiceURL(infoPath, resourcePath)
+}
+
+func importURL(c *gophercloud.ServiceClient, imageID string) string {
+	return c.ServiceURL(rootPath, imageID, resourcePath)
+}

vendor/modules.txt

@@ -274,6 +274,7 @@ github.com/gophercloud/gophercloud/openstack/identity/v2/tokens
 github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/ec2tokens
 github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/oauth1
 github.com/gophercloud/gophercloud/openstack/identity/v3/tokens
+github.com/gophercloud/gophercloud/openstack/imageservice/v2/imageimport
 github.com/gophercloud/gophercloud/openstack/imageservice/v2/images
 github.com/gophercloud/gophercloud/openstack/imageservice/v2/members
 github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external

website/pages/partials/builder/openstack/RunConfig-not-required.mdx

@@ -9,6 +9,8 @@
   connect via whichever IP address is returned first from the OpenStack
   API.
 
+- `external_source_image_format` (string) - The format of the external source image to use, e.g. qcow2, raw.
+
 - `availability_zone` (string) - The availability zone to launch the server in. If this isn't specified,
   the default enforced by your OpenStack cluster will be used. This may be
   required for some OpenStack clusters.

website/pages/partials/builder/openstack/RunConfig-required.mdx

@@ -8,6 +8,9 @@
 - `source_image_name` (string) - The name of the base image to use. This is an alternative way of
   providing source_image and only either of them can be specified.
 
+- `external_source_image_url` (string) - The URL of an external base image to use. This is an alternative way of
+  providing source_image and only either of them can be specified.
+
 - `source_image_filter` (ImageFilter) - Filters used to populate filter options. Example:
   
   ```json