Merge pull request #9555 from yandex-cloud/yandex-export-templating

yandex-export: templating

builder/yandex/builder.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/hashicorp/hcl/v2/hcldec"
+	"github.com/hashicorp/packer/builder"
 	"github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/helper/communicator"
 	"github.com/hashicorp/packer/helper/multistep"
@@ -31,7 +32,19 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) {
 	if errs != nil {
 		return nil, warnings, errs
 	}
-	return nil, warnings, nil
+	generatedData := []string{
+		"ImageID",
+		"ImageName",
+		"ImageFamily",
+		"ImageDescription",
+		"ImageFolderID",
+		"SourceImageID",
+		"SourceImageName",
+		"SourceImageDescription",
+		"SourceImageFamily",
+		"SourceImageFolderID",
+	}
+	return generatedData, warnings, nil
 }
 
 // Run executes a yandex Packer build and returns a packer.Artifact
@@ -51,6 +64,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
 	state.Put("sdk", driver.SDK())
 	state.Put("hook", hook)
 	state.Put("ui", ui)
+	generatedData := &builder.GeneratedData{State: state}
 
 	// Build the steps
 	steps := []multistep.Step{
@@ -61,6 +75,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
 		&StepCreateInstance{
 			Debug:         b.config.PackerDebug,
 			SerialLogFile: b.config.SerialLogFile,
+			GeneratedData: generatedData,
 		},
 		&stepInstanceInfo{},
 		&communicator.StepConnect{
@@ -73,7 +88,9 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
 			Comm: &b.config.Communicator,
 		},
 		&StepTeardownInstance{},
-		&stepCreateImage{},
+		&stepCreateImage{
+			GeneratedData: generatedData,
+		},
 	}
 
 	// Run the steps

builder/yandex/driver_yc.go

@@ -112,6 +112,7 @@ func (d *driverYC) GetImage(imageID string) (*Image, error) {
 		Labels:        image.Labels,
 		Licenses:      image.ProductIds,
 		Name:          image.Name,
+		Description:   image.Description,
 		FolderID:      image.FolderId,
 		MinDiskSizeGb: toGigabytes(image.MinDiskSize),
 		SizeGb:        toGigabytes(image.StorageSize),
@@ -132,6 +133,7 @@ func (d *driverYC) GetImageFromFolder(ctx context.Context, folderID string, fami
 		Labels:        image.Labels,
 		Licenses:      image.ProductIds,
 		Name:          image.Name,
+		Description:   image.Description,
 		FolderID:      image.FolderId,
 		Family:        image.Family,
 		MinDiskSizeGb: toGigabytes(image.MinDiskSize),

builder/yandex/image.go

@@ -7,6 +7,7 @@ type Image struct {
 	Licenses      []string
 	MinDiskSizeGb int
 	Name          string
+	Description   string
 	Family        string
 	SizeGb        int
 }

builder/yandex/step_create_image.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log"
 
+	"github.com/hashicorp/packer/builder"
 	"github.com/hashicorp/packer/helper/multistep"
 	"github.com/hashicorp/packer/packer"
 
@@ -13,9 +14,11 @@ import (
 	ycsdk "github.com/yandex-cloud/go-sdk"
 )
 
-type stepCreateImage struct{}
+type stepCreateImage struct {
+	GeneratedData *builder.GeneratedData
+}
 
-func (stepCreateImage) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+func (s *stepCreateImage) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
 	sdk := state.Get("sdk").(*ycsdk.SDK)
 	ui := state.Get("ui").(packer.Ui)
 	c := state.Get("config").(*Config)
@@ -52,7 +55,7 @@ func (stepCreateImage) Run(ctx context.Context, state multistep.StateBag) multis
 
 	image, ok := resp.(*compute.Image)
 	if !ok {
-		return stepHaltWithError(state, errors.New("Response doesn't contain Image"))
+		return stepHaltWithError(state, errors.New("API call response doesn't contain Compute Image"))
 	}
 
 	log.Printf("Image ID: %s", image.Id)
@@ -62,6 +65,14 @@ func (stepCreateImage) Run(ctx context.Context, state multistep.StateBag) multis
 	log.Printf("Image Storage size: %d", image.StorageSize)
 	state.Put("image", image)
 
+	// provision generated_data from declared in Builder.Prepare func
+	// see doc https://www.packer.io/docs/extending/custom-builders#build-variables for details
+	s.GeneratedData.Put("ImageID", image.Id)
+	s.GeneratedData.Put("ImageName", image.Name)
+	s.GeneratedData.Put("ImageFamily", image.Family)
+	s.GeneratedData.Put("ImageDescription", image.Description)
+	s.GeneratedData.Put("ImageFolderID", image.FolderId)
+
 	return multistep.ActionContinue
 }
 

builder/yandex/step_create_instance.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 
 	"github.com/c2h5oh/datasize"
+	"github.com/hashicorp/packer/builder"
 	"github.com/hashicorp/packer/common/uuid"
 	"github.com/hashicorp/packer/helper/multistep"
 	"github.com/hashicorp/packer/packer"
@@ -21,6 +22,8 @@ const StandardImagesFolderID = "standard-images"
 type StepCreateInstance struct {
 	Debug         bool
 	SerialLogFile string
+
+	GeneratedData *builder.GeneratedData
 }
 
 func createNetwork(ctx context.Context, c *Config, d Driver) (*vpc.Network, error) {
@@ -262,6 +265,14 @@ runcmd:
 		ui.Message(fmt.Sprintf("Disk ID %s. ", instance.BootDisk.DiskId))
 	}
 
+	// provision generated_data from declared in Builder.Prepare func
+	// see doc https://www.packer.io/docs/extending/custom-builders#build-variables for details
+	s.GeneratedData.Put("SourceImageID", sourceImage.ID)
+	s.GeneratedData.Put("SourceImageName", sourceImage.Name)
+	s.GeneratedData.Put("SourceImageDescription", sourceImage.Description)
+	s.GeneratedData.Put("SourceImageFamily", sourceImage.Family)
+	s.GeneratedData.Put("SourceImageFolderID", sourceImage.FolderID)
+
 	return multistep.ActionContinue
 }
 

packer/builder.go

@@ -30,7 +30,7 @@ type Builder interface {
 	// configuration.
 	//
 	// Prepare should return a list of variables that will be made accessible to
-	// users during the provison methods, a list of warnings along with any
+	// users during the provision methods, a list of warnings along with any
 	// errors that occurred while preparing.
 	Prepare(...interface{}) ([]string, []string, error)
 

post-processor/yandex-export/post-processor.go

@@ -6,6 +6,7 @@ package yandexexport
 import (
 	"context"
 	"fmt"
+	"log"
 	"os"
 	"strings"
 	"time"
@@ -13,6 +14,7 @@ import (
 	"github.com/yandex-cloud/go-sdk/iamkey"
 
 	"github.com/hashicorp/hcl/v2/hcldec"
+	"github.com/hashicorp/packer/builder"
 	"github.com/hashicorp/packer/builder/yandex"
 	"github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/helper/config"
@@ -25,7 +27,10 @@ import (
 type Config struct {
 	common.PackerConfig `mapstructure:",squash"`
 
-	// Paths to Yandex Object Storage where exported image will be uploaded
+	// List of paths to Yandex Object Storage where exported image will be uploaded.
+	// Please be aware that use of space char inside path not supported.
+	// Also this param support [build](/docs/templates/engine) template function.
+	// Check available template data for [Yandex](/docs/builders/yandex#build-template-data) builder.
 	Paths []string `mapstructure:"paths" required:"true"`
 	// The folder ID that will be used to launch a temporary instance.
 	// Alternatively you may set value by environment variable YC_FOLDER_ID.
@@ -67,6 +72,11 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 	err := config.Decode(&p.config, &config.DecodeOpts{
 		Interpolate:        true,
 		InterpolateContext: &p.config.ctx,
+		InterpolateFilter: &interpolate.RenderFilter{
+			Exclude: []string{
+				"paths",
+			},
+		},
 	}, raws...)
 	if err != nil {
 		return err
@@ -79,6 +89,14 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 			errs, fmt.Errorf("paths must be specified"))
 	}
 
+	// Validate templates in 'paths'
+	for _, path := range p.config.Paths {
+		if err = interpolate.Validate(path, &p.config.ctx); err != nil {
+			errs = packer.MultiErrorAppend(
+				errs, fmt.Errorf("Error parsing one of 'paths' template: %s", err))
+		}
+	}
+
 	// provision config by OS environment variables
 	if p.config.Token == "" {
 		p.config.Token = os.Getenv("YC_TOKEN")
@@ -143,14 +161,38 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 		return nil, false, false, err
 	}
 
-	builderID := artifact.State("ImageID").(string)
+	// prepare and render values
+	var generatedData map[interface{}]interface{}
+	stateData := artifact.State("generated_data")
+	if stateData != nil {
+		// Make sure it's not a nil map so we can assign to it later.
+		generatedData = stateData.(map[interface{}]interface{})
+	}
+	// If stateData has a nil map generatedData will be nil
+	// and we need to make sure it's not
+	if generatedData == nil {
+		generatedData = make(map[interface{}]interface{})
+	}
+	p.config.ctx.Data = generatedData
 
-	ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", builderID, p.config.Paths))
+	var err error
+	// Render this key since we didn't in the configure phase
+	for i, path := range p.config.Paths {
+		p.config.Paths[i], err = interpolate.Render(path, &p.config.ctx)
+		if err != nil {
+			return nil, false, false, fmt.Errorf("Error rendering one of 'path' template: %s", err)
+		}
+	}
+
+	log.Printf("Rendered path items: %v", p.config.Paths)
+
+	imageID := artifact.State("ImageID").(string)
+	ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", imageID, p.config.Paths))
 
 	// Set up exporter instance configuration.
 	exporterName := fmt.Sprintf("%s-exporter", artifact.Id())
 	exporterMetadata := map[string]string{
-		"image_id":  builderID,
+		"image_id":  imageID,
 		"name":      exporterName,
 		"paths":     strings.Join(p.config.Paths, " "),
 		"user-data": CloudInitScript,
@@ -195,7 +237,8 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 			DebugKeyPath: fmt.Sprintf("yc_pp_%s.pem", p.config.PackerBuildName),
 		},
 		&yandex.StepCreateInstance{
-			Debug: p.config.PackerDebug,
+			Debug:         p.config.PackerDebug,
+			GeneratedData: &builder.GeneratedData{State: state},
 		},
 		new(yandex.StepWaitCloudInitScript),
 		new(yandex.StepTeardownInstance),

website/pages/docs/builders/yandex.mdx

@@ -73,3 +73,62 @@ can be configured for this builder.
 ### Optional:
 
 @include 'builder/yandex/Config-not-required.mdx'
+
+
+## Build template data
+
+In configuration directives the following variables are available:
+
+- `ImageID` - ID of the built image.
+- `ImageName` - Name of the built image.
+- `ImageFamily` - Family of the built image.
+- `ImageDescription` - Description of the built image.
+- `ImageFolderID` - Folder ID where the built image is stored.
+- `SourceImageID` - The source image ID (for example `fd8fjtn3mj2kfe7h6f0r`) used to build the image.
+- `SourceImageName` - The source image name (for example `ubuntu-1604-lts-1579200746`) used to build the image.
+- `SourceImageDescription` - The source image description (for example `ubuntu 16.04 lts`).
+- `SourceImageFamily` - The source image family (for example `ubuntu-1604-lts`).
+- `SourceImageFolderID` - The folder ID where source image located (for example `standard-images`).
+
+
+## Build Shared Information Variables
+
+This builder generates data that are shared with provisioner and post-processor via build function of
+[template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables)
+for HCL2.
+
+The generated variables available for this builder see above
+
+Usage example:
+
+<Tabs>
+<Tab heading="JSON">
+
+```json
+"post-processors": [
+    {
+      "type": "manifest",
+      "output": "manifest.json",
+      "strip_path": true,
+      "custom_data": {
+        "source_image_id": "{{ build `SourceImageID` }}"
+      }
+    }
+]
+```
+
+</Tab>
+<Tab heading="HCL2">
+
+```hcl
+post-processor "manifest" {
+    output = "manifest.json"
+    strip_path = true
+    custom_data = {
+        source_image_id = "${build.SourceImageID}"
+    }
+}
+```
+
+</Tab>
+</Tabs>

website/pages/docs/post-processors/yandex-export.mdx

@@ -80,7 +80,7 @@ must have write access to both `s3://packer-export/my-exported-image.qcow2` and
 
       "paths": [
         "s3://packer-export-bucket/my-exported-image.qcow2",
-        "s3://packer-export-bucket/image-number-two.qcow2"
+        "s3://packer-export-bucket/template-supported-get-{{build `ImageID` }}-right-here.qcow2"
       ],
       "keep_input_artifact": true
     }

website/pages/partials/post-processor/yandex-export/Config-required.mdx

@@ -1,6 +1,9 @@
 <!-- Code generated from the comments of the Config struct in post-processor/yandex-export/post-processor.go; DO NOT EDIT MANUALLY -->
 
--   `paths` ([]string) - Paths to Yandex Object Storage where exported image will be uploaded
+-   `paths` ([]string) - List of paths to Yandex Object Storage where exported image will be uploaded.
+    Please be aware that use of space char inside path not supported.
+    Also this param support [build](/docs/templates/engine) template function.
+    Check available template data for [Yandex](/docs/builders/yandex#build-template-data) builder.
     
 -   `folder_id` (string) - The folder ID that will be used to launch a temporary instance.
     Alternatively you may set value by environment variable YC_FOLDER_ID.