packer-cn/post-processor/yandex-import/post-processor.go

//go:generate struct-markdown
//go:generate mapstructure-to-hcl2 -type Config

package yandeximport

import (
	"context"
	"fmt"
	"os"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/hashicorp/packer/builder/yandex"
	"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
	"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
	"github.com/yandex-cloud/go-sdk/iamkey"

	"github.com/hashicorp/hcl/v2/hcldec"
	"github.com/hashicorp/packer/builder/file"
	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/post-processor/artifice"
	"github.com/hashicorp/packer/post-processor/compress"
	yandexexport "github.com/hashicorp/packer/post-processor/yandex-export"
	"github.com/hashicorp/packer/template/interpolate"
)

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	// The folder ID that will be used to store imported Image.
	FolderID string `mapstructure:"folder_id" required:"true"`
	// Service Account ID with proper permission to use Storage service
	// for operations 'upload' and 'delete' object to `bucket`
	ServiceAccountID string `mapstructure:"service_account_id" required:"true"`

	// OAuth token to use to authenticate to Yandex.Cloud.
	Token string `mapstructure:"token" required:"false"`
	// Path to file with Service Account key in json format. This
	// is an alternative method to authenticate to Yandex.Cloud.
	ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"`

	// The name of the bucket where the qcow2 file will be uploaded to for import.
	// This bucket must exist when the post-processor is run.
	//
	// If import occurred after Yandex-Export post-processor, artifact already
	// in storage service and first paths (URL) is used to, so no need to set this param.
	Bucket string `mapstructure:"bucket" required:"false"`
	// The name of the object key in `bucket` where the qcow2 file will be copied to import.
	// This is a [template engine](/docs/templates/engine).
	// Therefore, you may use user variables and template functions in this field.
	ObjectName string `mapstructure:"object_name" required:"false"`
	// Whether skip removing the qcow2 file uploaded to Storage
	// after the import process has completed. Possible values are: `true` to
	// leave it in the bucket, `false` to remove it. (Default: `false`).
	SkipClean bool `mapstructure:"skip_clean" required:"false"`

	// The name of the image, which contains 1-63 characters and only
	// supports lowercase English characters, numbers and hyphen.
	ImageName string `mapstructure:"image_name" required:"false"`
	// The description of the image.
	ImageDescription string `mapstructure:"image_description" required:"false"`
	// The family name of the imported image.
	ImageFamily string `mapstructure:"image_family" required:"false"`
	// Key/value pair labels to apply to the imported image.
	ImageLabels map[string]string `mapstructure:"image_labels" required:"false"`

	ctx interpolate.Context
}

type PostProcessor struct {
	config Config
}

func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() }

func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{
				"object_name",
			},
		},
	}, raws...)
	if err != nil {
		return err
	}

	errs := new(packer.MultiError)

	// provision config by OS environment variables
	if p.config.Token == "" {
		p.config.Token = os.Getenv("YC_TOKEN")
	}

	if p.config.ServiceAccountKeyFile == "" {
		p.config.ServiceAccountKeyFile = os.Getenv("YC_SERVICE_ACCOUNT_KEY_FILE")
	}

	if p.config.Token != "" {
		packer.LogSecretFilter.Set(p.config.Token)
	}

	if p.config.ServiceAccountKeyFile != "" {
		if _, err := iamkey.ReadFromJSONFile(p.config.ServiceAccountKeyFile); err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("fail to read service account key file: %s", err))
		}
	}

	if p.config.FolderID == "" {
		p.config.FolderID = os.Getenv("YC_FOLDER_ID")
	}

	// Set defaults
	if p.config.ObjectName == "" {
		p.config.ObjectName = "packer-import-{{timestamp}}.qcow2"
	}

	// Check and render object_name
	if err = interpolate.Validate(p.config.ObjectName, &p.config.ctx); err != nil {
		errs = packer.MultiErrorAppend(
			errs, fmt.Errorf("error parsing object_name template: %s", err))
	}

	// TODO: make common code to check and prepare Yandex.Cloud auth configuration data

	templates := map[string]*string{
		"object_name": &p.config.ObjectName,
		"folder_id":   &p.config.FolderID,
	}

	for key, ptr := range templates {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("%s must be set", key))
		}
	}

	if len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) {
	generatedData := artifact.State("generated_data")
	if generatedData == nil {
		// Make sure it's not a nil map so we can assign to it later.
		generatedData = make(map[string]interface{})
	}
	p.config.ctx.Data = generatedData

	cfg := &yandex.Config{
		Token:                 p.config.Token,
		ServiceAccountKeyFile: p.config.ServiceAccountKeyFile,
	}

	client, err := yandex.NewDriverYC(ui, cfg)
	if err != nil {
		return nil, false, false, err
	}

	p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx)
	if err != nil {
		return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
	}

	var url string
	var fileSource bool

	// Create temporary storage Access Key
	respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
		ServiceAccountId: p.config.ServiceAccountID,
		Description:      "this temporary key is for upload image to storage; created by Packer",
	})
	if err != nil {
		return nil, false, false, err
	}

	storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
	if err != nil {
		return nil, false, false, fmt.Errorf("error create object storage client: %s", err)
	}

	switch artifact.BuilderId() {
	case compress.BuilderId, artifice.BuilderId, file.BuilderId:
		// Artifact as a file, need to be uploaded to storage before create Compute Image
		fileSource = true

		// As `bucket` option validate input here
		if p.config.Bucket == "" {
			return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value")
		}

		url, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
		if err != nil {
			return nil, false, false, err
		}

	case yandexexport.BuilderId:
		// Artifact already in storage, just get URL
		url = artifact.Id()

	case BuilderId:
		// Artifact from prev yandex-import PP, reuse URL
		url = artifact.Id()

	default:
		err := fmt.Errorf(
			"Unknown artifact type: %s\nCan only import from Yandex-Export, Yandex-Import, Compress, Artifice and File post-processor artifacts.",
			artifact.BuilderId())
		return nil, false, false, err
	}

	presignedUrl, err := presignUrl(storageClient, ui, url)
	if err != nil {
		return nil, false, false, err
	}

	ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, presignedUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
	if err != nil {
		return nil, false, false, err
	}

	if fileSource && !p.config.SkipClean {
		err = deleteFromBucket(storageClient, ui, url)
		if err != nil {
			return nil, false, false, err
		}
	}

	// Delete temporary storage Access Key
	_, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{
		AccessKeyId: respWithKey.GetAccessKey().GetId(),
	})
	if err != nil {
		return nil, false, false, fmt.Errorf("error delete static access key: %s", err)
	}

	return &Artifact{
		imageID:   ycImage.GetId(),
		sourceURL: url,
	}, false, false, nil
}

func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
	ui.Say("Looking for qcow2 file in list of artifacts...")
	source := ""
	for _, path := range artifact.Files() {
		ui.Say(fmt.Sprintf("Found artifact %v...", path))
		if strings.HasSuffix(path, ".qcow2") {
			source = path
			break
		}
	}

	if source == "" {
		return "", fmt.Errorf("no qcow2 file found in list of artifacts")
	}

	artifactFile, err := os.Open(source)
	if err != nil {
		err := fmt.Errorf("error opening %v", source)
		return "", err
	}

	ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName))

	_, err = s3conn.PutObject(&s3.PutObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(objectName),
		Body:   artifactFile,
	})

	if err != nil {
		ui.Say(fmt.Sprintf("Failed to upload: %v", objectName))
		return "", err
	}

	req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(objectName),
	})

	// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
	req.Config.S3ForcePathStyle = aws.Bool(true)

	return req.HTTPRequest.URL.String(), nil
}

func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) {
	op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, &compute.CreateImageRequest{
		FolderId:    folderID,
		Name:        imageName,
		Description: imageDescription,
		Labels:      imageLabels,
		Family:      imageFamily,
		Source:      &compute.CreateImageRequest_Uri{Uri: rawImageURL},
	}))
	if err != nil {
		ui.Say("Error creating Yandex Compute Image")
		return nil, err
	}

	ui.Say(fmt.Sprintf("Source url for Image creation: %v", rawImageURL))

	ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id()))

	ui.Say("Waiting for Yandex Compute Image creation operation to complete...")
	err = op.Wait(ctx)

	// fail if image creation operation has an error
	if err != nil {
		return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err)
	}

	protoMetadata, err := op.Metadata()
	if err != nil {
		return nil, fmt.Errorf("error while get image create operation metadata: %s", err)
	}

	md, ok := protoMetadata.(*compute.CreateImageMetadata)
	if !ok {
		return nil, fmt.Errorf("could not get Image ID from create operation metadata")
	}

	image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
		ImageId: md.ImageId,
	})
	if err != nil {
		return nil, fmt.Errorf("error while image get request: %s", err)
	}

	return image, nil

}

func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error {
	bucket, objectName, err := s3URLToBucketKey(url)
	if err != nil {
		return err
	}

	ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))

	_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(objectName),
	})
	if err != nil {
		ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName))
		return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err)
	}

	return nil
}