From 553d203e68d2c68e89eb35e7c6da34b1ecce16e2 Mon Sep 17 00:00:00 2001 From: Gennady Lipenkov Date: Wed, 8 Jul 2020 21:58:56 +0300 Subject: [PATCH] Add yandex-import post-processor --- command/plugin.go | 2 + .../yandex-import/post-processor.go | 309 ++++++++++++++++++ .../yandex-import/post-processor.hcl2spec.go | 64 ++++ post-processor/yandex-import/storage.go | 42 +++ 4 files changed, 417 insertions(+) create mode 100644 post-processor/yandex-import/post-processor.go create mode 100644 post-processor/yandex-import/post-processor.hcl2spec.go create mode 100644 post-processor/yandex-import/storage.go diff --git a/command/plugin.go b/command/plugin.go index 96d28ae45..82a4ad016 100644 --- a/command/plugin.go +++ b/command/plugin.go @@ -84,6 +84,7 @@ import ( vspherepostprocessor "github.com/hashicorp/packer/post-processor/vsphere" vspheretemplatepostprocessor "github.com/hashicorp/packer/post-processor/vsphere-template" yandexexportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-export" + yandeximportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-import" ansibleprovisioner "github.com/hashicorp/packer/provisioner/ansible" ansiblelocalprovisioner "github.com/hashicorp/packer/provisioner/ansible-local" azuredtlartifactprovisioner "github.com/hashicorp/packer/provisioner/azure-dtlartifact" @@ -204,6 +205,7 @@ var PostProcessors = map[string]packer.PostProcessor{ "vsphere": new(vspherepostprocessor.PostProcessor), "vsphere-template": new(vspheretemplatepostprocessor.PostProcessor), "yandex-export": new(yandexexportpostprocessor.PostProcessor), + "yandex-import": new(yandeximportpostprocessor.PostProcessor), } var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)") diff --git a/post-processor/yandex-import/post-processor.go b/post-processor/yandex-import/post-processor.go new file mode 100644 index 000000000..50383de14 --- /dev/null +++ b/post-processor/yandex-import/post-processor.go @@ -0,0 +1,309 @@ +//go:generate mapstructure-to-hcl2 -type Config + +package yandeximport + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/hashicorp/packer/builder/yandex" + "github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1" + + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer/builder/file" + "github.com/hashicorp/packer/common" + "github.com/hashicorp/packer/helper/config" + "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/post-processor/artifice" + "github.com/hashicorp/packer/post-processor/compress" + "github.com/hashicorp/packer/template/interpolate" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + // The folder ID that will be used to store imported Image. + // Alternatively you may set value by environment variable YC_FOLDER_ID. + FolderID string `mapstructure:"folder_id" required:"true"` + // Service Account ID with proper permission to use Storage service + // for operations 'upload' and 'delete' object to `bucket` + ServiceAccountID string `mapstructure:"service_account_id" required:"true"` + + // OAuth token to use to authenticate to Yandex.Cloud. Alternatively you may set + // value by environment variable YC_TOKEN. + Token string `mapstructure:"token" required:"false"` + // Path to file with Service Account key in json format. This + // is an alternative method to authenticate to Yandex.Cloud. Alternatively you may set environment variable + // YC_SERVICE_ACCOUNT_KEY_FILE. + ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"` + + // The name of the bucket where the qcow2 file will be copied to for import. + // This bucket must exist when the post-processor is run. + Bucket string `mapstructure:"bucket" required:"true"` + // The name of the object key in + // `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine). + // Therefore, you may use user variables and template functions in this field. + ObjectName string `mapstructure:"object_name" required:"false"` + // Whether skip removing the qcow2 file uploaded to Storage + // after the import process has completed. Possible values are: `true` to + // leave it in the bucket, `false` to remove it. (Default: `false`). + SkipClean bool `mapstructure:"skip_clean" required:"false"` + + // The name of the image, which contains 1-63 characters and only + // supports lowercase English characters, numbers and hyphen. + ImageName string `mapstructure:"image_name" required:"true"` + // The description of the image. + ImageDescription string `mapstructure:"image_description" required:"false"` + // The family name of the imported image. + ImageFamily string `mapstructure:"image_family" required:"false"` + // Key/value pair labels to apply to the imported image. + ImageLabels map[string]string `mapstructure:"image_labels" required:"false"` + + ctx interpolate.Context +} + +type PostProcessor struct { + config Config +} + +func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() } + +func (p *PostProcessor) Configure(raws ...interface{}) error { + err := config.Decode(&p.config, &config.DecodeOpts{ + Interpolate: true, + InterpolateContext: &p.config.ctx, + InterpolateFilter: &interpolate.RenderFilter{ + Exclude: []string{ + "object_name", + }, + }, + }, raws...) + if err != nil { + return err + } + + errs := new(packer.MultiError) + + // Set defaults + if p.config.ObjectName == "" { + p.config.ObjectName = "packer-import-{{timestamp}}.qcow2" + } + + // Check and render object_name + if err = interpolate.Validate(p.config.ObjectName, &p.config.ctx); err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("error parsing object_name template: %s", err)) + } + + // TODO: make common code to check and prepare Yandex.Cloud auth configuration data + + templates := map[string]*string{ + "bucket": &p.config.Bucket, + "object_name": &p.config.ObjectName, + "folder_id": &p.config.FolderID, + } + + for key, ptr := range templates { + if *ptr == "" { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("%s must be set", key)) + } + } + + if len(errs.Errors) > 0 { + return errs + } + + return nil +} + +func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { + generatedData := artifact.State("generated_data") + if generatedData == nil { + // Make sure it's not a nil map so we can assign to it later. + generatedData = make(map[string]interface{}) + } + p.config.ctx.Data = generatedData + + cfg := &yandex.Config{ + Token: p.config.Token, + ServiceAccountKeyFile: p.config.ServiceAccountKeyFile, + } + + client, err := yandex.NewDriverYC(ui, cfg) + if err != nil { + return nil, false, false, err + } + + p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx) + if err != nil { + return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err) + } + + respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{ + ServiceAccountId: p.config.ServiceAccountID, + Description: "this key is for upload image to storage", + }) + if err != nil { + return nil, false, false, err + } + + switch artifact.BuilderId() { + case compress.BuilderId, artifice.BuilderId, file.BuilderId: + break + default: + err := fmt.Errorf( + "Unknown artifact type: %s\nCan only import from Compress, Artifice and File post-processor artifacts.", + artifact.BuilderId()) + return nil, false, false, err + } + + storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret()) + if err != nil { + return nil, false, false, fmt.Errorf("error create object_storage client: %s", err) + } + + rawImageUrl, err := uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName) + if err != nil { + return nil, false, false, err + } + + ycImageArtifact, err := createYCImage(ctx, client, ui, p.config.FolderID, rawImageUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels) + if err != nil { + return nil, false, false, err + } + + if !p.config.SkipClean { + err = deleteFromBucket(storageClient, ui, p.config.Bucket, p.config.ObjectName) + if err != nil { + return nil, false, false, err + } + } + + // cleanup static access keys + _, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{ + AccessKeyId: respWithKey.GetAccessKey().GetId(), + }) + if err != nil { + return nil, false, false, fmt.Errorf("error delete static access key: %s", err) + } + + return ycImageArtifact, false, false, nil +} + +func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) { + ui.Say("Looking for qcow2 file in list of artifacts...") + source := "" + for _, path := range artifact.Files() { + ui.Say(fmt.Sprintf("Found artifact %v...", path)) + if strings.HasSuffix(path, ".qcow2") { + source = path + break + } + } + + if source == "" { + return "", fmt.Errorf("no qcow2 file found in list of artifacts") + } + + artifactFile, err := os.Open(source) + if err != nil { + err := fmt.Errorf("error opening %v", source) + return "", err + } + + ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName)) + + _, err = s3conn.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectName), + Body: artifactFile, + }) + + if err != nil { + ui.Say(fmt.Sprintf("Failed to upload: %v", objectName)) + return "", err + } + + req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectName), + }) + + // Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process + req.Config.S3ForcePathStyle = aws.Bool(true) + + urlStr, _, err := req.PresignRequest(15 * time.Minute) + + return urlStr, nil +} + +func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (packer.Artifact, error) { + op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, &compute.CreateImageRequest{ + FolderId: folderID, + Name: imageName, + Description: imageDescription, + Labels: imageLabels, + Family: imageFamily, + Source: &compute.CreateImageRequest_Uri{Uri: rawImageURL}, + })) + if err != nil { + ui.Say("Error creating Yandex Compute Image") + return nil, err + } + + ui.Say(fmt.Sprintf("Source url for Image creation: %v", rawImageURL)) + + ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id())) + + ui.Say("Waiting for Yandex Compute Image creation operation to complete...") + err = op.Wait(ctx) + + // fail if image creation operation has an error + if err != nil { + return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err) + } + + protoMetadata, err := op.Metadata() + if err != nil { + return nil, fmt.Errorf("error while get image create operation metadata: %s", err) + } + + md, ok := protoMetadata.(*compute.CreateImageMetadata) + if !ok { + return nil, fmt.Errorf("could not get Image ID from create operation metadata") + } + + image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{ + ImageId: md.ImageId, + }) + if err != nil { + return nil, fmt.Errorf("error while image get request: %s", err) + } + + return &yandex.Artifact{ + Image: image, + }, nil +} + +func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, bucket string, objectName string) error { + ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName)) + + _, err := s3conn.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectName), + }) + if err != nil { + ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName)) + return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err) + } + + return nil +} diff --git a/post-processor/yandex-import/post-processor.hcl2spec.go b/post-processor/yandex-import/post-processor.hcl2spec.go new file mode 100644 index 000000000..68d34c2f0 --- /dev/null +++ b/post-processor/yandex-import/post-processor.hcl2spec.go @@ -0,0 +1,64 @@ +// Code generated by "mapstructure-to-hcl2 -type Config"; DO NOT EDIT. +package yandeximport + +import ( + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +// FlatConfig is an auto-generated flat version of Config. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatConfig struct { + PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"` + PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"` + PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"` + PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"` + PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"` + PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"` + PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"` + FolderID *string `mapstructure:"folder_id" required:"true" cty:"folder_id" hcl:"folder_id"` + ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"` + Token *string `mapstructure:"token" required:"false" cty:"token" hcl:"token"` + ServiceAccountKeyFile *string `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"` + Bucket *string `mapstructure:"bucket" required:"true" cty:"bucket" hcl:"bucket"` + ObjectName *string `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"` + SkipClean *bool `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"` + ImageName *string `mapstructure:"image_name" required:"true" cty:"image_name" hcl:"image_name"` + ImageDescription *string `mapstructure:"image_description" required:"false" cty:"image_description" hcl:"image_description"` + ImageFamily *string `mapstructure:"image_family" required:"false" cty:"image_family" hcl:"image_family"` + ImageLabels map[string]string `mapstructure:"image_labels" required:"false" cty:"image_labels" hcl:"image_labels"` +} + +// FlatMapstructure returns a new FlatConfig. +// FlatConfig is an auto-generated flat version of Config. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatConfig) +} + +// HCL2Spec returns the hcl spec of a Config. +// This spec is used by HCL to read the fields of Config. +// The decoded values from this spec will then be applied to a FlatConfig. +func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false}, + "packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false}, + "packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false}, + "packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false}, + "packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false}, + "packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false}, + "packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false}, + "folder_id": &hcldec.AttrSpec{Name: "folder_id", Type: cty.String, Required: false}, + "service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false}, + "token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false}, + "service_account_key_file": &hcldec.AttrSpec{Name: "service_account_key_file", Type: cty.String, Required: false}, + "bucket": &hcldec.AttrSpec{Name: "bucket", Type: cty.String, Required: false}, + "object_name": &hcldec.AttrSpec{Name: "object_name", Type: cty.String, Required: false}, + "skip_clean": &hcldec.AttrSpec{Name: "skip_clean", Type: cty.Bool, Required: false}, + "image_name": &hcldec.AttrSpec{Name: "image_name", Type: cty.String, Required: false}, + "image_description": &hcldec.AttrSpec{Name: "image_description", Type: cty.String, Required: false}, + "image_family": &hcldec.AttrSpec{Name: "image_family", Type: cty.String, Required: false}, + "image_labels": &hcldec.AttrSpec{Name: "image_labels", Type: cty.Map(cty.String), Required: false}, + } + return s +} diff --git a/post-processor/yandex-import/storage.go b/post-processor/yandex-import/storage.go new file mode 100644 index 000000000..006afcaca --- /dev/null +++ b/post-processor/yandex-import/storage.go @@ -0,0 +1,42 @@ +package yandeximport + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +const defaultS3Region = "ru-central1" +const defaultStorageEndpoint = "storage.yandexcloud.net" + +func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, error) { + var creds *credentials.Credentials + + if storageEndpoint == "" { + storageEndpoint = defaultStorageEndpoint + } + + s3Config := &aws.Config{ + Endpoint: aws.String(storageEndpoint), + Region: aws.String(defaultS3Region), + } + + switch { + case accessKey != "" && secretKey != "": + creds = credentials.NewStaticCredentials(accessKey, secretKey, "") + default: + return nil, fmt.Errorf("either access or secret key not provided") + } + + s3Config.Credentials = creds + newSession, err := session.NewSession(s3Config) + + if err != nil { + return nil, err + } + + return s3.New(newSession), nil +}