Add yandex-import post-processor
This commit is contained in:
Gennady Lipenkov
parent
cc1605fe94
commit
553d203e68
|
|
@ -84,6 +84,7 @@ import (
|
|||
vspherepostprocessor "github.com/hashicorp/packer/post-processor/vsphere"
|
||||
vspheretemplatepostprocessor "github.com/hashicorp/packer/post-processor/vsphere-template"
|
||||
yandexexportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-export"
|
||||
yandeximportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-import"
|
||||
ansibleprovisioner "github.com/hashicorp/packer/provisioner/ansible"
|
||||
ansiblelocalprovisioner "github.com/hashicorp/packer/provisioner/ansible-local"
|
||||
azuredtlartifactprovisioner "github.com/hashicorp/packer/provisioner/azure-dtlartifact"
|
||||
|
|
@ -204,6 +205,7 @@ var PostProcessors = map[string]packer.PostProcessor{
|
|||
"vsphere": new(vspherepostprocessor.PostProcessor),
|
||||
"vsphere-template": new(vspheretemplatepostprocessor.PostProcessor),
|
||||
"yandex-export": new(yandexexportpostprocessor.PostProcessor),
|
||||
"yandex-import": new(yandeximportpostprocessor.PostProcessor),
|
||||
}
|
||||
|
||||
var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,309 @@
|
|||
//go:generate mapstructure-to-hcl2 -type Config
|
||||
|
||||
package yandeximport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/hashicorp/packer/builder/yandex"
|
||||
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
|
||||
|
||||
"github.com/hashicorp/hcl/v2/hcldec"
|
||||
"github.com/hashicorp/packer/builder/file"
|
||||
"github.com/hashicorp/packer/common"
|
||||
"github.com/hashicorp/packer/helper/config"
|
||||
"github.com/hashicorp/packer/packer"
|
||||
"github.com/hashicorp/packer/post-processor/artifice"
|
||||
"github.com/hashicorp/packer/post-processor/compress"
|
||||
"github.com/hashicorp/packer/template/interpolate"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
common.PackerConfig `mapstructure:",squash"`
|
||||
|
||||
// The folder ID that will be used to store imported Image.
|
||||
// Alternatively you may set value by environment variable YC_FOLDER_ID.
|
||||
FolderID string `mapstructure:"folder_id" required:"true"`
|
||||
// Service Account ID with proper permission to use Storage service
|
||||
// for operations 'upload' and 'delete' object to `bucket`
|
||||
ServiceAccountID string `mapstructure:"service_account_id" required:"true"`
|
||||
|
||||
// OAuth token to use to authenticate to Yandex.Cloud. Alternatively you may set
|
||||
// value by environment variable YC_TOKEN.
|
||||
Token string `mapstructure:"token" required:"false"`
|
||||
// Path to file with Service Account key in json format. This
|
||||
// is an alternative method to authenticate to Yandex.Cloud. Alternatively you may set environment variable
|
||||
// YC_SERVICE_ACCOUNT_KEY_FILE.
|
||||
ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"`
|
||||
|
||||
// The name of the bucket where the qcow2 file will be copied to for import.
|
||||
// This bucket must exist when the post-processor is run.
|
||||
Bucket string `mapstructure:"bucket" required:"true"`
|
||||
// The name of the object key in
|
||||
// `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
|
||||
// Therefore, you may use user variables and template functions in this field.
|
||||
ObjectName string `mapstructure:"object_name" required:"false"`
|
||||
// Whether skip removing the qcow2 file uploaded to Storage
|
||||
// after the import process has completed. Possible values are: `true` to
|
||||
// leave it in the bucket, `false` to remove it. (Default: `false`).
|
||||
SkipClean bool `mapstructure:"skip_clean" required:"false"`
|
||||
|
||||
// The name of the image, which contains 1-63 characters and only
|
||||
// supports lowercase English characters, numbers and hyphen.
|
||||
ImageName string `mapstructure:"image_name" required:"true"`
|
||||
// The description of the image.
|
||||
ImageDescription string `mapstructure:"image_description" required:"false"`
|
||||
// The family name of the imported image.
|
||||
ImageFamily string `mapstructure:"image_family" required:"false"`
|
||||
// Key/value pair labels to apply to the imported image.
|
||||
ImageLabels map[string]string `mapstructure:"image_labels" required:"false"`
|
||||
|
||||
ctx interpolate.Context
|
||||
}
|
||||
|
||||
type PostProcessor struct {
|
||||
config Config
|
||||
}
|
||||
|
||||
func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() }
|
||||
|
||||
func (p *PostProcessor) Configure(raws ...interface{}) error {
|
||||
err := config.Decode(&p.config, &config.DecodeOpts{
|
||||
Interpolate: true,
|
||||
InterpolateContext: &p.config.ctx,
|
||||
InterpolateFilter: &interpolate.RenderFilter{
|
||||
Exclude: []string{
|
||||
"object_name",
|
||||
},
|
||||
},
|
||||
}, raws...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errs := new(packer.MultiError)
|
||||
|
||||
// Set defaults
|
||||
if p.config.ObjectName == "" {
|
||||
p.config.ObjectName = "packer-import-{{timestamp}}.qcow2"
|
||||
}
|
||||
|
||||
// Check and render object_name
|
||||
if err = interpolate.Validate(p.config.ObjectName, &p.config.ctx); err != nil {
|
||||
errs = packer.MultiErrorAppend(
|
||||
errs, fmt.Errorf("error parsing object_name template: %s", err))
|
||||
}
|
||||
|
||||
// TODO: make common code to check and prepare Yandex.Cloud auth configuration data
|
||||
|
||||
templates := map[string]*string{
|
||||
"bucket": &p.config.Bucket,
|
||||
"object_name": &p.config.ObjectName,
|
||||
"folder_id": &p.config.FolderID,
|
||||
}
|
||||
|
||||
for key, ptr := range templates {
|
||||
if *ptr == "" {
|
||||
errs = packer.MultiErrorAppend(
|
||||
errs, fmt.Errorf("%s must be set", key))
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs.Errors) > 0 {
|
||||
return errs
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) {
|
||||
generatedData := artifact.State("generated_data")
|
||||
if generatedData == nil {
|
||||
// Make sure it's not a nil map so we can assign to it later.
|
||||
generatedData = make(map[string]interface{})
|
||||
}
|
||||
p.config.ctx.Data = generatedData
|
||||
|
||||
cfg := &yandex.Config{
|
||||
Token: p.config.Token,
|
||||
ServiceAccountKeyFile: p.config.ServiceAccountKeyFile,
|
||||
}
|
||||
|
||||
client, err := yandex.NewDriverYC(ui, cfg)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx)
|
||||
if err != nil {
|
||||
return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
|
||||
}
|
||||
|
||||
respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
|
||||
ServiceAccountId: p.config.ServiceAccountID,
|
||||
Description: "this key is for upload image to storage",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
switch artifact.BuilderId() {
|
||||
case compress.BuilderId, artifice.BuilderId, file.BuilderId:
|
||||
break
|
||||
default:
|
||||
err := fmt.Errorf(
|
||||
"Unknown artifact type: %s\nCan only import from Compress, Artifice and File post-processor artifacts.",
|
||||
artifact.BuilderId())
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
|
||||
if err != nil {
|
||||
return nil, false, false, fmt.Errorf("error create object_storage client: %s", err)
|
||||
}
|
||||
|
||||
rawImageUrl, err := uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
ycImageArtifact, err := createYCImage(ctx, client, ui, p.config.FolderID, rawImageUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
if !p.config.SkipClean {
|
||||
err = deleteFromBucket(storageClient, ui, p.config.Bucket, p.config.ObjectName)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
}
|
||||
|
||||
// cleanup static access keys
|
||||
_, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{
|
||||
AccessKeyId: respWithKey.GetAccessKey().GetId(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, false, fmt.Errorf("error delete static access key: %s", err)
|
||||
}
|
||||
|
||||
return ycImageArtifact, false, false, nil
|
||||
}
|
||||
|
||||
func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
|
||||
ui.Say("Looking for qcow2 file in list of artifacts...")
|
||||
source := ""
|
||||
for _, path := range artifact.Files() {
|
||||
ui.Say(fmt.Sprintf("Found artifact %v...", path))
|
||||
if strings.HasSuffix(path, ".qcow2") {
|
||||
source = path
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if source == "" {
|
||||
return "", fmt.Errorf("no qcow2 file found in list of artifacts")
|
||||
}
|
||||
|
||||
artifactFile, err := os.Open(source)
|
||||
if err != nil {
|
||||
err := fmt.Errorf("error opening %v", source)
|
||||
return "", err
|
||||
}
|
||||
|
||||
ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName))
|
||||
|
||||
_, err = s3conn.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(objectName),
|
||||
Body: artifactFile,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
ui.Say(fmt.Sprintf("Failed to upload: %v", objectName))
|
||||
return "", err
|
||||
}
|
||||
|
||||
req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(objectName),
|
||||
})
|
||||
|
||||
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
|
||||
req.Config.S3ForcePathStyle = aws.Bool(true)
|
||||
|
||||
urlStr, _, err := req.PresignRequest(15 * time.Minute)
|
||||
|
||||
return urlStr, nil
|
||||
}
|
||||
|
||||
func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (packer.Artifact, error) {
|
||||
op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, &compute.CreateImageRequest{
|
||||
FolderId: folderID,
|
||||
Name: imageName,
|
||||
Description: imageDescription,
|
||||
Labels: imageLabels,
|
||||
Family: imageFamily,
|
||||
Source: &compute.CreateImageRequest_Uri{Uri: rawImageURL},
|
||||
}))
|
||||
if err != nil {
|
||||
ui.Say("Error creating Yandex Compute Image")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ui.Say(fmt.Sprintf("Source url for Image creation: %v", rawImageURL))
|
||||
|
||||
ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id()))
|
||||
|
||||
ui.Say("Waiting for Yandex Compute Image creation operation to complete...")
|
||||
err = op.Wait(ctx)
|
||||
|
||||
// fail if image creation operation has an error
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err)
|
||||
}
|
||||
|
||||
protoMetadata, err := op.Metadata()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error while get image create operation metadata: %s", err)
|
||||
}
|
||||
|
||||
md, ok := protoMetadata.(*compute.CreateImageMetadata)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("could not get Image ID from create operation metadata")
|
||||
}
|
||||
|
||||
image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
|
||||
ImageId: md.ImageId,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error while image get request: %s", err)
|
||||
}
|
||||
|
||||
return &yandex.Artifact{
|
||||
Image: image,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, bucket string, objectName string) error {
|
||||
ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
|
||||
|
||||
_, err := s3conn.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(objectName),
|
||||
})
|
||||
if err != nil {
|
||||
ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName))
|
||||
return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
// Code generated by "mapstructure-to-hcl2 -type Config"; DO NOT EDIT.
|
||||
package yandeximport
|
||||
|
||||
import (
|
||||
"github.com/hashicorp/hcl/v2/hcldec"
|
||||
"github.com/zclconf/go-cty/cty"
|
||||
)
|
||||
|
||||
// FlatConfig is an auto-generated flat version of Config.
|
||||
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
|
||||
type FlatConfig struct {
|
||||
PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"`
|
||||
PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"`
|
||||
PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"`
|
||||
PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"`
|
||||
PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"`
|
||||
PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
|
||||
PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
|
||||
FolderID *string `mapstructure:"folder_id" required:"true" cty:"folder_id" hcl:"folder_id"`
|
||||
ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"`
|
||||
Token *string `mapstructure:"token" required:"false" cty:"token" hcl:"token"`
|
||||
ServiceAccountKeyFile *string `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"`
|
||||
Bucket *string `mapstructure:"bucket" required:"true" cty:"bucket" hcl:"bucket"`
|
||||
ObjectName *string `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"`
|
||||
SkipClean *bool `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"`
|
||||
ImageName *string `mapstructure:"image_name" required:"true" cty:"image_name" hcl:"image_name"`
|
||||
ImageDescription *string `mapstructure:"image_description" required:"false" cty:"image_description" hcl:"image_description"`
|
||||
ImageFamily *string `mapstructure:"image_family" required:"false" cty:"image_family" hcl:"image_family"`
|
||||
ImageLabels map[string]string `mapstructure:"image_labels" required:"false" cty:"image_labels" hcl:"image_labels"`
|
||||
}
|
||||
|
||||
// FlatMapstructure returns a new FlatConfig.
|
||||
// FlatConfig is an auto-generated flat version of Config.
|
||||
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
|
||||
func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
|
||||
return new(FlatConfig)
|
||||
}
|
||||
|
||||
// HCL2Spec returns the hcl spec of a Config.
|
||||
// This spec is used by HCL to read the fields of Config.
|
||||
// The decoded values from this spec will then be applied to a FlatConfig.
|
||||
func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
|
||||
s := map[string]hcldec.Spec{
|
||||
"packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false},
|
||||
"packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false},
|
||||
"packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false},
|
||||
"packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false},
|
||||
"packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false},
|
||||
"packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
|
||||
"packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
|
||||
"folder_id": &hcldec.AttrSpec{Name: "folder_id", Type: cty.String, Required: false},
|
||||
"service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false},
|
||||
"token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false},
|
||||
"service_account_key_file": &hcldec.AttrSpec{Name: "service_account_key_file", Type: cty.String, Required: false},
|
||||
"bucket": &hcldec.AttrSpec{Name: "bucket", Type: cty.String, Required: false},
|
||||
"object_name": &hcldec.AttrSpec{Name: "object_name", Type: cty.String, Required: false},
|
||||
"skip_clean": &hcldec.AttrSpec{Name: "skip_clean", Type: cty.Bool, Required: false},
|
||||
"image_name": &hcldec.AttrSpec{Name: "image_name", Type: cty.String, Required: false},
|
||||
"image_description": &hcldec.AttrSpec{Name: "image_description", Type: cty.String, Required: false},
|
||||
"image_family": &hcldec.AttrSpec{Name: "image_family", Type: cty.String, Required: false},
|
||||
"image_labels": &hcldec.AttrSpec{Name: "image_labels", Type: cty.Map(cty.String), Required: false},
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package yandeximport
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
const defaultS3Region = "ru-central1"
|
||||
const defaultStorageEndpoint = "storage.yandexcloud.net"
|
||||
|
||||
func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, error) {
|
||||
var creds *credentials.Credentials
|
||||
|
||||
if storageEndpoint == "" {
|
||||
storageEndpoint = defaultStorageEndpoint
|
||||
}
|
||||
|
||||
s3Config := &aws.Config{
|
||||
Endpoint: aws.String(storageEndpoint),
|
||||
Region: aws.String(defaultS3Region),
|
||||
}
|
||||
|
||||
switch {
|
||||
case accessKey != "" && secretKey != "":
|
||||
creds = credentials.NewStaticCredentials(accessKey, secretKey, "")
|
||||
default:
|
||||
return nil, fmt.Errorf("either access or secret key not provided")
|
||||
}
|
||||
|
||||
s3Config.Credentials = creds
|
||||
newSession, err := session.NewSession(s3Config)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s3.New(newSession), nil
|
||||
}
|
||||
Loading…
Reference in New Issue