Merge pull request #9553 from GennadySpb/yandex-import-post-processor

[WIP] Yandex Import post-processor
This commit is contained in:
Megan Marsh 2020-07-08 16:25:08 -07:00 committed by GitHub
commit e08c1a461a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 537 additions and 9 deletions

View File

@ -1,5 +1,8 @@
## 1.6.1 (Upcoming)
### FEATURES:
* **New post-processor** Yandex Import [GH-9553]
### IMPROVEMENTS:
* builder/file: Create parent directories of target file, if they don't exist.
[GH-9452]

View File

@ -90,5 +90,6 @@
/post-processor/exoscale-import/ @falzm @mcorbin
/post-processor/googlecompute-export/ crunkleton@google.com
/post-processor/yandex-export/ @GennadySpb
/post-processor/yandex-import/ @GennadySpb
/post-processor/vsphere-template/ nelson@bennu.cl
/post-processor/ucloud-import/ @shawnmssu

View File

@ -9,7 +9,7 @@ import (
type Artifact struct {
config *Config
driver Driver
image *compute.Image
Image *compute.Image
// StateData should store data such as GeneratedData
// to be shared with post-processors
@ -22,7 +22,7 @@ func (*Artifact) BuilderId() string {
}
func (a *Artifact) Id() string {
return a.image.Id
return a.Image.Id
}
func (*Artifact) Files() []string {
@ -31,7 +31,7 @@ func (*Artifact) Files() []string {
//revive:enable:var-naming
func (a *Artifact) String() string {
return fmt.Sprintf("A disk image was created: %v (id: %v) with family name %v", a.image.Name, a.image.Id, a.image.Family)
return fmt.Sprintf("A disk image was created: %v (id: %v) with family name %v", a.Image.Name, a.Image.Id, a.Image.Family)
}
func (a *Artifact) State(name string) interface{} {
@ -41,14 +41,14 @@ func (a *Artifact) State(name string) interface{} {
switch name {
case "ImageID":
return a.image.Id
return a.Image.Id
case "FolderID":
return a.image.FolderId
return a.Image.FolderId
}
return nil
}
func (a *Artifact) Destroy() error {
return a.driver.DeleteImage(a.image.Id)
return a.driver.DeleteImage(a.Image.Id)
}

View File

@ -18,7 +18,7 @@ func TestArtifact_Id(t *testing.T) {
FolderId: "test-folder-id",
}
a := &Artifact{
image: i}
Image: i}
expected := "test-id-value"
if a.Id() != expected {
@ -34,7 +34,7 @@ func TestArtifact_String(t *testing.T) {
Family: "test-family",
}
a := &Artifact{
image: i}
Image: i}
expected := "A disk image was created: test-name (id: test-id-value) with family name test-family"
if a.String() != expected {

View File

@ -91,7 +91,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
}
artifact := &Artifact{
image: image.(*compute.Image),
Image: image.(*compute.Image),
config: &b.config,
driver: driver,
StateData: map[string]interface{}{"generated_data": state.Get("generated_data")},

View File

@ -84,6 +84,7 @@ import (
vspherepostprocessor "github.com/hashicorp/packer/post-processor/vsphere"
vspheretemplatepostprocessor "github.com/hashicorp/packer/post-processor/vsphere-template"
yandexexportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-export"
yandeximportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-import"
ansibleprovisioner "github.com/hashicorp/packer/provisioner/ansible"
ansiblelocalprovisioner "github.com/hashicorp/packer/provisioner/ansible-local"
azuredtlartifactprovisioner "github.com/hashicorp/packer/provisioner/azure-dtlartifact"
@ -204,6 +205,7 @@ var PostProcessors = map[string]packer.PostProcessor{
"vsphere": new(vspherepostprocessor.PostProcessor),
"vsphere-template": new(vspheretemplatepostprocessor.PostProcessor),
"yandex-export": new(yandexexportpostprocessor.PostProcessor),
"yandex-import": new(yandeximportpostprocessor.PostProcessor),
}
var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")

View File

@ -0,0 +1,311 @@
//go:generate struct-markdown
//go:generate mapstructure-to-hcl2 -type Config
package yandeximport
import (
"context"
"fmt"
"os"
"strings"
"time"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/packer/builder/yandex"
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/packer/builder/file"
"github.com/hashicorp/packer/common"
"github.com/hashicorp/packer/helper/config"
"github.com/hashicorp/packer/packer"
"github.com/hashicorp/packer/post-processor/artifice"
"github.com/hashicorp/packer/post-processor/compress"
"github.com/hashicorp/packer/template/interpolate"
)
type Config struct {
common.PackerConfig `mapstructure:",squash"`
// The folder ID that will be used to store imported Image.
FolderID string `mapstructure:"folder_id" required:"true"`
// Service Account ID with proper permission to use Storage service
// for operations 'upload' and 'delete' object to `bucket`
ServiceAccountID string `mapstructure:"service_account_id" required:"true"`
// OAuth token to use to authenticate to Yandex.Cloud.
Token string `mapstructure:"token" required:"false"`
// Path to file with Service Account key in json format. This
// is an alternative method to authenticate to Yandex.Cloud.
ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"`
// The name of the bucket where the qcow2 file will be copied to for import.
// This bucket must exist when the post-processor is run.
Bucket string `mapstructure:"bucket" required:"true"`
// The name of the object key in
// `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
// Therefore, you may use user variables and template functions in this field.
ObjectName string `mapstructure:"object_name" required:"false"`
// Whether skip removing the qcow2 file uploaded to Storage
// after the import process has completed. Possible values are: `true` to
// leave it in the bucket, `false` to remove it. (Default: `false`).
SkipClean bool `mapstructure:"skip_clean" required:"false"`
// The name of the image, which contains 1-63 characters and only
// supports lowercase English characters, numbers and hyphen.
ImageName string `mapstructure:"image_name" required:"false"`
// The description of the image.
ImageDescription string `mapstructure:"image_description" required:"false"`
// The family name of the imported image.
ImageFamily string `mapstructure:"image_family" required:"false"`
// Key/value pair labels to apply to the imported image.
ImageLabels map[string]string `mapstructure:"image_labels" required:"false"`
ctx interpolate.Context
}
type PostProcessor struct {
config Config
}
func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() }
func (p *PostProcessor) Configure(raws ...interface{}) error {
err := config.Decode(&p.config, &config.DecodeOpts{
Interpolate: true,
InterpolateContext: &p.config.ctx,
InterpolateFilter: &interpolate.RenderFilter{
Exclude: []string{
"object_name",
},
},
}, raws...)
if err != nil {
return err
}
errs := new(packer.MultiError)
// Set defaults
if p.config.ObjectName == "" {
p.config.ObjectName = "packer-import-{{timestamp}}.qcow2"
}
// Check and render object_name
if err = interpolate.Validate(p.config.ObjectName, &p.config.ctx); err != nil {
errs = packer.MultiErrorAppend(
errs, fmt.Errorf("error parsing object_name template: %s", err))
}
// TODO: make common code to check and prepare Yandex.Cloud auth configuration data
templates := map[string]*string{
"bucket": &p.config.Bucket,
"object_name": &p.config.ObjectName,
"folder_id": &p.config.FolderID,
}
for key, ptr := range templates {
if *ptr == "" {
errs = packer.MultiErrorAppend(
errs, fmt.Errorf("%s must be set", key))
}
}
if len(errs.Errors) > 0 {
return errs
}
return nil
}
func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) {
generatedData := artifact.State("generated_data")
if generatedData == nil {
// Make sure it's not a nil map so we can assign to it later.
generatedData = make(map[string]interface{})
}
p.config.ctx.Data = generatedData
cfg := &yandex.Config{
Token: p.config.Token,
ServiceAccountKeyFile: p.config.ServiceAccountKeyFile,
}
client, err := yandex.NewDriverYC(ui, cfg)
if err != nil {
return nil, false, false, err
}
p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx)
if err != nil {
return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
}
respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
ServiceAccountId: p.config.ServiceAccountID,
Description: "this key is for upload image to storage",
})
if err != nil {
return nil, false, false, err
}
switch artifact.BuilderId() {
case compress.BuilderId, artifice.BuilderId, file.BuilderId:
break
default:
err := fmt.Errorf(
"Unknown artifact type: %s\nCan only import from Compress, Artifice and File post-processor artifacts.",
artifact.BuilderId())
return nil, false, false, err
}
storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
if err != nil {
return nil, false, false, fmt.Errorf("error create object_storage client: %s", err)
}
rawImageUrl, err := uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
if err != nil {
return nil, false, false, err
}
ycImageArtifact, err := createYCImage(ctx, client, ui, p.config.FolderID, rawImageUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
if err != nil {
return nil, false, false, err
}
if !p.config.SkipClean {
err = deleteFromBucket(storageClient, ui, p.config.Bucket, p.config.ObjectName)
if err != nil {
return nil, false, false, err
}
}
// cleanup static access keys
_, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{
AccessKeyId: respWithKey.GetAccessKey().GetId(),
})
if err != nil {
return nil, false, false, fmt.Errorf("error delete static access key: %s", err)
}
return ycImageArtifact, false, false, nil
}
func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
ui.Say("Looking for qcow2 file in list of artifacts...")
source := ""
for _, path := range artifact.Files() {
ui.Say(fmt.Sprintf("Found artifact %v...", path))
if strings.HasSuffix(path, ".qcow2") {
source = path
break
}
}
if source == "" {
return "", fmt.Errorf("no qcow2 file found in list of artifacts")
}
artifactFile, err := os.Open(source)
if err != nil {
err := fmt.Errorf("error opening %v", source)
return "", err
}
ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName))
_, err = s3conn.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
Body: artifactFile,
})
if err != nil {
ui.Say(fmt.Sprintf("Failed to upload: %v", objectName))
return "", err
}
req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
})
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
req.Config.S3ForcePathStyle = aws.Bool(true)
urlStr, _, err := req.PresignRequest(15 * time.Minute)
if err != nil {
ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
return "", err
}
return urlStr, nil
}
func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (packer.Artifact, error) {
op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, &compute.CreateImageRequest{
FolderId: folderID,
Name: imageName,
Description: imageDescription,
Labels: imageLabels,
Family: imageFamily,
Source: &compute.CreateImageRequest_Uri{Uri: rawImageURL},
}))
if err != nil {
ui.Say("Error creating Yandex Compute Image")
return nil, err
}
ui.Say(fmt.Sprintf("Source url for Image creation: %v", rawImageURL))
ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id()))
ui.Say("Waiting for Yandex Compute Image creation operation to complete...")
err = op.Wait(ctx)
// fail if image creation operation has an error
if err != nil {
return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err)
}
protoMetadata, err := op.Metadata()
if err != nil {
return nil, fmt.Errorf("error while get image create operation metadata: %s", err)
}
md, ok := protoMetadata.(*compute.CreateImageMetadata)
if !ok {
return nil, fmt.Errorf("could not get Image ID from create operation metadata")
}
image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
ImageId: md.ImageId,
})
if err != nil {
return nil, fmt.Errorf("error while image get request: %s", err)
}
return &yandex.Artifact{
Image: image,
}, nil
}
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, bucket string, objectName string) error {
ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
_, err := s3conn.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
})
if err != nil {
ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName))
return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err)
}
return nil
}

View File

@ -0,0 +1,64 @@
// Code generated by "mapstructure-to-hcl2 -type Config"; DO NOT EDIT.
package yandeximport
import (
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/zclconf/go-cty/cty"
)
// FlatConfig is an auto-generated flat version of Config.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatConfig struct {
PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"`
PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"`
PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"`
PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"`
PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"`
PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
FolderID *string `mapstructure:"folder_id" required:"true" cty:"folder_id" hcl:"folder_id"`
ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"`
Token *string `mapstructure:"token" required:"false" cty:"token" hcl:"token"`
ServiceAccountKeyFile *string `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"`
Bucket *string `mapstructure:"bucket" required:"true" cty:"bucket" hcl:"bucket"`
ObjectName *string `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"`
SkipClean *bool `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"`
ImageName *string `mapstructure:"image_name" required:"false" cty:"image_name" hcl:"image_name"`
ImageDescription *string `mapstructure:"image_description" required:"false" cty:"image_description" hcl:"image_description"`
ImageFamily *string `mapstructure:"image_family" required:"false" cty:"image_family" hcl:"image_family"`
ImageLabels map[string]string `mapstructure:"image_labels" required:"false" cty:"image_labels" hcl:"image_labels"`
}
// FlatMapstructure returns a new FlatConfig.
// FlatConfig is an auto-generated flat version of Config.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
return new(FlatConfig)
}
// HCL2Spec returns the hcl spec of a Config.
// This spec is used by HCL to read the fields of Config.
// The decoded values from this spec will then be applied to a FlatConfig.
func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
s := map[string]hcldec.Spec{
"packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false},
"packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false},
"packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false},
"packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false},
"packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false},
"packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
"packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
"folder_id": &hcldec.AttrSpec{Name: "folder_id", Type: cty.String, Required: false},
"service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false},
"token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false},
"service_account_key_file": &hcldec.AttrSpec{Name: "service_account_key_file", Type: cty.String, Required: false},
"bucket": &hcldec.AttrSpec{Name: "bucket", Type: cty.String, Required: false},
"object_name": &hcldec.AttrSpec{Name: "object_name", Type: cty.String, Required: false},
"skip_clean": &hcldec.AttrSpec{Name: "skip_clean", Type: cty.Bool, Required: false},
"image_name": &hcldec.AttrSpec{Name: "image_name", Type: cty.String, Required: false},
"image_description": &hcldec.AttrSpec{Name: "image_description", Type: cty.String, Required: false},
"image_family": &hcldec.AttrSpec{Name: "image_family", Type: cty.String, Required: false},
"image_labels": &hcldec.AttrSpec{Name: "image_labels", Type: cty.Map(cty.String), Required: false},
}
return s
}

View File

@ -0,0 +1,42 @@
package yandeximport
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
const defaultS3Region = "ru-central1"
const defaultStorageEndpoint = "storage.yandexcloud.net"
func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, error) {
var creds *credentials.Credentials
if storageEndpoint == "" {
storageEndpoint = defaultStorageEndpoint
}
s3Config := &aws.Config{
Endpoint: aws.String(storageEndpoint),
Region: aws.String(defaultS3Region),
}
switch {
case accessKey != "" && secretKey != "":
creds = credentials.NewStaticCredentials(accessKey, secretKey, "")
default:
return nil, fmt.Errorf("either access or secret key not provided")
}
s3Config.Credentials = creds
newSession, err := session.NewSession(s3Config)
if err != nil {
return nil, err
}
return s3.New(newSession), nil
}

View File

@ -268,6 +268,7 @@ export default [
'vsphere',
'vsphere-template',
'yandex-export',
'yandex-import',
],
},
'----------',

View File

@ -0,0 +1,70 @@
---
description: >
The Yandex.Cloud Compute Image Import post-processor create an image from a
qcow2 image (or from provided Storage object in near future).
It uploads qcow2 to Yandex Object Storage and create new one Compute Image
in target folder.
layout: docs
page_title: Yandex.Cloud Compute Image Import - Post-Processors
sidebar_title: Yandex.Cloud Compute Import
---
# Yandex.Cloud Compute Image Import Post-Processor
Type: `yandex-import`
The Yandex.Cloud Compute Image Import post-processor create new Compute Image
from a qcow2 file. As Compute service support image creation from Storage service object
just before request to create its upload file into Storage service.
Assigned Service Account must have write permissions to the Yandex Object Storage.
A new temporary static access keys from assigned Service Account used to upload
file.
## Configuration
### Required:
@include 'post-processor/yandex-import/Config-required.mdx'
### Optional:
@include 'post-processor/yandex-import/Config-not-required.mdx'
## Basic Example
TBD
```json
{
"variables": {
"token": "{{env `YC_TOKEN`}}"
},
"sensitive-variables": ["token"],
"builders": [
{
"type": "file",
"source": "xenial-server-cloudimg-amd64-disk1.img",
"target": "test_artifact.qcow2"
}
],
"post-processors": [
{
"type": "yandex-import",
"token": "{{user `token`}}",
"folder_id": "b1g8jvfcgmitdrslcn86",
"service_account_id": "ajeui8kdvg8qs44fbrbr",
"bucket": "bucket1",
"image_name": "my-first-imported-image-{{isotime \"02-Jan-06-03-04-05\" | lower }}",
"keep_input_artifact": false
}
]
}
```

View File

@ -0,0 +1,24 @@
<!-- Code generated from the comments of the Config struct in post-processor/yandex-import/post-processor.go; DO NOT EDIT MANUALLY -->
- `token` (string) - OAuth token to use to authenticate to Yandex.Cloud.
- `service_account_key_file` (string) - Path to file with Service Account key in json format. This
is an alternative method to authenticate to Yandex.Cloud.
- `object_name` (string) - The name of the object key in
`bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
Therefore, you may use user variables and template functions in this field.
- `skip_clean` (bool) - Whether skip removing the qcow2 file uploaded to Storage
after the import process has completed. Possible values are: `true` to
leave it in the bucket, `false` to remove it. (Default: `false`).
- `image_name` (string) - The name of the image, which contains 1-63 characters and only
supports lowercase English characters, numbers and hyphen.
- `image_description` (string) - The description of the image.
- `image_family` (string) - The family name of the imported image.
- `image_labels` (map[string]string) - Key/value pair labels to apply to the imported image.

View File

@ -0,0 +1,10 @@
<!-- Code generated from the comments of the Config struct in post-processor/yandex-import/post-processor.go; DO NOT EDIT MANUALLY -->
- `folder_id` (string) - The folder ID that will be used to store imported Image.
- `service_account_id` (string) - Service Account ID with proper permission to use Storage service
for operations 'upload' and 'delete' object to `bucket`
- `bucket` (string) - The name of the bucket where the qcow2 file will be copied to for import.
This bucket must exist when the post-processor is run.