iSharkFly-Docs
/
packer-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #9601 from GennadySpb/yndx-import-pp-url-input 

yandex-import: support work with URL from yandex-export pp
This commit is contained in:
Megan Marsh 2020-07-17 09:11:25 -07:00 committed by GitHub
parent 6cdfdcc3c5 b16cccde3d
commit e6130afce6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 316 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
post-processor/yandex-export/artifact.go
View File

@ -8,14 +8,15 @@ const BuilderId = "packer.post-processor.yandex-export"
type Artifact struct {
paths []string
urls []string
}
func (*Artifact) BuilderId() string {
return BuilderId
}
func (*Artifact) Id() string {
return ""
func (a *Artifact) Id() string {
return a.urls[0]
}
func (a *Artifact) Files() []string {

21
post-processor/yandex-export/post-processor.go
View File

@ -24,6 +24,8 @@ import (
"github.com/hashicorp/packer/template/interpolate"
)
const defaultStorageEndpoint = "storage.yandexcloud.net"
type Config struct {
common.PackerConfig `mapstructure:",squash"`
@ -31,12 +33,13 @@ type Config struct {
// Please be aware that use of space char inside path not supported.
// Also this param support [build](/docs/templates/engine) template function.
// Check available template data for [Yandex](/docs/builders/yandex#build-template-data) builder.
// Paths to Yandex Object Storage where exported image will be uploaded.
Paths []string `mapstructure:"paths" required:"true"`
// The folder ID that will be used to launch a temporary instance.
// Alternatively you may set value by environment variable YC_FOLDER_ID.
FolderID string `mapstructure:"folder_id" required:"true"`
// Service Account ID with proper permission to modify an instance, create and attach disk and
// make upload to specific Yandex Object Storage paths
// make upload to specific Yandex Object Storage paths.
ServiceAccountID string `mapstructure:"service_account_id" required:"true"`
// The size of the disk in GB. This defaults to `100`, which is 100GB.
DiskSizeGb int `mapstructure:"disk_size" required:"false"`
@ -234,7 +237,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
steps := []multistep.Step{
&yandex.StepCreateSSHKey{
Debug: p.config.PackerDebug,
DebugKeyPath: fmt.Sprintf("yc_pp_%s.pem", p.config.PackerBuildName),
DebugKeyPath: fmt.Sprintf("yc_export_pp_%s.pem", p.config.PackerBuildName),
},
&yandex.StepCreateInstance{
Debug: p.config.PackerDebug,
@ -248,7 +251,10 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)
p.runner.Run(ctx, state)
result := &Artifact{paths: p.config.Paths}
result := &Artifact{
paths: p.config.Paths,
urls: formUrls(p.config.Paths),
}
return result, false, false, nil
}
@ -271,3 +277,12 @@ func ycSaneDefaults() yandex.Config {
StateTimeout: 3 * time.Minute,
}
}
func formUrls(paths []string) []string {
result := []string{}
for _, path := range paths {
url := fmt.Sprintf("https://%s/%s", defaultStorageEndpoint, strings.TrimPrefix(path, "s3://"))
result = append(result, url)
}
return result
}

48
post-processor/yandex-export/post-processor_test.go
View File

@ -3,6 +3,8 @@ package yandexexport
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/hashicorp/packer/helper/multistep"
)
@ -64,3 +66,49 @@ func TestPostProcessor_Configure(t *testing.T) {
})
}
}
func Test_formUrls(t *testing.T) {
type args struct {
paths []string
}
tests := []struct {
name string
args args
wantResult []string
}{
{
name: "empty list",
args: args{
paths: []string{},
},
wantResult: []string{},
},
{
name: "one element",
args: args{
paths: []string{"s3://bucket1/object1"},
},
wantResult: []string{"https://" + defaultStorageEndpoint + "/bucket1/object1"},
},
{
name: "several elements",
args: args{
paths: []string{
"s3://bucket1/object1",
"s3://bucket-name/object-with/prefix/filename.blob",
"s3://bucket-too/foo/bar.test",
},
},
wantResult: []string{
"https://" + defaultStorageEndpoint + "/bucket1/object1",
"https://" + defaultStorageEndpoint + "/bucket-name/object-with/prefix/filename.blob",
"https://" + defaultStorageEndpoint + "/bucket-too/foo/bar.test",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
require.Equal(t, tt.wantResult, formUrls(tt.args.paths))
})
}
}

36
post-processor/yandex-import/artifact.go Normal file
View File

@ -0,0 +1,36 @@
package yandeximport
import (
"fmt"
)
const BuilderId = "packer.post-processor.yandex-import"
type Artifact struct {
imageID string
sourceURL string
}
func (*Artifact) BuilderId() string {
return BuilderId
}
func (a *Artifact) Id() string {
return a.sourceURL
}
func (a *Artifact) Files() []string {
return nil
}
func (a *Artifact) String() string {
return fmt.Sprintf("Create image %v from URL %v", a.imageID, a.sourceURL)
}
func (*Artifact) State(name string) interface{} {
return nil
}
func (a *Artifact) Destroy() error {
return nil
}

125
post-processor/yandex-import/post-processor.go
View File

@ -8,14 +8,13 @@ import (
"fmt"
"os"
"strings"
"time"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/packer/builder/yandex"
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
"github.com/yandex-cloud/go-sdk/iamkey"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/packer/builder/file"
@ -24,6 +23,7 @@ import (
"github.com/hashicorp/packer/packer"
"github.com/hashicorp/packer/post-processor/artifice"
"github.com/hashicorp/packer/post-processor/compress"
yandexexport "github.com/hashicorp/packer/post-processor/yandex-export"
"github.com/hashicorp/packer/template/interpolate"
)
@ -42,12 +42,15 @@ type Config struct {
// is an alternative method to authenticate to Yandex.Cloud.
ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"`
// The name of the bucket where the qcow2 file will be copied to for import.
// The name of the bucket where the qcow2 file will be uploaded to for import.
// This bucket must exist when the post-processor is run.
Bucket string `mapstructure:"bucket" required:"true"`
// The name of the object key in
// `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
// Therefore, you may use user variables and template functions in this field.
//
// If import occurred after Yandex-Export post-processor, artifact already
// in storage service and first paths (URL) is used to, so no need to set this param.
Bucket string `mapstructure:"bucket" required:"false"`
// The name of the object key in `bucket` where the qcow2 file will be copied to import.
// This is a [template engine](/docs/templates/engine).
// Therefore, you may use user variables and template functions in this field.
ObjectName string `mapstructure:"object_name" required:"false"`
// Whether skip removing the qcow2 file uploaded to Storage
// after the import process has completed. Possible values are: `true` to
@ -59,7 +62,7 @@ type Config struct {
ImageName string `mapstructure:"image_name" required:"false"`
// The description of the image.
ImageDescription string `mapstructure:"image_description" required:"false"`
// The family name of the imported image.
// The family name of the imported image.
ImageFamily string `mapstructure:"image_family" required:"false"`
// Key/value pair labels to apply to the imported image.
ImageLabels map[string]string `mapstructure:"image_labels" required:"false"`
@ -89,6 +92,30 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
errs := new(packer.MultiError)
// provision config by OS environment variables
if p.config.Token == "" {
p.config.Token = os.Getenv("YC_TOKEN")
}
if p.config.ServiceAccountKeyFile == "" {
p.config.ServiceAccountKeyFile = os.Getenv("YC_SERVICE_ACCOUNT_KEY_FILE")
}
if p.config.Token != "" {
packer.LogSecretFilter.Set(p.config.Token)
}
if p.config.ServiceAccountKeyFile != "" {
if _, err := iamkey.ReadFromJSONFile(p.config.ServiceAccountKeyFile); err != nil {
errs = packer.MultiErrorAppend(
errs, fmt.Errorf("fail to read service account key file: %s", err))
}
}
if p.config.FolderID == "" {
p.config.FolderID = os.Getenv("YC_FOLDER_ID")
}
// Set defaults
if p.config.ObjectName == "" {
p.config.ObjectName = "packer-import-{{timestamp}}.qcow2"
@ -103,7 +130,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
// TODO: make common code to check and prepare Yandex.Cloud auth configuration data
templates := map[string]*string{
"bucket": &p.config.Bucket,
"object_name": &p.config.ObjectName,
"folder_id": &p.config.FolderID,
}
@ -145,47 +171,71 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
}
var url string
var fileSource bool
// Create temporary storage Access Key
respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
ServiceAccountId: p.config.ServiceAccountID,
Description: "this key is for upload image to storage",
Description: "this temporary key is for upload image to storage; created by Packer",
})
if err != nil {
return nil, false, false, err
}
storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
if err != nil {
return nil, false, false, fmt.Errorf("error create object storage client: %s", err)
}
switch artifact.BuilderId() {
case compress.BuilderId, artifice.BuilderId, file.BuilderId:
break
// Artifact as a file, need to be uploaded to storage before create Compute Image
fileSource = true
// As `bucket` option validate input here
if p.config.Bucket == "" {
return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value")
}
url, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
if err != nil {
return nil, false, false, err
}
case yandexexport.BuilderId:
// Artifact already in storage, just get URL
url = artifact.Id()
case BuilderId:
// Artifact from prev yandex-import PP, reuse URL
url = artifact.Id()
default:
err := fmt.Errorf(
"Unknown artifact type: %s\nCan only import from Compress, Artifice and File post-processor artifacts.",
"Unknown artifact type: %s\nCan only import from Yandex-Export, Yandex-Import, Compress, Artifice and File post-processor artifacts.",
artifact.BuilderId())
return nil, false, false, err
}
storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
if err != nil {
return nil, false, false, fmt.Errorf("error create object_storage client: %s", err)
}
rawImageUrl, err := uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
presignedUrl, err := presignUrl(storageClient, ui, url)
if err != nil {
return nil, false, false, err
}
ycImageArtifact, err := createYCImage(ctx, client, ui, p.config.FolderID, rawImageUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, presignedUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
if err != nil {
return nil, false, false, err
}
if !p.config.SkipClean {
err = deleteFromBucket(storageClient, ui, p.config.Bucket, p.config.ObjectName)
if fileSource && !p.config.SkipClean {
err = deleteFromBucket(storageClient, ui, url)
if err != nil {
return nil, false, false, err
}
}
// cleanup static access keys
// Delete temporary storage Access Key
_, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{
AccessKeyId: respWithKey.GetAccessKey().GetId(),
})
@ -193,7 +243,10 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
return nil, false, false, fmt.Errorf("error delete static access key: %s", err)
}
return ycImageArtifact, false, false, nil
return &Artifact{
imageID: ycImage.GetId(),
sourceURL: url,
}, false, false, nil
}
func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
@ -238,16 +291,10 @@ func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucke
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
req.Config.S3ForcePathStyle = aws.Bool(true)
urlStr, _, err := req.PresignRequest(15 * time.Minute)
if err != nil {
ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
return "", err
}
return urlStr, nil
return req.HTTPRequest.URL.String(), nil
}
func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (packer.Artifact, error) {
func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) {
op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, &compute.CreateImageRequest{
FolderId: folderID,
Name: imageName,
@ -290,15 +337,19 @@ func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, fold
return nil, fmt.Errorf("error while image get request: %s", err)
}
return &yandex.Artifact{
Image: image,
}, nil
return image, nil
}
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, bucket string, objectName string) error {
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error {
bucket, objectName, err := s3URLToBucketKey(url)
if err != nil {
return err
}
ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
_, err := s3conn.DeleteObject(&s3.DeleteObjectInput{
_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
})

2
post-processor/yandex-import/post-processor.hcl2spec.go
View File

@ -20,7 +20,7 @@ type FlatConfig struct {
ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"`
Token *string `mapstructure:"token" required:"false" cty:"token" hcl:"token"`
ServiceAccountKeyFile *string `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"`
Bucket *string `mapstructure:"bucket" required:"true" cty:"bucket" hcl:"bucket"`
Bucket *string `mapstructure:"bucket" required:"false" cty:"bucket" hcl:"bucket"`
ObjectName *string `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"`
SkipClean *bool `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"`
ImageName *string `mapstructure:"image_name" required:"false" cty:"image_name" hcl:"image_name"`

54
post-processor/yandex-import/storage.go
View File

@ -2,11 +2,15 @@ package yandeximport
import (
"fmt"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/packer/packer"
)
const defaultS3Region = "ru-central1"
@ -40,3 +44,53 @@ func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, e
return s3.New(newSession), nil
}
// Get path-style S3 URL and return presigned URL
func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) {
bucket, key, err := s3URLToBucketKey(fullUrl)
if err != nil {
return "", err
}
req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
req.Config.S3ForcePathStyle = aws.Bool(true)
urlStr, _, err := req.PresignRequest(30 * time.Minute)
if err != nil {
ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
return "", err
}
return urlStr, nil
}
func s3URLToBucketKey(storageURL string) (bucket string, key string, err error) {
u, err := url.Parse(storageURL)
if err != nil {
return
}
if u.Scheme == "s3" {
// s3://bucket/key
bucket = u.Host
key = strings.TrimLeft(u.Path, "/")
} else if u.Scheme == "https" {
// https://***.storage.yandexcloud.net/...
if u.Host == defaultStorageEndpoint {
// No bucket name in the host part
path := strings.SplitN(u.Path, "/", 3)
bucket = path[1]
key = path[2]
} else {
// Bucket name in host
bucket = strings.TrimSuffix(u.Host, "."+defaultStorageEndpoint)
key = strings.TrimLeft(u.Path, "/")
}
}
return
}

57
post-processor/yandex-import/storage_test.go Normal file
View File

@ -0,0 +1,57 @@
package yandeximport
import (
"testing"
"github.com/stretchr/testify/assert"
)
func Test_s3URLToBucketKey(t *testing.T) {
tests := []struct {
name string
storageURL string
wantBucket string
wantKey string
wantErr bool
}{
{
name: "path-style url #1",
storageURL: "https://storage.yandexcloud.net/bucket1/key1/foobar.txt",
wantBucket: "bucket1",
wantKey: "key1/foobar.txt",
wantErr: false,
},
{
name: "path-style url #2",
storageURL: "https://storage.yandexcloud.net/bucket1.with.dots/key1/foobar.txt",
wantBucket: "bucket1.with.dots",
wantKey: "key1/foobar.txt",
wantErr: false,
},
{
name: "host-style url #1",
storageURL: "https://bucket1.with.dots.storage.yandexcloud.net/key1/foobar.txt",
wantBucket: "bucket1.with.dots",
wantKey: "key1/foobar.txt",
wantErr: false,
},
{
name: "host-style url #2",
storageURL: "https://bucket-with-dash.storage.yandexcloud.net/key2/foobar.txt",
wantBucket: "bucket-with-dash",
wantKey: "key2/foobar.txt",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotBucket, gotKey, err := s3URLToBucketKey(tt.storageURL)
if (err != nil) != tt.wantErr {
t.Errorf("s3URLToBucketKey() error = %v, wantErr %v", err, tt.wantErr)
return
}
assert.Equal(t, tt.wantBucket, gotBucket)
assert.Equal(t, tt.wantKey, gotKey)
})
}
}

3
website/pages/partials/post-processor/yandex-export/Config-required.mdx
View File

@ -4,9 +4,10 @@
Please be aware that use of space char inside path not supported.
Also this param support [build](/docs/templates/engine) template function.
Check available template data for [Yandex](/docs/builders/yandex#build-template-data) builder.
Paths to Yandex Object Storage where exported image will be uploaded.
- `folder_id` (string) - The folder ID that will be used to launch a temporary instance.
Alternatively you may set value by environment variable YC_FOLDER_ID.
- `service_account_id` (string) - Service Account ID with proper permission to modify an instance, create and attach disk and
make upload to specific Yandex Object Storage paths
make upload to specific Yandex Object Storage paths.

12
website/pages/partials/post-processor/yandex-import/Config-not-required.mdx
View File

@ -5,9 +5,15 @@
- `service_account_key_file` (string) - Path to file with Service Account key in json format. This
is an alternative method to authenticate to Yandex.Cloud.
- `object_name` (string) - The name of the object key in
`bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
Therefore, you may use user variables and template functions in this field.
- `bucket` (string) - The name of the bucket where the qcow2 file will be uploaded to for import.
This bucket must exist when the post-processor is run.
If import occurred after Yandex-Export post-processor, artifact already
in storage service and first paths (URL) is used to, so no need to set this param.
- `object_name` (string) - The name of the object key in `bucket` where the qcow2 file will be copied to import.
This is a [template engine](/docs/templates/engine).
Therefore, you may use user variables and template functions in this field.
- `skip_clean` (bool) - Whether skip removing the qcow2 file uploaded to Storage
after the import process has completed. Possible values are: `true` to

3
website/pages/partials/post-processor/yandex-import/Config-required.mdx
View File

@ -4,6 +4,3 @@
- `service_account_id` (string) - Service Account ID with proper permission to use Storage service
for operations 'upload' and 'delete' object to `bucket`
- `bucket` (string) - The name of the bucket where the qcow2 file will be copied to for import.
This bucket must exist when the post-processor is run.