Separate workflow for file on disk and prepared URL
This commit is contained in:
Gennady Lipenkov
parent
59aaaf7a91
commit
29a6687475
|
|
@ -6,17 +6,15 @@ package yandeximport
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
yandexexport "github.com/hashicorp/packer/post-processor/yandex-export"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/hashicorp/packer/builder/yandex"
|
||||
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
|
||||
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
|
||||
"github.com/yandex-cloud/go-sdk/iamkey"
|
||||
|
||||
"github.com/hashicorp/hcl/v2/hcldec"
|
||||
"github.com/hashicorp/packer/builder/file"
|
||||
|
|
@ -25,6 +23,7 @@ import (
|
|||
"github.com/hashicorp/packer/packer"
|
||||
"github.com/hashicorp/packer/post-processor/artifice"
|
||||
"github.com/hashicorp/packer/post-processor/compress"
|
||||
"github.com/hashicorp/packer/post-processor/yandex-export"
|
||||
"github.com/hashicorp/packer/template/interpolate"
|
||||
)
|
||||
|
||||
|
|
@ -43,12 +42,15 @@ type Config struct {
|
|||
// is an alternative method to authenticate to Yandex.Cloud.
|
||||
ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"`
|
||||
|
||||
// The name of the bucket where the qcow2 file will be copied to for import.
|
||||
// The name of the bucket where the qcow2 file will be uploaded to for import.
|
||||
// This bucket must exist when the post-processor is run.
|
||||
Bucket string `mapstructure:"bucket" required:"true"`
|
||||
// The name of the object key in
|
||||
// `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
|
||||
// Therefore, you may use user variables and template functions in this field.
|
||||
//
|
||||
// If import occurred after Yandex-Export post-processor, artifact already
|
||||
// in storage service and first paths (URL) is used to, so no need to set this param.
|
||||
Bucket string `mapstructure:"bucket" required:"false"`
|
||||
// The name of the object key in `bucket` where the qcow2 file will be copied to import.
|
||||
// This is a [template engine](/docs/templates/engine).
|
||||
// Therefore, you may use user variables and template functions in this field.
|
||||
ObjectName string `mapstructure:"object_name" required:"false"`
|
||||
// Whether skip removing the qcow2 file uploaded to Storage
|
||||
// after the import process has completed. Possible values are: `true` to
|
||||
|
|
@ -60,7 +62,7 @@ type Config struct {
|
|||
ImageName string `mapstructure:"image_name" required:"false"`
|
||||
// The description of the image.
|
||||
ImageDescription string `mapstructure:"image_description" required:"false"`
|
||||
// The family name of the imported image.
|
||||
// The family name of the imported image.
|
||||
ImageFamily string `mapstructure:"image_family" required:"false"`
|
||||
// Key/value pair labels to apply to the imported image.
|
||||
ImageLabels map[string]string `mapstructure:"image_labels" required:"false"`
|
||||
|
|
@ -128,7 +130,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
|
|||
// TODO: make common code to check and prepare Yandex.Cloud auth configuration data
|
||||
|
||||
templates := map[string]*string{
|
||||
"bucket": &p.config.Bucket,
|
||||
"object_name": &p.config.ObjectName,
|
||||
"folder_id": &p.config.FolderID,
|
||||
}
|
||||
|
|
@ -170,17 +171,40 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
|
|||
return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
|
||||
}
|
||||
|
||||
var url string
|
||||
|
||||
// Create temporary storage Access Key
|
||||
respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
|
||||
ServiceAccountId: p.config.ServiceAccountID,
|
||||
Description: "this key is for upload image to storage",
|
||||
Description: "this temporary key is for upload image to storage; created by Packer",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
|
||||
if err != nil {
|
||||
return nil, false, false, fmt.Errorf("error create object storage client: %s", err)
|
||||
}
|
||||
|
||||
switch artifact.BuilderId() {
|
||||
case compress.BuilderId, artifice.BuilderId, file.BuilderId, yandexexport.BuilderId:
|
||||
break
|
||||
case compress.BuilderId, artifice.BuilderId, file.BuilderId:
|
||||
// Artifact as a file, need to be uploaded to storage before create Compute Image
|
||||
|
||||
// As `bucket` option validate input here
|
||||
if p.config.Bucket == "" {
|
||||
return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value")
|
||||
}
|
||||
|
||||
url, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
case yandexexport.BuilderId:
|
||||
// Artifact already in storage, just get URL
|
||||
url = artifact.Id()
|
||||
|
||||
default:
|
||||
err := fmt.Errorf(
|
||||
"Unknown artifact type: %s\nCan only import from Yandex-Export, Compress, Artifice and File post-processor artifacts.",
|
||||
|
|
@ -188,29 +212,24 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
|
|||
return nil, false, false, err
|
||||
}
|
||||
|
||||
storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
|
||||
if err != nil {
|
||||
return nil, false, false, fmt.Errorf("error create object_storage client: %s", err)
|
||||
}
|
||||
|
||||
rawImageUrl, err := uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
|
||||
presignedUrl, err := presignUrl(storageClient, ui, url)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
ycImageArtifact, err := createYCImage(ctx, client, ui, p.config.FolderID, rawImageUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
|
||||
ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, presignedUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
if !p.config.SkipClean {
|
||||
err = deleteFromBucket(storageClient, ui, p.config.Bucket, p.config.ObjectName)
|
||||
err = deleteFromBucket(storageClient, ui, url)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
}
|
||||
|
||||
// cleanup static access keys
|
||||
// Delete temporary storage Access Key
|
||||
_, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{
|
||||
AccessKeyId: respWithKey.GetAccessKey().GetId(),
|
||||
})
|
||||
|
|
@ -218,7 +237,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
|
|||
return nil, false, false, fmt.Errorf("error delete static access key: %s", err)
|
||||
}
|
||||
|
||||
return ycImageArtifact, false, false, nil
|
||||
return ycImage, false, false, nil
|
||||
}
|
||||
|
||||
func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
|
||||
|
|
@ -263,13 +282,7 @@ func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucke
|
|||
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
|
||||
req.Config.S3ForcePathStyle = aws.Bool(true)
|
||||
|
||||
urlStr, _, err := req.PresignRequest(15 * time.Minute)
|
||||
if err != nil {
|
||||
ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
|
||||
return "", err
|
||||
}
|
||||
|
||||
return urlStr, nil
|
||||
return req.HTTPRequest.URL.String(), nil
|
||||
}
|
||||
|
||||
func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (packer.Artifact, error) {
|
||||
|
|
@ -320,10 +333,15 @@ func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, fold
|
|||
}, nil
|
||||
}
|
||||
|
||||
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, bucket string, objectName string) error {
|
||||
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error {
|
||||
bucket, objectName, err := s3URLToBucketKey(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
|
||||
|
||||
_, err := s3conn.DeleteObject(&s3.DeleteObjectInput{
|
||||
_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(objectName),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ type FlatConfig struct {
|
|||
ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"`
|
||||
Token *string `mapstructure:"token" required:"false" cty:"token" hcl:"token"`
|
||||
ServiceAccountKeyFile *string `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"`
|
||||
Bucket *string `mapstructure:"bucket" required:"true" cty:"bucket" hcl:"bucket"`
|
||||
Bucket *string `mapstructure:"bucket" required:"false" cty:"bucket" hcl:"bucket"`
|
||||
ObjectName *string `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"`
|
||||
SkipClean *bool `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"`
|
||||
ImageName *string `mapstructure:"image_name" required:"false" cty:"image_name" hcl:"image_name"`
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) {
|
|||
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
|
||||
req.Config.S3ForcePathStyle = aws.Bool(true)
|
||||
|
||||
urlStr, _, err := req.PresignRequest(15 * time.Minute)
|
||||
urlStr, _, err := req.PresignRequest(30 * time.Minute)
|
||||
if err != nil {
|
||||
ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
|
||||
return "", err
|
||||
|
|
|
|||
|
|
@ -5,9 +5,15 @@
|
|||
- `service_account_key_file` (string) - Path to file with Service Account key in json format. This
|
||||
is an alternative method to authenticate to Yandex.Cloud.
|
||||
|
||||
- `object_name` (string) - The name of the object key in
|
||||
`bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
|
||||
Therefore, you may use user variables and template functions in this field.
|
||||
- `bucket` (string) - The name of the bucket where the qcow2 file will be uploaded to for import.
|
||||
This bucket must exist when the post-processor is run.
|
||||
|
||||
If import occurred after Yandex-Export post-processor, artifact already
|
||||
in storage service and first paths (URL) is used to, so no need to set this param.
|
||||
|
||||
- `object_name` (string) - The name of the object key in `bucket` where the qcow2 file will be copied to import.
|
||||
This is a [template engine](/docs/templates/engine).
|
||||
Therefore, you may use user variables and template functions in this field.
|
||||
|
||||
- `skip_clean` (bool) - Whether skip removing the qcow2 file uploaded to Storage
|
||||
after the import process has completed. Possible values are: `true` to
|
||||
|
|
|
|||
|
|
@ -4,6 +4,3 @@
|
|||
|
||||
- `service_account_id` (string) - Service Account ID with proper permission to use Storage service
|
||||
for operations 'upload' and 'delete' object to `bucket`
|
||||
|
||||
- `bucket` (string) - The name of the bucket where the qcow2 file will be copied to for import.
|
||||
This bucket must exist when the post-processor is run.
|
||||
|
|
|
|||
Loading…
Reference in New Issue