Separate workflow for file on disk and prepared URL

post-processor/yandex-import/post-processor.go

@@ -6,17 +6,15 @@ package yandeximport
 import (
 	"context"
 	"fmt"
-	yandexexport "github.com/hashicorp/packer/post-processor/yandex-export"
 	"os"
 	"strings"
-	"time"
-
-	"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/hashicorp/packer/builder/yandex"
 	"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
+	"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
+	"github.com/yandex-cloud/go-sdk/iamkey"
 
 	"github.com/hashicorp/hcl/v2/hcldec"
 	"github.com/hashicorp/packer/builder/file"
@@ -25,6 +23,7 @@ import (
 	"github.com/hashicorp/packer/packer"
 	"github.com/hashicorp/packer/post-processor/artifice"
 	"github.com/hashicorp/packer/post-processor/compress"
+	"github.com/hashicorp/packer/post-processor/yandex-export"
 	"github.com/hashicorp/packer/template/interpolate"
 )
 
@@ -43,12 +42,15 @@ type Config struct {
 	// is an alternative method to authenticate to Yandex.Cloud.
 	ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"`
 
-	// The name of the bucket where the qcow2 file will be copied to for import.
+	// The name of the bucket where the qcow2 file will be uploaded to for import.
 	// This bucket must exist when the post-processor is run.
-	Bucket string `mapstructure:"bucket" required:"true"`
+	//
-	// The name of the object key in
+	// If import occurred after Yandex-Export post-processor, artifact already
-	//  `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
+	// in storage service and first paths (URL) is used to, so no need to set this param.
-	//  Therefore, you may use user variables and template functions in this field.
+	Bucket string `mapstructure:"bucket" required:"false"`
+	// The name of the object key in `bucket` where the qcow2 file will be copied to import.
+	// This is a [template engine](/docs/templates/engine).
+	// Therefore, you may use user variables and template functions in this field.
 	ObjectName string `mapstructure:"object_name" required:"false"`
 	// Whether skip removing the qcow2 file uploaded to Storage
 	// after the import process has completed. Possible values are: `true` to
@@ -60,7 +62,7 @@ type Config struct {
 	ImageName string `mapstructure:"image_name" required:"false"`
 	// The description of the image.
 	ImageDescription string `mapstructure:"image_description" required:"false"`
-	//  The family name of the imported image.
+	// The family name of the imported image.
 	ImageFamily string `mapstructure:"image_family" required:"false"`
 	// Key/value pair labels to apply to the imported image.
 	ImageLabels map[string]string `mapstructure:"image_labels" required:"false"`
@@ -128,7 +130,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 	// TODO: make common code to check and prepare Yandex.Cloud auth configuration data
 
 	templates := map[string]*string{
-		"bucket":      &p.config.Bucket,
 		"object_name": &p.config.ObjectName,
 		"folder_id":   &p.config.FolderID,
 	}
@@ -170,17 +171,40 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 		return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
 	}
 
+	var url string
+
+	// Create temporary storage Access Key
 	respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
 		ServiceAccountId: p.config.ServiceAccountID,
-		Description:      "this key is for upload image to storage",
+		Description:      "this temporary key is for upload image to storage; created by Packer",
 	})
 	if err != nil {
 		return nil, false, false, err
 	}
 
+	storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
+	if err != nil {
+		return nil, false, false, fmt.Errorf("error create object storage client: %s", err)
+	}
+
 	switch artifact.BuilderId() {
-	case compress.BuilderId, artifice.BuilderId, file.BuilderId, yandexexport.BuilderId:
+	case compress.BuilderId, artifice.BuilderId, file.BuilderId:
-		break
+		// Artifact as a file, need to be uploaded to storage before create Compute Image
+
+		// As `bucket` option validate input here
+		if p.config.Bucket == "" {
+			return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value")
+		}
+
+		url, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
+		if err != nil {
+			return nil, false, false, err
+		}
+
+	case yandexexport.BuilderId:
+		// Artifact already in storage, just get URL
+		url = artifact.Id()
+
 	default:
 		err := fmt.Errorf(
 			"Unknown artifact type: %s\nCan only import from Yandex-Export, Compress, Artifice and File post-processor artifacts.",
@@ -188,29 +212,24 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 		return nil, false, false, err
 	}
 
-	storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
+	presignedUrl, err := presignUrl(storageClient, ui, url)
-	if err != nil {
-		return nil, false, false, fmt.Errorf("error create object_storage client: %s", err)
-	}
-
-	rawImageUrl, err := uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
 	if err != nil {
 		return nil, false, false, err
 	}
 
-	ycImageArtifact, err := createYCImage(ctx, client, ui, p.config.FolderID, rawImageUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
+	ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, presignedUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
 	if err != nil {
 		return nil, false, false, err
 	}
 
 	if !p.config.SkipClean {
-		err = deleteFromBucket(storageClient, ui, p.config.Bucket, p.config.ObjectName)
+		err = deleteFromBucket(storageClient, ui, url)
 		if err != nil {
 			return nil, false, false, err
 		}
 	}
 
-	// cleanup static access keys
+	// Delete temporary storage Access Key
 	_, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{
 		AccessKeyId: respWithKey.GetAccessKey().GetId(),
 	})
@@ -218,7 +237,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 		return nil, false, false, fmt.Errorf("error delete static access key: %s", err)
 	}
 
-	return ycImageArtifact, false, false, nil
+	return ycImage, false, false, nil
 }
 
 func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
@@ -263,13 +282,7 @@ func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucke
 	// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
 	req.Config.S3ForcePathStyle = aws.Bool(true)
 
-	urlStr, _, err := req.PresignRequest(15 * time.Minute)
+	return req.HTTPRequest.URL.String(), nil
-	if err != nil {
-		ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
-		return "", err
-	}
-
-	return urlStr, nil
 }
 
 func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (packer.Artifact, error) {
@@ -320,10 +333,15 @@ func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, fold
 	}, nil
 }
 
-func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, bucket string, objectName string) error {
+func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error {
+	bucket, objectName, err := s3URLToBucketKey(url)
+	if err != nil {
+		return err
+	}
+
 	ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
 
-	_, err := s3conn.DeleteObject(&s3.DeleteObjectInput{
+	_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(objectName),
 	})

post-processor/yandex-import/post-processor.hcl2spec.go

@@ -20,7 +20,7 @@ type FlatConfig struct {
 	ServiceAccountID      *string           `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"`
 	Token                 *string           `mapstructure:"token" required:"false" cty:"token" hcl:"token"`
 	ServiceAccountKeyFile *string           `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"`
-	Bucket                *string           `mapstructure:"bucket" required:"true" cty:"bucket" hcl:"bucket"`
+	Bucket                *string           `mapstructure:"bucket" required:"false" cty:"bucket" hcl:"bucket"`
 	ObjectName            *string           `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"`
 	SkipClean             *bool             `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"`
 	ImageName             *string           `mapstructure:"image_name" required:"false" cty:"image_name" hcl:"image_name"`

post-processor/yandex-import/storage.go

@@ -60,7 +60,7 @@ func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) {
 	// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
 	req.Config.S3ForcePathStyle = aws.Bool(true)
 
-	urlStr, _, err := req.PresignRequest(15 * time.Minute)
+	urlStr, _, err := req.PresignRequest(30 * time.Minute)
 	if err != nil {
 		ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
 		return "", err

website/pages/partials/post-processor/yandex-import/Config-not-required.mdx

@@ -5,9 +5,15 @@
 - `service_account_key_file` (string) - Path to file with Service Account key in json format. This
   is an alternative method to authenticate to Yandex.Cloud.
 
-- `object_name` (string) - The name of the object key in
+- `bucket` (string) - The name of the bucket where the qcow2 file will be uploaded to for import.
-   `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
+  This bucket must exist when the post-processor is run.
-   Therefore, you may use user variables and template functions in this field.
+  
+  If import occurred after Yandex-Export post-processor, artifact already
+  in storage service and first paths (URL) is used to, so no need to set this param.
+
+- `object_name` (string) - The name of the object key in `bucket` where the qcow2 file will be copied to import.
+  This is a [template engine](/docs/templates/engine).
+  Therefore, you may use user variables and template functions in this field.
 
 - `skip_clean` (bool) - Whether skip removing the qcow2 file uploaded to Storage
   after the import process has completed. Possible values are: `true` to

website/pages/partials/post-processor/yandex-import/Config-required.mdx

@@ -4,6 +4,3 @@
 
 - `service_account_id` (string) - Service Account ID with proper permission to use Storage service
   for operations 'upload' and 'delete' object to `bucket`
-
-- `bucket` (string) - The name of the bucket where the qcow2 file will be copied to for import.
-  This bucket must exist when the post-processor is run.