Merge pull request #9614 from GennadySpb/yndx-import-create-new-from-another-one

yandex-import: allow create image based on another one

CHANGELOG.md

@@ -70,6 +70,7 @@
     file [GH-9379]
 * post-processor/yandex-import: Support using URL from yandex-export pp
     [GH-9601]
+* post-processor/yandex-import: Support create the new Image based on another one
 * provisioner/ansible: Add template option for templating the inventory file
     lines [GH-9438]
 

post-processor/yandex-import/artifact.go

@@ -7,8 +7,13 @@ import (
 const BuilderId = "packer.post-processor.yandex-import"
 
 type Artifact struct {
-	imageID   string
+	imageID    string
-	sourceURL string
+	sourceType string
+	sourceID   string
+
+	// StateData should store data such as GeneratedData
+	// to be shared with post-processors
+	StateData map[string]interface{}
 }
 
 func (*Artifact) BuilderId() string {
@@ -16,7 +21,7 @@ func (*Artifact) BuilderId() string {
 }
 
 func (a *Artifact) Id() string {
-	return a.sourceURL
+	return a.imageID
 }
 
 func (a *Artifact) Files() []string {
@@ -24,10 +29,14 @@ func (a *Artifact) Files() []string {
 }
 
 func (a *Artifact) String() string {
-	return fmt.Sprintf("Create image %v from URL %v", a.imageID, a.sourceURL)
+	return fmt.Sprintf("Create image %v from source type %v with ID/URL %v", a.imageID, a.sourceType, a.sourceID)
 }
 
-func (*Artifact) State(name string) interface{} {
+func (a *Artifact) State(name string) interface{} {
+	if _, ok := a.StateData[name]; ok {
+		return a.StateData[name]
+	}
+
 	return nil
 }
 

post-processor/yandex-import/artifact_test.go

@@ -0,0 +1,27 @@
+package yandeximport
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestArtifactState_StateData(t *testing.T) {
+	expectedData := "this is the data"
+	artifact := &Artifact{
+		StateData: map[string]interface{}{"state_data": expectedData},
+	}
+
+	// Valid state
+	result := artifact.State("state_data")
+	require.Equal(t, expectedData, result)
+
+	// Invalid state
+	result = artifact.State("invalid_key")
+	require.Equal(t, nil, result)
+
+	// Nil StateData should not fail and should return nil
+	artifact = &Artifact{}
+	result = artifact.State("key")
+	require.Equal(t, nil, result)
+}

post-processor/yandex-import/post-processor.go

@@ -7,17 +7,10 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/hashicorp/packer/builder/yandex"
-	"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
-	"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
-	"github.com/yandex-cloud/go-sdk/iamkey"
 
 	"github.com/hashicorp/hcl/v2/hcldec"
 	"github.com/hashicorp/packer/builder/file"
+	"github.com/hashicorp/packer/builder/yandex"
 	"github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/helper/config"
 	"github.com/hashicorp/packer/packer"
@@ -25,6 +18,8 @@ import (
 	"github.com/hashicorp/packer/post-processor/compress"
 	yandexexport "github.com/hashicorp/packer/post-processor/yandex-export"
 	"github.com/hashicorp/packer/template/interpolate"
+	"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
+	"github.com/yandex-cloud/go-sdk/iamkey"
 )
 
 type Config struct {
@@ -149,6 +144,10 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 }
 
 func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) {
+	var imageSrc cloudImageSource
+	var fileSource bool
+	var err error
+
 	generatedData := artifact.State("generated_data")
 	if generatedData == nil {
 		// Make sure it's not a nil map so we can assign to it later.
@@ -156,6 +155,11 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 	}
 	p.config.ctx.Data = generatedData
 
+	p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx)
+	if err != nil {
+		return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
+	}
+
 	cfg := &yandex.Config{
 		Token:                 p.config.Token,
 		ServiceAccountKeyFile: p.config.ServiceAccountKeyFile,
@@ -166,14 +170,6 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 		return nil, false, false, err
 	}
 
-	p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx)
-	if err != nil {
-		return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
-	}
-
-	var url string
-	var fileSource bool
-
 	// Create temporary storage Access Key
 	respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
 		ServiceAccountId: p.config.ServiceAccountID,
@@ -198,18 +194,29 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 			return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value")
 		}
 
-		url, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
+		imageSrc, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
 		if err != nil {
 			return nil, false, false, err
 		}
 
 	case yandexexport.BuilderId:
 		// Artifact already in storage, just get URL
-		url = artifact.Id()
+		imageSrc, err = presignUrl(storageClient, ui, artifact.Id())
+		if err != nil {
+			return nil, false, false, err
+		}
 
+	case yandex.BuilderID:
+		// Artifact is plain Yandex Compute Image, just create new one based on provided
+		imageSrc = &imageSource{
+			imageID: artifact.Id(),
+		}
 	case BuilderId:
-		// Artifact from prev yandex-import PP, reuse URL
+		// Artifact from prev yandex-import PP, reuse URL or Cloud Image ID
-		url = artifact.Id()
+		imageSrc, err = chooseSource(artifact)
+		if err != nil {
+			return nil, false, false, err
+		}
 
 	default:
 		err := fmt.Errorf(
@@ -218,18 +225,13 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 		return nil, false, false, err
 	}
 
-	presignedUrl, err := presignUrl(storageClient, ui, url)
+	ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, imageSrc, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
-	if err != nil {
-		return nil, false, false, err
-	}
-
-	ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, presignedUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
 	if err != nil {
 		return nil, false, false, err
 	}
 
 	if fileSource && !p.config.SkipClean {
-		err = deleteFromBucket(storageClient, ui, url)
+		err = deleteFromBucket(storageClient, ui, imageSrc)
 		if err != nil {
 			return nil, false, false, err
 		}
@@ -244,119 +246,29 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 	}
 
 	return &Artifact{
-		imageID:   ycImage.GetId(),
+		imageID: ycImage.GetId(),
-		sourceURL: url,
+		StateData: map[string]interface{}{
+			"source_type": imageSrc.GetSourceType(),
+			"source_id":   imageSrc.GetSourceID(),
+		},
 	}, false, false, nil
 }
 
-func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
+func chooseSource(a packer.Artifact) (cloudImageSource, error) {
-	ui.Say("Looking for qcow2 file in list of artifacts...")
+	st := a.State("source_type").(string)
-	source := ""
+	if st == "" {
-	for _, path := range artifact.Files() {
+		return nil, fmt.Errorf("could not determine source type of yandex-import artifact: %v", a)
-		ui.Say(fmt.Sprintf("Found artifact %v...", path))
-		if strings.HasSuffix(path, ".qcow2") {
-			source = path
-			break
-		}
 	}
+	switch st {
+	case sourceType_IMAGE:
+		return &imageSource{
+			imageID: a.State("source_id").(string),
+		}, nil
 
-	if source == "" {
+	case sourceType_OBJECT:
-		return "", fmt.Errorf("no qcow2 file found in list of artifacts")
+		return &objectSource{
+			url: a.State("source_id").(string),
+		}, nil
 	}
-
+	return nil, fmt.Errorf("unknow source type of yandex-import artifact: %s", st)
-	artifactFile, err := os.Open(source)
-	if err != nil {
-		err := fmt.Errorf("error opening %v", source)
-		return "", err
-	}
-
-	ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName))
-
-	_, err = s3conn.PutObject(&s3.PutObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(objectName),
-		Body:   artifactFile,
-	})
-
-	if err != nil {
-		ui.Say(fmt.Sprintf("Failed to upload: %v", objectName))
-		return "", err
-	}
-
-	req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(objectName),
-	})
-
-	// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
-	req.Config.S3ForcePathStyle = aws.Bool(true)
-
-	return req.HTTPRequest.URL.String(), nil
-}
-
-func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) {
-	op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, &compute.CreateImageRequest{
-		FolderId:    folderID,
-		Name:        imageName,
-		Description: imageDescription,
-		Labels:      imageLabels,
-		Family:      imageFamily,
-		Source:      &compute.CreateImageRequest_Uri{Uri: rawImageURL},
-	}))
-	if err != nil {
-		ui.Say("Error creating Yandex Compute Image")
-		return nil, err
-	}
-
-	ui.Say(fmt.Sprintf("Source url for Image creation: %v", rawImageURL))
-
-	ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id()))
-
-	ui.Say("Waiting for Yandex Compute Image creation operation to complete...")
-	err = op.Wait(ctx)
-
-	// fail if image creation operation has an error
-	if err != nil {
-		return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err)
-	}
-
-	protoMetadata, err := op.Metadata()
-	if err != nil {
-		return nil, fmt.Errorf("error while get image create operation metadata: %s", err)
-	}
-
-	md, ok := protoMetadata.(*compute.CreateImageMetadata)
-	if !ok {
-		return nil, fmt.Errorf("could not get Image ID from create operation metadata")
-	}
-
-	image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
-		ImageId: md.ImageId,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("error while image get request: %s", err)
-	}
-
-	return image, nil
-
-}
-
-func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error {
-	bucket, objectName, err := s3URLToBucketKey(url)
-	if err != nil {
-		return err
-	}
-
-	ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
-
-	_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(objectName),
-	})
-	if err != nil {
-		ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName))
-		return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err)
-	}
-
-	return nil
 }

post-processor/yandex-import/source.go

@@ -0,0 +1,44 @@
+package yandeximport
+
+import "fmt"
+
+const sourceType_IMAGE = "image"
+const sourceType_OBJECT = "object"
+
+type cloudImageSource interface {
+	GetSourceID() string
+	GetSourceType() string
+	Description() string
+}
+
+type imageSource struct {
+	imageID string
+}
+
+func (i *imageSource) GetSourceID() string {
+	return i.imageID
+}
+
+func (i *imageSource) GetSourceType() string {
+	return sourceType_IMAGE
+}
+
+func (i *imageSource) Description() string {
+	return fmt.Sprintf("%s source, id: %s", i.GetSourceType(), i.imageID)
+}
+
+type objectSource struct {
+	url string
+}
+
+func (i *objectSource) GetSourceID() string {
+	return i.url
+}
+
+func (i *objectSource) GetSourceType() string {
+	return sourceType_OBJECT
+}
+
+func (i *objectSource) Description() string {
+	return fmt.Sprintf("%s source, url: %s", i.GetSourceType(), i.url)
+}

post-processor/yandex-import/storage.go

@@ -46,10 +46,10 @@ func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, e
 }
 
 // Get path-style S3 URL and return presigned URL
-func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) {
+func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (cloudImageSource, error) {
 	bucket, key, err := s3URLToBucketKey(fullUrl)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
@@ -63,10 +63,12 @@ func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) {
 	urlStr, _, err := req.PresignRequest(30 * time.Minute)
 	if err != nil {
 		ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
-		return "", err
+		return nil, err
 	}
 
-	return urlStr, nil
+	return &objectSource{
+		urlStr,
+	}, nil
 }
 
 func s3URLToBucketKey(storageURL string) (bucket string, key string, err error) {

post-processor/yandex-import/utils.go

@@ -0,0 +1,146 @@
+package yandeximport
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/hashicorp/packer/builder/yandex"
+	"github.com/hashicorp/packer/packer"
+	"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
+)
+
+func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (cloudImageSource, error) {
+	ui.Say("Looking for qcow2 file in list of artifacts...")
+	source := ""
+	for _, path := range artifact.Files() {
+		ui.Say(fmt.Sprintf("Found artifact %v...", path))
+		if strings.HasSuffix(path, ".qcow2") {
+			source = path
+			break
+		}
+	}
+
+	if source == "" {
+		return nil, fmt.Errorf("no qcow2 file found in list of artifacts")
+	}
+
+	artifactFile, err := os.Open(source)
+	if err != nil {
+		err := fmt.Errorf("error opening %v", source)
+		return nil, err
+	}
+
+	ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName))
+
+	_, err = s3conn.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(objectName),
+		Body:   artifactFile,
+	})
+
+	if err != nil {
+		ui.Say(fmt.Sprintf("Failed to upload: %v", objectName))
+		return nil, err
+	}
+
+	req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(objectName),
+	})
+
+	// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
+	req.Config.S3ForcePathStyle = aws.Bool(true)
+
+	return &objectSource{
+		url: req.HTTPRequest.URL.String(),
+	}, nil
+}
+
+func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, imageSrc cloudImageSource, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) {
+	req := &compute.CreateImageRequest{
+		FolderId:    folderID,
+		Name:        imageName,
+		Description: imageDescription,
+		Labels:      imageLabels,
+		Family:      imageFamily,
+	}
+
+	// switch on cloudImageSource type: cloud image id or storage URL
+	switch v := imageSrc.(type) {
+	case *imageSource:
+		req.Source = &compute.CreateImageRequest_ImageId{ImageId: v.imageID}
+	case *objectSource:
+		req.Source = &compute.CreateImageRequest_Uri{Uri: v.url}
+	}
+
+	op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, req))
+	if err != nil {
+		ui.Say("Error creating Yandex Compute Image")
+		return nil, err
+	}
+
+	ui.Say(fmt.Sprintf("Source of Image creation: %s", imageSrc.Description()))
+
+	ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id()))
+
+	ui.Say("Waiting for Yandex Compute Image creation operation to complete...")
+	err = op.Wait(ctx)
+
+	// fail if image creation operation has an error
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err)
+	}
+
+	protoMetadata, err := op.Metadata()
+	if err != nil {
+		return nil, fmt.Errorf("error while get image create operation metadata: %s", err)
+	}
+
+	md, ok := protoMetadata.(*compute.CreateImageMetadata)
+	if !ok {
+		return nil, fmt.Errorf("could not get Image ID from create operation metadata")
+	}
+
+	image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
+		ImageId: md.ImageId,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error while image get request: %s", err)
+	}
+
+	return image, nil
+
+}
+
+func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, imageSrc cloudImageSource) error {
+	var url string
+	// switch on cloudImageSource type: cloud image id or storage URL
+	switch v := imageSrc.(type) {
+	case *objectSource:
+		url = v.GetSourceID()
+	case *imageSource:
+		return fmt.Errorf("invalid argument for `deleteFromBucket` method: %v", v)
+	}
+
+	bucket, objectName, err := s3URLToBucketKey(url)
+	if err != nil {
+		return err
+	}
+
+	ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
+
+	_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(objectName),
+	})
+	if err != nil {
+		ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName))
+		return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err)
+	}
+
+	return nil
+}