diff --git a/CHANGELOG.md b/CHANGELOG.md index dbe326d63..fefd5c75a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ file [GH-9379] * post-processor/yandex-import: Support using URL from yandex-export pp [GH-9601] +* post-processor/yandex-import: Support create the new Image based on another one * provisioner/ansible: Add template option for templating the inventory file lines [GH-9438] diff --git a/post-processor/yandex-import/artifact.go b/post-processor/yandex-import/artifact.go index c1112019a..6775a1190 100644 --- a/post-processor/yandex-import/artifact.go +++ b/post-processor/yandex-import/artifact.go @@ -7,8 +7,13 @@ import ( const BuilderId = "packer.post-processor.yandex-import" type Artifact struct { - imageID string - sourceURL string + imageID string + sourceType string + sourceID string + + // StateData should store data such as GeneratedData + // to be shared with post-processors + StateData map[string]interface{} } func (*Artifact) BuilderId() string { @@ -16,7 +21,7 @@ func (*Artifact) BuilderId() string { } func (a *Artifact) Id() string { - return a.sourceURL + return a.imageID } func (a *Artifact) Files() []string { @@ -24,10 +29,14 @@ func (a *Artifact) Files() []string { } func (a *Artifact) String() string { - return fmt.Sprintf("Create image %v from URL %v", a.imageID, a.sourceURL) + return fmt.Sprintf("Create image %v from source type %v with ID/URL %v", a.imageID, a.sourceType, a.sourceID) } -func (*Artifact) State(name string) interface{} { +func (a *Artifact) State(name string) interface{} { + if _, ok := a.StateData[name]; ok { + return a.StateData[name] + } + return nil } diff --git a/post-processor/yandex-import/artifact_test.go b/post-processor/yandex-import/artifact_test.go new file mode 100644 index 000000000..efc007fc2 --- /dev/null +++ b/post-processor/yandex-import/artifact_test.go @@ -0,0 +1,27 @@ +package yandeximport + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestArtifactState_StateData(t *testing.T) { + expectedData := "this is the data" + artifact := &Artifact{ + StateData: map[string]interface{}{"state_data": expectedData}, + } + + // Valid state + result := artifact.State("state_data") + require.Equal(t, expectedData, result) + + // Invalid state + result = artifact.State("invalid_key") + require.Equal(t, nil, result) + + // Nil StateData should not fail and should return nil + artifact = &Artifact{} + result = artifact.State("key") + require.Equal(t, nil, result) +} diff --git a/post-processor/yandex-import/post-processor.go b/post-processor/yandex-import/post-processor.go index 32e5c71c9..eaeab5ae8 100644 --- a/post-processor/yandex-import/post-processor.go +++ b/post-processor/yandex-import/post-processor.go @@ -7,17 +7,10 @@ import ( "context" "fmt" "os" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/hashicorp/packer/builder/yandex" - "github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1" - "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility" - "github.com/yandex-cloud/go-sdk/iamkey" "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/packer/builder/file" + "github.com/hashicorp/packer/builder/yandex" "github.com/hashicorp/packer/common" "github.com/hashicorp/packer/helper/config" "github.com/hashicorp/packer/packer" @@ -25,6 +18,8 @@ import ( "github.com/hashicorp/packer/post-processor/compress" yandexexport "github.com/hashicorp/packer/post-processor/yandex-export" "github.com/hashicorp/packer/template/interpolate" + "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility" + "github.com/yandex-cloud/go-sdk/iamkey" ) type Config struct { @@ -149,6 +144,10 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { + var imageSrc cloudImageSource + var fileSource bool + var err error + generatedData := artifact.State("generated_data") if generatedData == nil { // Make sure it's not a nil map so we can assign to it later. @@ -156,6 +155,11 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact } p.config.ctx.Data = generatedData + p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx) + if err != nil { + return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err) + } + cfg := &yandex.Config{ Token: p.config.Token, ServiceAccountKeyFile: p.config.ServiceAccountKeyFile, @@ -166,14 +170,6 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact return nil, false, false, err } - p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx) - if err != nil { - return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err) - } - - var url string - var fileSource bool - // Create temporary storage Access Key respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{ ServiceAccountId: p.config.ServiceAccountID, @@ -198,18 +194,29 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value") } - url, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName) + imageSrc, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName) if err != nil { return nil, false, false, err } case yandexexport.BuilderId: // Artifact already in storage, just get URL - url = artifact.Id() + imageSrc, err = presignUrl(storageClient, ui, artifact.Id()) + if err != nil { + return nil, false, false, err + } + case yandex.BuilderID: + // Artifact is plain Yandex Compute Image, just create new one based on provided + imageSrc = &imageSource{ + imageID: artifact.Id(), + } case BuilderId: - // Artifact from prev yandex-import PP, reuse URL - url = artifact.Id() + // Artifact from prev yandex-import PP, reuse URL or Cloud Image ID + imageSrc, err = chooseSource(artifact) + if err != nil { + return nil, false, false, err + } default: err := fmt.Errorf( @@ -218,18 +225,13 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact return nil, false, false, err } - presignedUrl, err := presignUrl(storageClient, ui, url) - if err != nil { - return nil, false, false, err - } - - ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, presignedUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels) + ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, imageSrc, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels) if err != nil { return nil, false, false, err } if fileSource && !p.config.SkipClean { - err = deleteFromBucket(storageClient, ui, url) + err = deleteFromBucket(storageClient, ui, imageSrc) if err != nil { return nil, false, false, err } @@ -244,119 +246,29 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact } return &Artifact{ - imageID: ycImage.GetId(), - sourceURL: url, + imageID: ycImage.GetId(), + StateData: map[string]interface{}{ + "source_type": imageSrc.GetSourceType(), + "source_id": imageSrc.GetSourceID(), + }, }, false, false, nil } -func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) { - ui.Say("Looking for qcow2 file in list of artifacts...") - source := "" - for _, path := range artifact.Files() { - ui.Say(fmt.Sprintf("Found artifact %v...", path)) - if strings.HasSuffix(path, ".qcow2") { - source = path - break - } +func chooseSource(a packer.Artifact) (cloudImageSource, error) { + st := a.State("source_type").(string) + if st == "" { + return nil, fmt.Errorf("could not determine source type of yandex-import artifact: %v", a) } + switch st { + case sourceType_IMAGE: + return &imageSource{ + imageID: a.State("source_id").(string), + }, nil - if source == "" { - return "", fmt.Errorf("no qcow2 file found in list of artifacts") + case sourceType_OBJECT: + return &objectSource{ + url: a.State("source_id").(string), + }, nil } - - artifactFile, err := os.Open(source) - if err != nil { - err := fmt.Errorf("error opening %v", source) - return "", err - } - - ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName)) - - _, err = s3conn.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(objectName), - Body: artifactFile, - }) - - if err != nil { - ui.Say(fmt.Sprintf("Failed to upload: %v", objectName)) - return "", err - } - - req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(objectName), - }) - - // Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process - req.Config.S3ForcePathStyle = aws.Bool(true) - - return req.HTTPRequest.URL.String(), nil -} - -func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) { - op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, &compute.CreateImageRequest{ - FolderId: folderID, - Name: imageName, - Description: imageDescription, - Labels: imageLabels, - Family: imageFamily, - Source: &compute.CreateImageRequest_Uri{Uri: rawImageURL}, - })) - if err != nil { - ui.Say("Error creating Yandex Compute Image") - return nil, err - } - - ui.Say(fmt.Sprintf("Source url for Image creation: %v", rawImageURL)) - - ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id())) - - ui.Say("Waiting for Yandex Compute Image creation operation to complete...") - err = op.Wait(ctx) - - // fail if image creation operation has an error - if err != nil { - return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err) - } - - protoMetadata, err := op.Metadata() - if err != nil { - return nil, fmt.Errorf("error while get image create operation metadata: %s", err) - } - - md, ok := protoMetadata.(*compute.CreateImageMetadata) - if !ok { - return nil, fmt.Errorf("could not get Image ID from create operation metadata") - } - - image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{ - ImageId: md.ImageId, - }) - if err != nil { - return nil, fmt.Errorf("error while image get request: %s", err) - } - - return image, nil - -} - -func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error { - bucket, objectName, err := s3URLToBucketKey(url) - if err != nil { - return err - } - - ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName)) - - _, err = s3conn.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(objectName), - }) - if err != nil { - ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName)) - return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err) - } - - return nil + return nil, fmt.Errorf("unknow source type of yandex-import artifact: %s", st) } diff --git a/post-processor/yandex-import/source.go b/post-processor/yandex-import/source.go new file mode 100644 index 000000000..d25e31a69 --- /dev/null +++ b/post-processor/yandex-import/source.go @@ -0,0 +1,44 @@ +package yandeximport + +import "fmt" + +const sourceType_IMAGE = "image" +const sourceType_OBJECT = "object" + +type cloudImageSource interface { + GetSourceID() string + GetSourceType() string + Description() string +} + +type imageSource struct { + imageID string +} + +func (i *imageSource) GetSourceID() string { + return i.imageID +} + +func (i *imageSource) GetSourceType() string { + return sourceType_IMAGE +} + +func (i *imageSource) Description() string { + return fmt.Sprintf("%s source, id: %s", i.GetSourceType(), i.imageID) +} + +type objectSource struct { + url string +} + +func (i *objectSource) GetSourceID() string { + return i.url +} + +func (i *objectSource) GetSourceType() string { + return sourceType_OBJECT +} + +func (i *objectSource) Description() string { + return fmt.Sprintf("%s source, url: %s", i.GetSourceType(), i.url) +} diff --git a/post-processor/yandex-import/storage.go b/post-processor/yandex-import/storage.go index e218be3ff..536515ea4 100644 --- a/post-processor/yandex-import/storage.go +++ b/post-processor/yandex-import/storage.go @@ -46,10 +46,10 @@ func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, e } // Get path-style S3 URL and return presigned URL -func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) { +func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (cloudImageSource, error) { bucket, key, err := s3URLToBucketKey(fullUrl) if err != nil { - return "", err + return nil, err } req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{ @@ -63,10 +63,12 @@ func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) { urlStr, _, err := req.PresignRequest(30 * time.Minute) if err != nil { ui.Say(fmt.Sprintf("Failed to presign url: %s", err)) - return "", err + return nil, err } - return urlStr, nil + return &objectSource{ + urlStr, + }, nil } func s3URLToBucketKey(storageURL string) (bucket string, key string, err error) { diff --git a/post-processor/yandex-import/utils.go b/post-processor/yandex-import/utils.go new file mode 100644 index 000000000..74c23dbc8 --- /dev/null +++ b/post-processor/yandex-import/utils.go @@ -0,0 +1,146 @@ +package yandeximport + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/hashicorp/packer/builder/yandex" + "github.com/hashicorp/packer/packer" + "github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1" +) + +func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (cloudImageSource, error) { + ui.Say("Looking for qcow2 file in list of artifacts...") + source := "" + for _, path := range artifact.Files() { + ui.Say(fmt.Sprintf("Found artifact %v...", path)) + if strings.HasSuffix(path, ".qcow2") { + source = path + break + } + } + + if source == "" { + return nil, fmt.Errorf("no qcow2 file found in list of artifacts") + } + + artifactFile, err := os.Open(source) + if err != nil { + err := fmt.Errorf("error opening %v", source) + return nil, err + } + + ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName)) + + _, err = s3conn.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectName), + Body: artifactFile, + }) + + if err != nil { + ui.Say(fmt.Sprintf("Failed to upload: %v", objectName)) + return nil, err + } + + req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectName), + }) + + // Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process + req.Config.S3ForcePathStyle = aws.Bool(true) + + return &objectSource{ + url: req.HTTPRequest.URL.String(), + }, nil +} + +func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, imageSrc cloudImageSource, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) { + req := &compute.CreateImageRequest{ + FolderId: folderID, + Name: imageName, + Description: imageDescription, + Labels: imageLabels, + Family: imageFamily, + } + + // switch on cloudImageSource type: cloud image id or storage URL + switch v := imageSrc.(type) { + case *imageSource: + req.Source = &compute.CreateImageRequest_ImageId{ImageId: v.imageID} + case *objectSource: + req.Source = &compute.CreateImageRequest_Uri{Uri: v.url} + } + + op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, req)) + if err != nil { + ui.Say("Error creating Yandex Compute Image") + return nil, err + } + + ui.Say(fmt.Sprintf("Source of Image creation: %s", imageSrc.Description())) + + ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id())) + + ui.Say("Waiting for Yandex Compute Image creation operation to complete...") + err = op.Wait(ctx) + + // fail if image creation operation has an error + if err != nil { + return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err) + } + + protoMetadata, err := op.Metadata() + if err != nil { + return nil, fmt.Errorf("error while get image create operation metadata: %s", err) + } + + md, ok := protoMetadata.(*compute.CreateImageMetadata) + if !ok { + return nil, fmt.Errorf("could not get Image ID from create operation metadata") + } + + image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{ + ImageId: md.ImageId, + }) + if err != nil { + return nil, fmt.Errorf("error while image get request: %s", err) + } + + return image, nil + +} + +func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, imageSrc cloudImageSource) error { + var url string + // switch on cloudImageSource type: cloud image id or storage URL + switch v := imageSrc.(type) { + case *objectSource: + url = v.GetSourceID() + case *imageSource: + return fmt.Errorf("invalid argument for `deleteFromBucket` method: %v", v) + } + + bucket, objectName, err := s3URLToBucketKey(url) + if err != nil { + return err + } + + ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName)) + + _, err = s3conn.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectName), + }) + if err != nil { + ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName)) + return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err) + } + + return nil +}