diff --git a/builder/yandex/step_create_instance.go b/builder/yandex/step_create_instance.go index 5217bb9e5..f5f0f176b 100644 --- a/builder/yandex/step_create_instance.go +++ b/builder/yandex/step_create_instance.go @@ -227,12 +227,6 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) } } - secDisk, ok := state.GetOk("secondary_disk") - if !ok { - secDisk = []*compute.AttachedDiskSpec{} - } - secDiskSpec := secDisk.([]*compute.AttachedDiskSpec) - req := &compute.CreateInstanceRequest{ FolderId: config.FolderID, Name: config.InstanceName, @@ -254,7 +248,6 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) DiskId: disk.Id, }, }, - SecondaryDiskSpecs: secDiskSpec, NetworkInterfaceSpecs: []*compute.NetworkInterfaceSpec{ { SubnetId: instanceSubnetID, diff --git a/post-processor/yandex-export/cloud-init-script.go b/post-processor/yandex-export/cloud-init-script.go index a6bb81068..0e957a0bf 100644 --- a/post-processor/yandex-export/cloud-init-script.go +++ b/post-processor/yandex-export/cloud-init-script.go @@ -8,6 +8,8 @@ GetMetadata() { curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null } +[[ "$(GetMetadata debug)" == "1" || "$(GetMetadata debug)" == "true" ]] && set -x + InstallPackages() { sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli } @@ -16,7 +18,7 @@ WaitFile() { local RETRIES=60 while [[ ${RETRIES} -gt 0 ]]; do echo "Wait ${1}" - if [ -f "${1}" ]; then + if [ -e "${1}" ]; then echo "[${1}] has been found" return 0 fi @@ -29,6 +31,7 @@ WaitFile() { PATHS=$(GetMetadata paths) S3_ENDPOINT="https://storage.yandexcloud.net" +DISK_EXPORT_PATH="/dev/disk/by-id/virtio-doexport" export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials" export AWS_REGION=ru-central1 @@ -52,9 +55,15 @@ if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then echo "Failed wait credentials" Exit 1 fi +udevadm trigger || true + +if ! WaitFile "${DISK_EXPORT_PATH}"; then + echo "Failed wait attach disk" + Exit 1 +fi echo "Dumping disk..." -if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then +if ! qemu-img convert -p -O qcow2 -o cluster_size=2M "${DISK_EXPORT_PATH}" disk.qcow2; then echo "Failed to dump disk to qcow2 image." Exit 1 fi diff --git a/post-processor/yandex-export/post-processor.go b/post-processor/yandex-export/post-processor.go index 6d76adb53..f149de5e8 100644 --- a/post-processor/yandex-export/post-processor.go +++ b/post-processor/yandex-export/post-processor.go @@ -21,7 +21,6 @@ import ( "github.com/hashicorp/packer/packer-plugin-sdk/template/config" "github.com/hashicorp/packer/packer-plugin-sdk/template/interpolate" "github.com/hashicorp/packer/post-processor/artifice" - "github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1" "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1" ycsdk "github.com/yandex-cloud/go-sdk" ) @@ -48,8 +47,9 @@ type Config struct { // The `~` can be used in path and will be expanded to the home directory // of current user. Login for attach: `ubuntu` SSHPrivateKeyFile string `mapstructure:"ssh_private_key_file" required:"false"` - - ctx interpolate.Context + // Number of attempts to wait for export (must be greater than 0). Default: 1000 + Tries int `mapstructure:"tries" required:"false"` + ctx interpolate.Context } type PostProcessor struct { @@ -83,6 +83,9 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { if p.config.DiskSizeGb == 0 { p.config.DiskSizeGb = 100 } + if p.config.Tries <= 0 { + p.config.Tries = 1000 + } errs = p.config.CommonConfig.Prepare(errs) errs = p.config.ExchangeConfig.Prepare(errs) @@ -171,26 +174,6 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa return nil, false, false, err } - imageDesc, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{ - ImageId: imageID, - }) - if err != nil { - return nil, false, false, err - } - secDiskSpec := []*compute.AttachedDiskSpec{ - { - AutoDelete: true, - DeviceName: "doexport", - Disk: &compute.AttachedDiskSpec_DiskSpec_{ - DiskSpec: &compute.AttachedDiskSpec_DiskSpec{ - Source: &compute.AttachedDiskSpec_DiskSpec_ImageId{ - ImageId: imageID, - }, - Size: imageDesc.MinDiskSize, - }, - }, - }, - } // Set up exporter instance configuration. exporterName := fmt.Sprintf("%s-exporter", artifact.Id()) yandexConfig := ycSaneDefaults(&p.config, @@ -226,7 +209,6 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa state.Put("driver", driver) state.Put("sdk", driver.SDK()) state.Put("ui", ui) - state.Put("secondary_disk", secDiskSpec) // Build the steps. steps := []multistep.Step{ @@ -249,8 +231,14 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa Host: yandex.CommHost, SSHConfig: yandexConfig.Communicator.SSHConfigFunc(), }, + &StepAttachDisk{ + CommonConfig: p.config.CommonConfig, + ImageID: imageID, + }, new(StepUploadSecrets), - new(StepWaitCloudInitScript), + &StepWaitCloudInitScript{ + Tries: p.config.Tries, + }, &yandex.StepTeardownInstance{ SerialLogFile: yandexConfig.SerialLogFile, }, diff --git a/post-processor/yandex-export/post-processor.hcl2spec.go b/post-processor/yandex-export/post-processor.hcl2spec.go index 6d575bd05..93fafefed 100644 --- a/post-processor/yandex-export/post-processor.hcl2spec.go +++ b/post-processor/yandex-export/post-processor.hcl2spec.go @@ -45,6 +45,7 @@ type FlatConfig struct { ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"` Paths []string `mapstructure:"paths" required:"true" cty:"paths" hcl:"paths"` SSHPrivateKeyFile *string `mapstructure:"ssh_private_key_file" required:"false" cty:"ssh_private_key_file" hcl:"ssh_private_key_file"` + Tries *int `mapstructure:"tries" required:"false" cty:"tries" hcl:"tries"` } // FlatMapstructure returns a new FlatConfig. @@ -95,6 +96,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { "service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false}, "paths": &hcldec.AttrSpec{Name: "paths", Type: cty.List(cty.String), Required: false}, "ssh_private_key_file": &hcldec.AttrSpec{Name: "ssh_private_key_file", Type: cty.String, Required: false}, + "tries": &hcldec.AttrSpec{Name: "tries", Type: cty.Number, Required: false}, } return s } diff --git a/post-processor/yandex-export/scripts/export.sh b/post-processor/yandex-export/scripts/export.sh index de23d6a08..580409248 100644 --- a/post-processor/yandex-export/scripts/export.sh +++ b/post-processor/yandex-export/scripts/export.sh @@ -4,6 +4,8 @@ GetMetadata() { curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null } +[[ "$(GetMetadata debug)" == "1" || "$(GetMetadata debug)" == "true" ]] && set -x + InstallPackages() { sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli } @@ -12,7 +14,7 @@ WaitFile() { local RETRIES=60 while [[ ${RETRIES} -gt 0 ]]; do echo "Wait ${1}" - if [ -f "${1}" ]; then + if [ -e "${1}" ]; then echo "[${1}] has been found" return 0 fi @@ -25,6 +27,7 @@ WaitFile() { PATHS=$(GetMetadata paths) S3_ENDPOINT="https://storage.yandexcloud.net" +DISK_EXPORT_PATH="/dev/disk/by-id/virtio-doexport" export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials" export AWS_REGION=ru-central1 @@ -48,9 +51,15 @@ if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then echo "Failed wait credentials" Exit 1 fi +udevadm trigger || true + +if ! WaitFile "${DISK_EXPORT_PATH}"; then + echo "Failed wait attach disk" + Exit 1 +fi echo "Dumping disk..." -if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then +if ! qemu-img convert -p -O qcow2 -o cluster_size=2M "${DISK_EXPORT_PATH}" disk.qcow2; then echo "Failed to dump disk to qcow2 image." Exit 1 fi diff --git a/post-processor/yandex-export/step-attach-disk.go b/post-processor/yandex-export/step-attach-disk.go new file mode 100644 index 000000000..fa6e32765 --- /dev/null +++ b/post-processor/yandex-export/step-attach-disk.go @@ -0,0 +1,100 @@ +package yandexexport + +import ( + "context" + "fmt" + + "github.com/hashicorp/packer/builder/yandex" + "github.com/hashicorp/packer/packer-plugin-sdk/multistep" + "github.com/hashicorp/packer/packer-plugin-sdk/packer" + "github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1" +) + +type StepAttachDisk struct { + yandex.CommonConfig + ImageID string +} + +func (c *StepAttachDisk) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { + driver := state.Get("driver").(yandex.Driver) + ui := state.Get("ui").(packer.Ui) + instanceID := state.Get("instance_id").(string) + + ui.Say("Create disk from source image...") + + imageDesc, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{ + ImageId: c.ImageID, + }) + if err != nil { + return yandex.StepHaltWithError(state, err) + } + + op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Disk().Create(ctx, &compute.CreateDiskRequest{ + Source: &compute.CreateDiskRequest_ImageId{ + ImageId: c.ImageID, + }, + Name: fmt.Sprintf("export-%s-disk", instanceID), + Size: imageDesc.GetMinDiskSize(), + ZoneId: c.Zone, + FolderId: c.FolderID, + TypeId: c.DiskType, + Description: "Temporary disk for exporting", + })) + if op == nil { + return yandex.StepHaltWithError(state, err) + } + protoMD, err := op.Metadata() + if err != nil { + return yandex.StepHaltWithError(state, err) + } + md, ok := protoMD.(*compute.CreateDiskMetadata) + if !ok { + return yandex.StepHaltWithError(state, fmt.Errorf("could not get Instance ID from create operation metadata")) + } + state.Put("secondary_disk_id", md.GetDiskId()) + + if err := op.Wait(ctx); err != nil { + return yandex.StepHaltWithError(state, err) + } + + ui.Say("Attach disk...") + + op, err = driver.SDK().WrapOperation(driver.SDK().Compute().Instance().AttachDisk(ctx, &compute.AttachInstanceDiskRequest{ + InstanceId: instanceID, + AttachedDiskSpec: &compute.AttachedDiskSpec{ + AutoDelete: true, + DeviceName: "doexport", + Disk: &compute.AttachedDiskSpec_DiskId{ + DiskId: md.GetDiskId(), + }, + }, + })) + if err != nil { + return yandex.StepHaltWithError(state, err) + } + ui.Message("Wait attached disk...") + if err := op.Wait(ctx); err != nil { + return yandex.StepHaltWithError(state, err) + } + + state.Remove("secondary_disk_id") + return multistep.ActionContinue +} + +func (s *StepAttachDisk) Cleanup(state multistep.StateBag) { + ui := state.Get("ui").(packer.Ui) + driver := state.Get("driver").(yandex.Driver) + if diskID, ok := state.GetOk("secondary_disk_id"); ok { + ui.Say("Remove the secondary disk...") + op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Disk().Delete(context.Background(), &compute.DeleteDiskRequest{ + DiskId: diskID.(string), + })) + if err != nil { + ui.Error(err.Error()) + return + } + if err := op.Wait(context.Background()); err != nil { + ui.Error(err.Error()) + } + } +} diff --git a/post-processor/yandex-export/step-wait-cloud-init.go b/post-processor/yandex-export/step-wait-cloud-init.go index 2ec50ab6a..de96eb269 100644 --- a/post-processor/yandex-export/step-wait-cloud-init.go +++ b/post-processor/yandex-export/step-wait-cloud-init.go @@ -14,7 +14,9 @@ import ( "github.com/hashicorp/packer/packer-plugin-sdk/retry" ) -type StepWaitCloudInitScript int +type StepWaitCloudInitScript struct { + Tries int +} type cloudInitStatus struct { V1 struct { @@ -64,6 +66,7 @@ func (s *StepWaitCloudInitScript) Run(ctx context.Context, state multistep.State } return true }, + Tries: s.Tries, RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 2}).Linear, } diff --git a/website/pages/partials/post-processor/yandex-export/Config-not-required.mdx b/website/pages/partials/post-processor/yandex-export/Config-not-required.mdx index 192378239..82ecaff31 100644 --- a/website/pages/partials/post-processor/yandex-export/Config-not-required.mdx +++ b/website/pages/partials/post-processor/yandex-export/Config-not-required.mdx @@ -3,3 +3,5 @@ - `ssh_private_key_file` (string) - Path to a PEM encoded private key file to use to authenticate with SSH. The `~` can be used in path and will be expanded to the home directory of current user. Login for attach: `ubuntu` + +- `tries` (int) - Number of attempts to wait for export (must be greater than 0). Default: 1000