Merge pull request #10368 from GennadySpb/yandex-export-show-progress

Yandex export show progress

builder/yandex/step_create_instance.go

@@ -227,12 +227,6 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
 		}
 	}
 
-	secDisk, ok := state.GetOk("secondary_disk")
-	if !ok {
-		secDisk = []*compute.AttachedDiskSpec{}
-	}
-	secDiskSpec := secDisk.([]*compute.AttachedDiskSpec)
-
 	req := &compute.CreateInstanceRequest{
 		FolderId:   config.FolderID,
 		Name:       config.InstanceName,
@@ -254,7 +248,6 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
 				DiskId: disk.Id,
 			},
 		},
-		SecondaryDiskSpecs: secDiskSpec,
 		NetworkInterfaceSpecs: []*compute.NetworkInterfaceSpec{
 			{
 				SubnetId:             instanceSubnetID,

post-processor/yandex-export/cloud-init-script.go

@@ -8,6 +8,8 @@ GetMetadata() {
     curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null
 }
 
+[[ "$(GetMetadata debug)" == "1"  || "$(GetMetadata debug)" == "true" ]] && set -x
+
 InstallPackages() {
     sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli
 }
@@ -16,7 +18,7 @@ WaitFile() {
     local RETRIES=60
     while [[ ${RETRIES} -gt 0 ]]; do
         echo "Wait ${1}"
-        if [ -f "${1}" ]; then
+        if [ -e "${1}" ]; then
             echo "[${1}] has been found"
             return 0
         fi
@@ -29,6 +31,7 @@ WaitFile() {
 
 PATHS=$(GetMetadata paths)
 S3_ENDPOINT="https://storage.yandexcloud.net"
+DISK_EXPORT_PATH="/dev/disk/by-id/virtio-doexport"
 export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials"
 export AWS_REGION=ru-central1
 
@@ -52,9 +55,15 @@ if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then
     echo "Failed wait credentials"
     Exit 1
 fi
+udevadm trigger || true
+
+if ! WaitFile "${DISK_EXPORT_PATH}"; then
+    echo "Failed wait attach disk"
+    Exit 1
+fi
 
 echo "Dumping disk..."
-if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then
+if ! qemu-img convert -O qcow2 -o cluster_size=2M "${DISK_EXPORT_PATH}" disk.qcow2; then
     echo "Failed to dump disk to qcow2 image."
     Exit 1
 fi

post-processor/yandex-export/post-processor.go

@@ -21,7 +21,6 @@ import (
 	"github.com/hashicorp/packer/packer-plugin-sdk/template/config"
 	"github.com/hashicorp/packer/packer-plugin-sdk/template/interpolate"
 	"github.com/hashicorp/packer/post-processor/artifice"
-	"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
 	"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1"
 	ycsdk "github.com/yandex-cloud/go-sdk"
 )
@@ -48,8 +47,9 @@ type Config struct {
 	// The `~` can be used in path and will be expanded to the home directory
 	// of current user. Login for attach: `ubuntu`
 	SSHPrivateKeyFile string `mapstructure:"ssh_private_key_file" required:"false"`
-
+	// Number of attempts to wait for export (must be greater than 0). Default: 1000
-	ctx interpolate.Context
+	Tries int `mapstructure:"tries" required:"false"`
+	ctx   interpolate.Context
 }
 
 type PostProcessor struct {
@@ -83,6 +83,9 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 	if p.config.DiskSizeGb == 0 {
 		p.config.DiskSizeGb = 100
 	}
+	if p.config.Tries <= 0 {
+		p.config.Tries = 1000
+	}
 
 	errs = p.config.CommonConfig.Prepare(errs)
 	errs = p.config.ExchangeConfig.Prepare(errs)
@@ -171,26 +174,6 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa
 		return nil, false, false, err
 	}
 
-	imageDesc, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
-		ImageId: imageID,
-	})
-	if err != nil {
-		return nil, false, false, err
-	}
-	secDiskSpec := []*compute.AttachedDiskSpec{
-		{
-			AutoDelete: true,
-			DeviceName: "doexport",
-			Disk: &compute.AttachedDiskSpec_DiskSpec_{
-				DiskSpec: &compute.AttachedDiskSpec_DiskSpec{
-					Source: &compute.AttachedDiskSpec_DiskSpec_ImageId{
-						ImageId: imageID,
-					},
-					Size: imageDesc.MinDiskSize,
-				},
-			},
-		},
-	}
 	// Set up exporter instance configuration.
 	exporterName := fmt.Sprintf("%s-exporter", artifact.Id())
 	yandexConfig := ycSaneDefaults(&p.config,
@@ -226,7 +209,6 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa
 	state.Put("driver", driver)
 	state.Put("sdk", driver.SDK())
 	state.Put("ui", ui)
-	state.Put("secondary_disk", secDiskSpec)
 
 	// Build the steps.
 	steps := []multistep.Step{
@@ -249,8 +231,14 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa
 			Host:      yandex.CommHost,
 			SSHConfig: yandexConfig.Communicator.SSHConfigFunc(),
 		},
+		&StepAttachDisk{
+			CommonConfig: p.config.CommonConfig,
+			ImageID:      imageID,
+		},
 		new(StepUploadSecrets),
-		new(StepWaitCloudInitScript),
+		&StepWaitCloudInitScript{
+			Tries: p.config.Tries,
+		},
 		&yandex.StepTeardownInstance{
 			SerialLogFile: yandexConfig.SerialLogFile,
 		},

post-processor/yandex-export/post-processor.hcl2spec.go

@@ -45,6 +45,7 @@ type FlatConfig struct {
 	ServiceAccountID      *string           `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"`
 	Paths                 []string          `mapstructure:"paths" required:"true" cty:"paths" hcl:"paths"`
 	SSHPrivateKeyFile     *string           `mapstructure:"ssh_private_key_file" required:"false" cty:"ssh_private_key_file" hcl:"ssh_private_key_file"`
+	Tries                 *int              `mapstructure:"tries" required:"false" cty:"tries" hcl:"tries"`
 }
 
 // FlatMapstructure returns a new FlatConfig.
@@ -95,6 +96,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"service_account_id":         &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false},
 		"paths":                      &hcldec.AttrSpec{Name: "paths", Type: cty.List(cty.String), Required: false},
 		"ssh_private_key_file":       &hcldec.AttrSpec{Name: "ssh_private_key_file", Type: cty.String, Required: false},
+		"tries":                      &hcldec.AttrSpec{Name: "tries", Type: cty.Number, Required: false},
 	}
 	return s
 }

post-processor/yandex-export/scripts/export.sh

@@ -4,6 +4,8 @@ GetMetadata() {
     curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null
 }
 
+[[ "$(GetMetadata debug)" == "1"  || "$(GetMetadata debug)" == "true" ]] && set -x
+
 InstallPackages() {
     sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli
 }
@@ -12,7 +14,7 @@ WaitFile() {
     local RETRIES=60
     while [[ ${RETRIES} -gt 0 ]]; do
         echo "Wait ${1}"
-        if [ -f "${1}" ]; then
+        if [ -e "${1}" ]; then
             echo "[${1}] has been found"
             return 0
         fi
@@ -25,6 +27,7 @@ WaitFile() {
 
 PATHS=$(GetMetadata paths)
 S3_ENDPOINT="https://storage.yandexcloud.net"
+DISK_EXPORT_PATH="/dev/disk/by-id/virtio-doexport"
 export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials"
 export AWS_REGION=ru-central1
 
@@ -48,9 +51,15 @@ if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then
     echo "Failed wait credentials"
     Exit 1
 fi
+udevadm trigger || true
+
+if ! WaitFile "${DISK_EXPORT_PATH}"; then
+    echo "Failed wait attach disk"
+    Exit 1
+fi
 
 echo "Dumping disk..."
-if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then
+if ! qemu-img convert -O qcow2 -o cluster_size=2M "${DISK_EXPORT_PATH}" disk.qcow2; then
     echo "Failed to dump disk to qcow2 image."
     Exit 1
 fi

post-processor/yandex-export/step-attach-disk.go

@@ -0,0 +1,100 @@
+package yandexexport
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/hashicorp/packer/builder/yandex"
+	"github.com/hashicorp/packer/packer-plugin-sdk/multistep"
+	"github.com/hashicorp/packer/packer-plugin-sdk/packer"
+	"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
+)
+
+type StepAttachDisk struct {
+	yandex.CommonConfig
+	ImageID string
+}
+
+func (c *StepAttachDisk) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	driver := state.Get("driver").(yandex.Driver)
+	ui := state.Get("ui").(packer.Ui)
+	instanceID := state.Get("instance_id").(string)
+
+	ui.Say("Create secondary disk from image for export...")
+
+	imageDesc, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
+		ImageId: c.ImageID,
+	})
+	if err != nil {
+		return yandex.StepHaltWithError(state, err)
+	}
+
+	op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Disk().Create(ctx, &compute.CreateDiskRequest{
+		Source: &compute.CreateDiskRequest_ImageId{
+			ImageId: c.ImageID,
+		},
+		Name:        fmt.Sprintf("export-%s-disk", instanceID),
+		Size:        imageDesc.GetMinDiskSize(),
+		ZoneId:      c.Zone,
+		FolderId:    c.FolderID,
+		TypeId:      c.DiskType,
+		Description: "Temporary disk for exporting",
+	}))
+	if op == nil {
+		return yandex.StepHaltWithError(state, err)
+	}
+	protoMD, err := op.Metadata()
+	if err != nil {
+		return yandex.StepHaltWithError(state, err)
+	}
+	md, ok := protoMD.(*compute.CreateDiskMetadata)
+	if !ok {
+		return yandex.StepHaltWithError(state, fmt.Errorf("could not get Disk ID from create operation metadata"))
+	}
+	state.Put("secondary_disk_id", md.GetDiskId())
+
+	if err := op.Wait(ctx); err != nil {
+		return yandex.StepHaltWithError(state, err)
+	}
+
+	ui.Say("Attach secondary disk to instance...")
+
+	op, err = driver.SDK().WrapOperation(driver.SDK().Compute().Instance().AttachDisk(ctx, &compute.AttachInstanceDiskRequest{
+		InstanceId: instanceID,
+		AttachedDiskSpec: &compute.AttachedDiskSpec{
+			AutoDelete: true,
+			DeviceName: "doexport",
+			Disk: &compute.AttachedDiskSpec_DiskId{
+				DiskId: md.GetDiskId(),
+			},
+		},
+	}))
+	if err != nil {
+		return yandex.StepHaltWithError(state, err)
+	}
+	ui.Message("Wait attached disk...")
+	if err := op.Wait(ctx); err != nil {
+		return yandex.StepHaltWithError(state, err)
+	}
+
+	state.Remove("secondary_disk_id")
+	return multistep.ActionContinue
+}
+
+func (s *StepAttachDisk) Cleanup(state multistep.StateBag) {
+	ui := state.Get("ui").(packer.Ui)
+	driver := state.Get("driver").(yandex.Driver)
+	if diskID, ok := state.GetOk("secondary_disk_id"); ok {
+		ui.Say("Remove the secondary disk...")
+		op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Disk().Delete(context.Background(), &compute.DeleteDiskRequest{
+			DiskId: diskID.(string),
+		}))
+		if err != nil {
+			ui.Error(err.Error())
+			return
+		}
+		if err := op.Wait(context.Background()); err != nil {
+			ui.Error(err.Error())
+		}
+	}
+}

post-processor/yandex-export/step-wait-cloud-init.go

@@ -14,7 +14,9 @@ import (
 	"github.com/hashicorp/packer/packer-plugin-sdk/retry"
 )
 
-type StepWaitCloudInitScript int
+type StepWaitCloudInitScript struct {
+	Tries int
+}
 
 type cloudInitStatus struct {
 	V1 struct {
@@ -52,7 +54,21 @@ func (s *StepWaitCloudInitScript) Run(ctx context.Context, state multistep.State
 			ui.Error(err.Error())
 			return
 		}
-		ui.Message("Init output closed")
+		ui.Message("Cloud-init output closed")
+	}()
+
+	// periodically show progress by sending SIGUSR1 to `qemu-img` process
+	go func() {
+		cmd := &packersdk.RemoteCmd{
+			Command: "until pid=$(pidof qemu-img) ; do sleep 1 ; done ; " +
+				"while true ; do sudo kill -s SIGUSR1 ${pid}; sleep 10 ; done",
+		}
+
+		err := cmd.RunWithUi(ctxWithCancel, comm, ui)
+		if err != nil && !errors.Is(err, context.Canceled) {
+			ui.Error("qemu-img signal sender error: " + err.Error())
+			return
+		}
 	}()
 
 	// Keep checking the serial port output to see if the cloud-init script is done.
@@ -64,6 +80,7 @@ func (s *StepWaitCloudInitScript) Run(ctx context.Context, state multistep.State
 			}
 			return true
 		},
+		Tries:      s.Tries,
 		RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 2}).Linear,
 	}
 

website/pages/partials/post-processor/yandex-export/Config-not-required.mdx

@@ -3,3 +3,5 @@
 - `ssh_private_key_file` (string) - Path to a PEM encoded private key file to use to authenticate with SSH.
   The `~` can be used in path and will be expanded to the home directory
   of current user. Login for attach: `ubuntu`
+
+- `tries` (int) - Number of attempts to wait for export (must be greater than 0). Default: 1000