iSharkFly-Docs
/
packer-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #10368 from GennadySpb/yandex-export-show-progress 

Yandex export show progress
This commit is contained in:
Megan Marsh 2020-12-11 13:47:38 -08:00 committed by GitHub
parent e89db37717 d0367d90bd
commit edb9d73027
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 158 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
builder/yandex/step_create_instance.go
View File

@ -227,12 +227,6 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
}
}
secDisk, ok := state.GetOk("secondary_disk")
if !ok {
secDisk = []*compute.AttachedDiskSpec{}
}
secDiskSpec := secDisk.([]*compute.AttachedDiskSpec)
req := &compute.CreateInstanceRequest{
FolderId: config.FolderID,
Name: config.InstanceName,
@ -254,7 +248,6 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
DiskId: disk.Id,
},
},
SecondaryDiskSpecs: secDiskSpec,
NetworkInterfaceSpecs: []*compute.NetworkInterfaceSpec{
{
SubnetId: instanceSubnetID,

13
post-processor/yandex-export/cloud-init-script.go
View File

@ -8,6 +8,8 @@ GetMetadata() {
curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null
}
[[ "$(GetMetadata debug)" == "1" || "$(GetMetadata debug)" == "true" ]] && set -x
InstallPackages() {
sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli
}
@ -16,7 +18,7 @@ WaitFile() {
local RETRIES=60
while [[ ${RETRIES} -gt 0 ]]; do
echo "Wait ${1}"
if [ -f "${1}" ]; then
if [ -e "${1}" ]; then
echo "[${1}] has been found"
return 0
fi
@ -29,6 +31,7 @@ WaitFile() {
PATHS=$(GetMetadata paths)
S3_ENDPOINT="https://storage.yandexcloud.net"
DISK_EXPORT_PATH="/dev/disk/by-id/virtio-doexport"
export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials"
export AWS_REGION=ru-central1
@ -52,9 +55,15 @@ if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then
echo "Failed wait credentials"
Exit 1
fi
udevadm trigger || true
if ! WaitFile "${DISK_EXPORT_PATH}"; then
echo "Failed wait attach disk"
Exit 1
fi
echo "Dumping disk..."
if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then
if ! qemu-img convert -O qcow2 -o cluster_size=2M "${DISK_EXPORT_PATH}" disk.qcow2; then
echo "Failed to dump disk to qcow2 image."
Exit 1
fi

38
post-processor/yandex-export/post-processor.go
View File

@ -21,7 +21,6 @@ import (
"github.com/hashicorp/packer/packer-plugin-sdk/template/config"
"github.com/hashicorp/packer/packer-plugin-sdk/template/interpolate"
"github.com/hashicorp/packer/post-processor/artifice"
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1"
ycsdk "github.com/yandex-cloud/go-sdk"
)
@ -48,8 +47,9 @@ type Config struct {
// The `~` can be used in path and will be expanded to the home directory
// of current user. Login for attach: `ubuntu`
SSHPrivateKeyFile string `mapstructure:"ssh_private_key_file" required:"false"`
ctx interpolate.Context
// Number of attempts to wait for export (must be greater than 0). Default: 1000
Tries int `mapstructure:"tries" required:"false"`
ctx interpolate.Context
}
type PostProcessor struct {
@ -83,6 +83,9 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
if p.config.DiskSizeGb == 0 {
p.config.DiskSizeGb = 100
}
if p.config.Tries <= 0 {
p.config.Tries = 1000
}
errs = p.config.CommonConfig.Prepare(errs)
errs = p.config.ExchangeConfig.Prepare(errs)
@ -171,26 +174,6 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa
return nil, false, false, err
}
imageDesc, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
ImageId: imageID,
})
if err != nil {
return nil, false, false, err
}
secDiskSpec := []*compute.AttachedDiskSpec{
{
AutoDelete: true,
DeviceName: "doexport",
Disk: &compute.AttachedDiskSpec_DiskSpec_{
DiskSpec: &compute.AttachedDiskSpec_DiskSpec{
Source: &compute.AttachedDiskSpec_DiskSpec_ImageId{
ImageId: imageID,
},
Size: imageDesc.MinDiskSize,
},
},
},
}
// Set up exporter instance configuration.
exporterName := fmt.Sprintf("%s-exporter", artifact.Id())
yandexConfig := ycSaneDefaults(&p.config,
@ -226,7 +209,6 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa
state.Put("driver", driver)
state.Put("sdk", driver.SDK())
state.Put("ui", ui)
state.Put("secondary_disk", secDiskSpec)
// Build the steps.
steps := []multistep.Step{
@ -249,8 +231,14 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa
Host: yandex.CommHost,
SSHConfig: yandexConfig.Communicator.SSHConfigFunc(),
},
&StepAttachDisk{
CommonConfig: p.config.CommonConfig,
ImageID: imageID,
},
new(StepUploadSecrets),
new(StepWaitCloudInitScript),
&StepWaitCloudInitScript{
Tries: p.config.Tries,
},
&yandex.StepTeardownInstance{
SerialLogFile: yandexConfig.SerialLogFile,
},

2
post-processor/yandex-export/post-processor.hcl2spec.go
View File

@ -45,6 +45,7 @@ type FlatConfig struct {
ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"`
Paths []string `mapstructure:"paths" required:"true" cty:"paths" hcl:"paths"`
SSHPrivateKeyFile *string `mapstructure:"ssh_private_key_file" required:"false" cty:"ssh_private_key_file" hcl:"ssh_private_key_file"`
Tries *int `mapstructure:"tries" required:"false" cty:"tries" hcl:"tries"`
}
// FlatMapstructure returns a new FlatConfig.
@ -95,6 +96,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
"service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false},
"paths": &hcldec.AttrSpec{Name: "paths", Type: cty.List(cty.String), Required: false},
"ssh_private_key_file": &hcldec.AttrSpec{Name: "ssh_private_key_file", Type: cty.String, Required: false},
"tries": &hcldec.AttrSpec{Name: "tries", Type: cty.Number, Required: false},
}
return s
}

13
post-processor/yandex-export/scripts/export.sh
View File

@ -4,6 +4,8 @@ GetMetadata() {
curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null
}
[[ "$(GetMetadata debug)" == "1" || "$(GetMetadata debug)" == "true" ]] && set -x
InstallPackages() {
sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli
}
@ -12,7 +14,7 @@ WaitFile() {
local RETRIES=60
while [[ ${RETRIES} -gt 0 ]]; do
echo "Wait ${1}"
if [ -f "${1}" ]; then
if [ -e "${1}" ]; then
echo "[${1}] has been found"
return 0
fi
@ -25,6 +27,7 @@ WaitFile() {
PATHS=$(GetMetadata paths)
S3_ENDPOINT="https://storage.yandexcloud.net"
DISK_EXPORT_PATH="/dev/disk/by-id/virtio-doexport"
export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials"
export AWS_REGION=ru-central1
@ -48,9 +51,15 @@ if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then
echo "Failed wait credentials"
Exit 1
fi
udevadm trigger || true
if ! WaitFile "${DISK_EXPORT_PATH}"; then
echo "Failed wait attach disk"
Exit 1
fi
echo "Dumping disk..."
if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then
if ! qemu-img convert -O qcow2 -o cluster_size=2M "${DISK_EXPORT_PATH}" disk.qcow2; then
echo "Failed to dump disk to qcow2 image."
Exit 1
fi

100
post-processor/yandex-export/step-attach-disk.go Normal file
View File

@ -0,0 +1,100 @@
package yandexexport
import (
"context"
"fmt"
"github.com/hashicorp/packer/builder/yandex"
"github.com/hashicorp/packer/packer-plugin-sdk/multistep"
"github.com/hashicorp/packer/packer-plugin-sdk/packer"
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
)
type StepAttachDisk struct {
yandex.CommonConfig
ImageID string
}
func (c *StepAttachDisk) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
driver := state.Get("driver").(yandex.Driver)
ui := state.Get("ui").(packer.Ui)
instanceID := state.Get("instance_id").(string)
ui.Say("Create secondary disk from image for export...")
imageDesc, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
ImageId: c.ImageID,
})
if err != nil {
return yandex.StepHaltWithError(state, err)
}
op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Disk().Create(ctx, &compute.CreateDiskRequest{
Source: &compute.CreateDiskRequest_ImageId{
ImageId: c.ImageID,
},
Name: fmt.Sprintf("export-%s-disk", instanceID),
Size: imageDesc.GetMinDiskSize(),
ZoneId: c.Zone,
FolderId: c.FolderID,
TypeId: c.DiskType,
Description: "Temporary disk for exporting",
}))
if op == nil {
return yandex.StepHaltWithError(state, err)
}
protoMD, err := op.Metadata()
if err != nil {
return yandex.StepHaltWithError(state, err)
}
md, ok := protoMD.(*compute.CreateDiskMetadata)
if !ok {
return yandex.StepHaltWithError(state, fmt.Errorf("could not get Disk ID from create operation metadata"))
}
state.Put("secondary_disk_id", md.GetDiskId())
if err := op.Wait(ctx); err != nil {
return yandex.StepHaltWithError(state, err)
}
ui.Say("Attach secondary disk to instance...")
op, err = driver.SDK().WrapOperation(driver.SDK().Compute().Instance().AttachDisk(ctx, &compute.AttachInstanceDiskRequest{
InstanceId: instanceID,
AttachedDiskSpec: &compute.AttachedDiskSpec{
AutoDelete: true,
DeviceName: "doexport",
Disk: &compute.AttachedDiskSpec_DiskId{
DiskId: md.GetDiskId(),
},
},
}))
if err != nil {
return yandex.StepHaltWithError(state, err)
}
ui.Message("Wait attached disk...")
if err := op.Wait(ctx); err != nil {
return yandex.StepHaltWithError(state, err)
}
state.Remove("secondary_disk_id")
return multistep.ActionContinue
}
func (s *StepAttachDisk) Cleanup(state multistep.StateBag) {
ui := state.Get("ui").(packer.Ui)
driver := state.Get("driver").(yandex.Driver)
if diskID, ok := state.GetOk("secondary_disk_id"); ok {
ui.Say("Remove the secondary disk...")
op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Disk().Delete(context.Background(), &compute.DeleteDiskRequest{
DiskId: diskID.(string),
}))
if err != nil {
ui.Error(err.Error())
return
}
if err := op.Wait(context.Background()); err != nil {
ui.Error(err.Error())
}
}
}

21
post-processor/yandex-export/step-wait-cloud-init.go
View File

@ -14,7 +14,9 @@ import (
"github.com/hashicorp/packer/packer-plugin-sdk/retry"
)
type StepWaitCloudInitScript int
type StepWaitCloudInitScript struct {
Tries int
}
type cloudInitStatus struct {
V1 struct {
@ -52,7 +54,21 @@ func (s *StepWaitCloudInitScript) Run(ctx context.Context, state multistep.State
ui.Error(err.Error())
return
}
ui.Message("Init output closed")
ui.Message("Cloud-init output closed")
}()
// periodically show progress by sending SIGUSR1 to `qemu-img` process
go func() {
cmd := &packersdk.RemoteCmd{
Command: "until pid=$(pidof qemu-img) ; do sleep 1 ; done ; " +
"while true ; do sudo kill -s SIGUSR1 ${pid}; sleep 10 ; done",
}
err := cmd.RunWithUi(ctxWithCancel, comm, ui)
if err != nil && !errors.Is(err, context.Canceled) {
ui.Error("qemu-img signal sender error: " + err.Error())
return
}
}()
// Keep checking the serial port output to see if the cloud-init script is done.
@ -64,6 +80,7 @@ func (s *StepWaitCloudInitScript) Run(ctx context.Context, state multistep.State
}
return true
},
Tries: s.Tries,
RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 2}).Linear,
}

2
website/pages/partials/post-processor/yandex-export/Config-not-required.mdx
View File

@ -3,3 +3,5 @@
- `ssh_private_key_file` (string) - Path to a PEM encoded private key file to use to authenticate with SSH.
The `~` can be used in path and will be expanded to the home directory
of current user. Login for attach: `ubuntu`
- `tries` (int) - Number of attempts to wait for export (must be greater than 0). Default: 1000