diff --git a/CHANGELOG.md b/CHANGELOG.md index b72a388ac..db6f25757 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ * **New Core Feature** provisioners now support a `max_retries` option that can be used for retrying a provisioner on error [GH-9061] +* **New Post-Processor**: `yandex-export` Upload built image in Yandex Object Storage. + ### IMPROVEMENTS: * builder/azure-arm: Add `boot_diag_storage_account` option for enabling boot diagnostics on a virtual machine [GH-9053] diff --git a/CODEOWNERS b/CODEOWNERS index bb1ec41e4..6b145acc6 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -75,5 +75,6 @@ /post-processor/checksum/ v.tolstov@selfip.ru /post-processor/exoscale-import/ @falzm @mcorbin /post-processor/googlecompute-export/ crunkleton@google.com +/post-processor/yandex-export/ @GennadySpb /post-processor/vsphere-template/ nelson@bennu.cl /post-processor/ucloud-import/ @shawnmssu diff --git a/builder/yandex/artifact.go b/builder/yandex/artifact.go index 119bdd916..62b09b26f 100644 --- a/builder/yandex/artifact.go +++ b/builder/yandex/artifact.go @@ -26,7 +26,7 @@ func (a *Artifact) Id() string { } func (*Artifact) Files() []string { - return nil + return []string{""} } //revive:enable:var-naming diff --git a/builder/yandex/builder.go b/builder/yandex/builder.go index c4b544c97..74e701fd9 100644 --- a/builder/yandex/builder.go +++ b/builder/yandex/builder.go @@ -54,11 +54,11 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack // Build the steps steps := []multistep.Step{ - &stepCreateSSHKey{ + &StepCreateSSHKey{ Debug: b.config.PackerDebug, DebugKeyPath: fmt.Sprintf("yc_%s.pem", b.config.PackerBuildName), }, - &stepCreateInstance{ + &StepCreateInstance{ Debug: b.config.PackerDebug, SerialLogFile: b.config.SerialLogFile, }, @@ -72,7 +72,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack &common.StepCleanupTempKeys{ Comm: &b.config.Communicator, }, - &stepTeardownInstance{}, + &StepTeardownInstance{}, &stepCreateImage{}, } @@ -93,6 +93,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack artifact := &Artifact{ image: image.(*compute.Image), config: &b.config, + driver: driver, StateData: map[string]interface{}{"generated_data": state.Get("generated_data")}, } return artifact, nil diff --git a/builder/yandex/driver.go b/builder/yandex/driver.go index 39c38c0a1..97b555d4c 100644 --- a/builder/yandex/driver.go +++ b/builder/yandex/driver.go @@ -16,4 +16,5 @@ type Driver interface { DeleteInstance(ctx context.Context, instanceID string) error DeleteSubnet(ctx context.Context, subnetID string) error DeleteNetwork(ctx context.Context, networkID string) error + GetInstanceMetadata(ctx context.Context, instanceID string, key string) (string, error) } diff --git a/builder/yandex/driver_yc.go b/builder/yandex/driver_yc.go index b9ff35665..0da019b86 100644 --- a/builder/yandex/driver_yc.go +++ b/builder/yandex/driver_yc.go @@ -232,5 +232,22 @@ func (d *driverYC) DeleteDisk(ctx context.Context, diskID string) error { _, err = op.Response() return err - +} + +func (d *driverYC) GetInstanceMetadata(ctx context.Context, instanceID string, key string) (string, error) { + instance, err := d.sdk.Compute().Instance().Get(ctx, &compute.GetInstanceRequest{ + InstanceId: instanceID, + View: compute.InstanceView_FULL, + }) + if err != nil { + return "", err + } + + for k, v := range instance.GetMetadata() { + if k == key { + return v, nil + } + } + + return "", fmt.Errorf("Instance metadata key, %s, not found.", key) } diff --git a/builder/yandex/step_create_instance.go b/builder/yandex/step_create_instance.go index 2c4522d90..377d4f0d2 100644 --- a/builder/yandex/step_create_instance.go +++ b/builder/yandex/step_create_instance.go @@ -18,7 +18,7 @@ import ( const StandardImagesFolderID = "standard-images" -type stepCreateInstance struct { +type StepCreateInstance struct { Debug bool SerialLogFile string } @@ -106,7 +106,7 @@ func getImage(ctx context.Context, c *Config, d Driver) (*Image, error) { return &Image{}, errors.New("neither source_image_name nor source_image_family defined in config") } -func (s *stepCreateInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { +func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { sdk := state.Get("sdk").(*ycsdk.SDK) ui := state.Get("ui").(packer.Ui) config := state.Get("config").(*Config) @@ -265,7 +265,7 @@ runcmd: return multistep.ActionContinue } -func (s *stepCreateInstance) Cleanup(state multistep.StateBag) { +func (s *StepCreateInstance) Cleanup(state multistep.StateBag) { config := state.Get("config").(*Config) driver := state.Get("driver").(Driver) ui := state.Get("ui").(packer.Ui) @@ -339,7 +339,7 @@ func (s *stepCreateInstance) Cleanup(state multistep.StateBag) { } } -func (s *stepCreateInstance) writeSerialLogFile(ctx context.Context, state multistep.StateBag) error { +func (s *StepCreateInstance) writeSerialLogFile(ctx context.Context, state multistep.StateBag) error { sdk := state.Get("sdk").(*ycsdk.SDK) ui := state.Get("ui").(packer.Ui) diff --git a/builder/yandex/step_create_ssh_key.go b/builder/yandex/step_create_ssh_key.go index 6255c3c90..1a66af9f9 100644 --- a/builder/yandex/step_create_ssh_key.go +++ b/builder/yandex/step_create_ssh_key.go @@ -15,12 +15,12 @@ import ( "golang.org/x/crypto/ssh" ) -type stepCreateSSHKey struct { +type StepCreateSSHKey struct { Debug bool DebugKeyPath string } -func (s *stepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { +func (s *StepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) config := state.Get("config").(*Config) @@ -96,5 +96,5 @@ func (s *stepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) mult return multistep.ActionContinue } -func (s *stepCreateSSHKey) Cleanup(state multistep.StateBag) { +func (s *StepCreateSSHKey) Cleanup(state multistep.StateBag) { } diff --git a/builder/yandex/step_teardown_instance.go b/builder/yandex/step_teardown_instance.go index ee3d0dbd3..291fa004e 100644 --- a/builder/yandex/step_teardown_instance.go +++ b/builder/yandex/step_teardown_instance.go @@ -11,9 +11,9 @@ import ( ycsdk "github.com/yandex-cloud/go-sdk" ) -type stepTeardownInstance struct{} +type StepTeardownInstance struct{} -func (s *stepTeardownInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { +func (s *StepTeardownInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { sdk := state.Get("sdk").(*ycsdk.SDK) ui := state.Get("ui").(packer.Ui) c := state.Get("config").(*Config) @@ -52,6 +52,6 @@ func (s *stepTeardownInstance) Run(ctx context.Context, state multistep.StateBag return multistep.ActionContinue } -func (s *stepTeardownInstance) Cleanup(state multistep.StateBag) { +func (s *StepTeardownInstance) Cleanup(state multistep.StateBag) { // no cleanup } diff --git a/builder/yandex/step_wait_cloudinit_script.go b/builder/yandex/step_wait_cloudinit_script.go new file mode 100644 index 000000000..322f98f98 --- /dev/null +++ b/builder/yandex/step_wait_cloudinit_script.go @@ -0,0 +1,69 @@ +package yandex + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/hashicorp/packer/common/retry" + "github.com/hashicorp/packer/helper/multistep" + "github.com/hashicorp/packer/packer" +) + +const CloudInitScriptStatusKey = "cloud-init-status" +const StartupScriptStatusError = "cloud-init-error" +const StartupScriptStatusDone = "cloud-init-done" + +type StepWaitCloudInitScript int + +// Run reads the instance metadata and looks for the log entry +// indicating the cloud-init script finished. +func (s *StepWaitCloudInitScript) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { + _ = state.Get("config").(*Config) + driver := state.Get("driver").(Driver) + ui := state.Get("ui").(packer.Ui) + instanceID := state.Get("instance_id").(string) + + ui.Say("Waiting for any running cloud-init script to finish...") + + // Keep checking the serial port output to see if the cloud-init script is done. + err := retry.Config{ + ShouldRetry: func(error) bool { + return true + }, + RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 2}).Linear, + }.Run(ctx, func(ctx context.Context) error { + status, err := driver.GetInstanceMetadata(ctx, instanceID, CloudInitScriptStatusKey) + + if err != nil { + err := fmt.Errorf("Error getting cloud-init script status: %s", err) + return err + } + + if status == StartupScriptStatusError { + err = errors.New("Cloud-init script error.") + return err + } + + done := status == StartupScriptStatusDone + if !done { + ui.Say("Cloud-init script not finished yet. Waiting...") + return errors.New("Cloud-init script not done.") + } + + return nil + }) + + if err != nil { + err := fmt.Errorf("Error waiting for cloud-init script to finish: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + ui.Say("Cloud-init script has finished running.") + return multistep.ActionContinue +} + +// Cleanup. +func (s *StepWaitCloudInitScript) Cleanup(state multistep.StateBag) {} diff --git a/command/plugin.go b/command/plugin.go index fa57737a7..96d28ae45 100644 --- a/command/plugin.go +++ b/command/plugin.go @@ -83,6 +83,7 @@ import ( vagrantcloudpostprocessor "github.com/hashicorp/packer/post-processor/vagrant-cloud" vspherepostprocessor "github.com/hashicorp/packer/post-processor/vsphere" vspheretemplatepostprocessor "github.com/hashicorp/packer/post-processor/vsphere-template" + yandexexportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-export" ansibleprovisioner "github.com/hashicorp/packer/provisioner/ansible" ansiblelocalprovisioner "github.com/hashicorp/packer/provisioner/ansible-local" azuredtlartifactprovisioner "github.com/hashicorp/packer/provisioner/azure-dtlartifact" @@ -202,6 +203,7 @@ var PostProcessors = map[string]packer.PostProcessor{ "vagrant-cloud": new(vagrantcloudpostprocessor.PostProcessor), "vsphere": new(vspherepostprocessor.PostProcessor), "vsphere-template": new(vspheretemplatepostprocessor.PostProcessor), + "yandex-export": new(yandexexportpostprocessor.PostProcessor), } var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)") diff --git a/post-processor/yandex-export/artifact.go b/post-processor/yandex-export/artifact.go new file mode 100644 index 000000000..3c85f572c --- /dev/null +++ b/post-processor/yandex-export/artifact.go @@ -0,0 +1,37 @@ +package yandexexport + +import ( + "fmt" +) + +const BuilderId = "packer.post-processor.yandex-export" + +type Artifact struct { + paths []string +} + +func (*Artifact) BuilderId() string { + return BuilderId +} + +func (*Artifact) Id() string { + return "" +} + +func (a *Artifact) Files() []string { + pathsCopy := make([]string, len(a.paths)) + copy(pathsCopy, a.paths) + return pathsCopy +} + +func (a *Artifact) String() string { + return fmt.Sprintf("Exported artifacts in: %s", a.paths) +} + +func (*Artifact) State(name string) interface{} { + return nil +} + +func (a *Artifact) Destroy() error { + return nil +} diff --git a/post-processor/yandex-export/post-processor.go b/post-processor/yandex-export/post-processor.go new file mode 100644 index 000000000..1a37bb16b --- /dev/null +++ b/post-processor/yandex-export/post-processor.go @@ -0,0 +1,198 @@ +//go:generate struct-markdown +//go:generate mapstructure-to-hcl2 -type Config + +package yandexexport + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer/builder/yandex" + "github.com/hashicorp/packer/common" + "github.com/hashicorp/packer/helper/config" + "github.com/hashicorp/packer/helper/multistep" + "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/template/interpolate" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + // Paths to Yandex Object Storage where exported image will be uploaded + Paths []string `mapstructure:"paths" required:"true"` + // The folder ID that will be used to launch a temporary instance. + // Alternatively you may set value by environment variable YC_FOLDER_ID. + FolderID string `mapstructure:"folder_id" required:"true"` + // Service Account ID with proper permission to modify an instance, create and attach disk and + // make upload to specific Yandex Object Storage paths + ServiceAccountID string `mapstructure:"service_account_id" required:"true"` + // The size of the disk in GB. This defaults to `100`, which is 100GB. + DiskSizeGb int `mapstructure:"disk_size" required:"false"` + // Specify disk type for the launched instance. Defaults to `network-ssd`. + DiskType string `mapstructure:"disk_type" required:"false"` + // Identifier of the hardware platform configuration for the instance. This defaults to `standard-v2`. + PlatformID string `mapstructure:"platform_id" required:"false"` + // The Yandex VPC subnet id to use for + // the launched instance. Note, the zone of the subnet must match the + // zone in which the VM is launched. + SubnetID string `mapstructure:"subnet_id" required:"false"` + // The name of the zone to launch the instance. This defaults to `ru-central1-a`. + Zone string `mapstructure:"zone" required:"false"` + // OAuth token to use to authenticate to Yandex.Cloud. Alternatively you may set + // value by environment variable YC_TOKEN. + Token string `mapstructure:"token" required:"false"` + + ctx interpolate.Context +} + +type PostProcessor struct { + config Config + runner multistep.Runner +} + +func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() } + +func (p *PostProcessor) Configure(raws ...interface{}) error { + err := config.Decode(&p.config, &config.DecodeOpts{ + Interpolate: true, + InterpolateContext: &p.config.ctx, + }, raws...) + if err != nil { + return err + } + + errs := new(packer.MultiError) + + if len(p.config.Paths) == 0 { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("paths must be specified")) + } + + // provision config by OS environment variables + if p.config.Token == "" { + p.config.Token = os.Getenv("YC_TOKEN") + } + + if p.config.FolderID == "" { + p.config.FolderID = os.Getenv("YC_FOLDER_ID") + } + + // Set defaults. + if p.config.DiskSizeGb == 0 { + p.config.DiskSizeGb = 100 + } + + if p.config.DiskType == "" { + p.config.DiskType = "network-ssd" + } + + if p.config.PlatformID == "" { + p.config.PlatformID = "standard-v2" + } + + if p.config.Zone == "" { + p.config.Zone = "ru-central1-a" + } + + if len(errs.Errors) > 0 { + return errs + } + + return nil +} + +func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { + if artifact.BuilderId() != yandex.BuilderID { + err := fmt.Errorf( + "Unknown artifact type: %s\nCan only export from Yandex Cloud builder artifacts.", + artifact.BuilderId()) + return nil, false, false, err + } + + builderID := artifact.State("ImageID").(string) + + ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", builderID, p.config.Paths)) + + // Set up exporter instance configuration. + exporterName := fmt.Sprintf("%s-exporter", artifact.Id()) + exporterMetadata := map[string]string{ + "image_id": builderID, + "name": exporterName, + "paths": strings.Join(p.config.Paths, " "), + "user-data": CloudInitScript, + "zone": p.config.Zone, + } + + yandexConfig := ycSaneDefaults() + yandexConfig.Token = p.config.Token + yandexConfig.DiskName = exporterName + yandexConfig.InstanceName = exporterName + yandexConfig.DiskSizeGb = p.config.DiskSizeGb + yandexConfig.Metadata = exporterMetadata + yandexConfig.SubnetID = p.config.SubnetID + yandexConfig.FolderID = p.config.FolderID + yandexConfig.Zone = p.config.Zone + + if p.config.ServiceAccountID != "" { + yandexConfig.ServiceAccountID = p.config.ServiceAccountID + } + if p.config.PlatformID != "" { + yandexConfig.ServiceAccountID = p.config.ServiceAccountID + } + + driver, err := yandex.NewDriverYC(ui, &yandexConfig) + if err != nil { + return nil, false, false, err + } + + // Set up the state. + state := new(multistep.BasicStateBag) + state.Put("config", &yandexConfig) + state.Put("driver", driver) + state.Put("sdk", driver.SDK()) + state.Put("ui", ui) + + // Build the steps. + steps := []multistep.Step{ + &yandex.StepCreateSSHKey{ + Debug: p.config.PackerDebug, + DebugKeyPath: fmt.Sprintf("yc_pp_%s.pem", p.config.PackerBuildName), + }, + &yandex.StepCreateInstance{ + Debug: p.config.PackerDebug, + }, + new(yandex.StepWaitCloudInitScript), + new(yandex.StepTeardownInstance), + } + + // Run the steps. + p.runner = common.NewRunner(steps, p.config.PackerConfig, ui) + p.runner.Run(ctx, state) + + result := &Artifact{paths: p.config.Paths} + + return result, false, false, nil +} + +func ycSaneDefaults() yandex.Config { + return yandex.Config{ + DiskType: "network-ssd", + InstanceCores: 2, + InstanceMemory: 2, + Labels: map[string]string{ + "role": "exporter", + "target": "object-storage", + }, + PlatformID: "standard-v2", + Preemptible: true, + SourceImageFamily: "ubuntu-1604-lts", + SourceImageFolderID: yandex.StandardImagesFolderID, + UseIPv4Nat: true, + Zone: "ru-central1-a", + StateTimeout: 3 * time.Minute, + } +} diff --git a/post-processor/yandex-export/post-processor.hcl2spec.go b/post-processor/yandex-export/post-processor.hcl2spec.go new file mode 100644 index 000000000..099bad0f2 --- /dev/null +++ b/post-processor/yandex-export/post-processor.hcl2spec.go @@ -0,0 +1,60 @@ +// Code generated by "mapstructure-to-hcl2 -type Config"; DO NOT EDIT. +package yandexexport + +import ( + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +// FlatConfig is an auto-generated flat version of Config. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatConfig struct { + PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name"` + PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type"` + PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug"` + PackerForce *bool `mapstructure:"packer_force" cty:"packer_force"` + PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error"` + PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables"` + PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables"` + Paths []string `mapstructure:"paths" required:"true" cty:"paths"` + FolderID *string `mapstructure:"folder_id" required:"true" cty:"folder_id"` + ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id"` + DiskSizeGb *int `mapstructure:"disk_size" required:"false" cty:"disk_size"` + DiskType *string `mapstructure:"disk_type" required:"false" cty:"disk_type"` + PlatformID *string `mapstructure:"platform_id" required:"false" cty:"platform_id"` + SubnetID *string `mapstructure:"subnet_id" required:"false" cty:"subnet_id"` + Zone *string `mapstructure:"zone" required:"false" cty:"zone"` + Token *string `mapstructure:"token" required:"false" cty:"token"` +} + +// FlatMapstructure returns a new FlatConfig. +// FlatConfig is an auto-generated flat version of Config. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatConfig) +} + +// HCL2Spec returns the hcl spec of a Config. +// This spec is used by HCL to read the fields of Config. +// The decoded values from this spec will then be applied to a FlatConfig. +func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false}, + "packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false}, + "packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false}, + "packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false}, + "packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false}, + "packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false}, + "packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false}, + "paths": &hcldec.AttrSpec{Name: "paths", Type: cty.List(cty.String), Required: false}, + "folder_id": &hcldec.AttrSpec{Name: "folder_id", Type: cty.String, Required: false}, + "service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false}, + "disk_size": &hcldec.AttrSpec{Name: "disk_size", Type: cty.Number, Required: false}, + "disk_type": &hcldec.AttrSpec{Name: "disk_type", Type: cty.String, Required: false}, + "platform_id": &hcldec.AttrSpec{Name: "platform_id", Type: cty.String, Required: false}, + "subnet_id": &hcldec.AttrSpec{Name: "subnet_id", Type: cty.String, Required: false}, + "zone": &hcldec.AttrSpec{Name: "zone", Type: cty.String, Required: false}, + "token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false}, + } + return s +} diff --git a/post-processor/yandex-export/script.go b/post-processor/yandex-export/script.go new file mode 100644 index 000000000..de30d7844 --- /dev/null +++ b/post-processor/yandex-export/script.go @@ -0,0 +1,130 @@ +package yandexexport + +var CloudInitScript string = `#!/usr/bin/env bash +GetMetadata () { + echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2> /dev/null)" +} + +GetInstanceId () { + echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2> /dev/null)" +} + +GetServiceAccountId () { + yc compute instance get ${INSTANCE_ID} | grep service_account | cut -f2 -d' ' +} + +InstallYc () { + curl -s https://storage.yandexcloud.net/yandexcloud-yc/install.sh | sudo bash -s -- -n -i /usr/local +} + +InstallAwsCli () { + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o awscliv2.zip > /dev/null + sudo ./aws/install +} + +InstallPackages () { + sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils +} + +InstallTools () { + InstallPackages + InstallYc + InstallAwsCli +} + +IMAGE_ID=$(GetMetadata image_id) +INSTANCE_ID=$(GetInstanceId) +DISKNAME=${INSTANCE_ID}-toexport +PATHS=$(GetMetadata paths) +ZONE=$(GetMetadata zone) + +Exit () { + for i in ${PATHS}; do + LOGDEST="${i}.exporter.log" + echo "Uploading exporter log to ${LOGDEST}..." + aws s3 --endpoint-url=https://storage.yandexcloud.net cp /var/log/syslog ${LOGDEST} + done + exit $1 +} + +InstallTools + +echo "####### Export configuration #######" +echo "Image ID - ${IMAGE_ID}" +echo "Instance ID - ${INSTANCE_ID}" +echo "Instance zone - ${ZONE}" +echo "Disk name - ${DISKNAME}" +echo "Export paths - ${PATHS}" +echo "####################################" + +echo "Creating disk from image to be exported..." +if ! yc compute disk create --name ${DISKNAME} --source-image-id ${IMAGE_ID} --zone ${ZONE}; then + echo "Failed to create disk." + Exit 1 +fi + +echo "Attaching disk..." +if ! yc compute instance attach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} --device-name doexport --auto-delete ; then + echo "Failed to attach disk." + Exit 1 +fi + +echo "Dumping disk..." +if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2 ; then + echo "Failed to dump disk to qcow2 image." + Exit 1 +fi + +echo "Detaching disk..." +if ! yc compute instance detach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} ; then + echo "Failed to detach disk." +fi + +echo "Detect Service Account ID..." +SERVICE_ACCOUNT_ID=$(GetServiceAccountId) +echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}" + +echo "Create static access key..." +SEC_json=$(yc iam access-key create --service-account-id ${SERVICE_ACCOUNT_ID} \ + --description "this key is for export image to storage" --format json) + +if [ $? -ne 0 ]; then + echo "Failed to create static access key." + exit 1 +fi + +echo "Setup env variables to access storage..." +eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json} )" + +echo "Check access to storage..." +if ! aws s3 --endpoint-url=https://storage.yandexcloud.net ls > /dev/null ; then + echo "Failed to access storage." +fi + +FAIL=0 +echo "Deleting disk..." +if ! yc compute disk delete --name ${DISKNAME} ; then + echo "Failed to delete disk." + FAIL=1 +fi +for i in ${PATHS}; do + echo "Uploading qcow2 disk image to ${i}..." + if ! aws s3 --endpoint-url=https://storage.yandexcloud.net cp disk.qcow2 ${i}; then + echo "Failed to upload image to ${i}." + FAIL=1 + fi +done + +echo "Delete static access key..." +if ! yc iam access-key delete ${YC_SK_ID} ; then + echo "Failed to delete static access key." +fi + +echo "Set metadata key to 'cloud-init-status' to 'cloud-init-done' value" +if ! yc compute instance update ${INSTANCE_ID} --metadata cloud-init-status=cloud-init-done ; then + echo "Failed to attach disk." + Exit 1 +fi + +Exit ${FAIL}` diff --git a/website/data/docs-navigation.js b/website/data/docs-navigation.js index 77c72bb39..9afe6a4aa 100644 --- a/website/data/docs-navigation.js +++ b/website/data/docs-navigation.js @@ -256,6 +256,7 @@ export default [ 'vagrant-cloud', 'vsphere', 'vsphere-template', + 'yandex-export', ], }, '----------', diff --git a/website/pages/docs/post-processors/yandex-export.mdx b/website/pages/docs/post-processors/yandex-export.mdx new file mode 100644 index 000000000..56fc2484d --- /dev/null +++ b/website/pages/docs/post-processors/yandex-export.mdx @@ -0,0 +1,88 @@ +--- +description: > + The Yandex.Cloud Compute Image Exporter post-processor exports an image from a + Packer + + yandex builder run and uploads it to Yandex Object Storage. The exported + + images can be easily shared and uploaded to other Yandex.Cloud Cloud folders. +layout: docs +page_title: Yandex.Cloud Compute Image Exporter - Post-Processors +sidebar_title: Yandex.Cloud Compute Export +--- + +# Yandex.Cloud Compute Image Exporter Post-Processor + +Type: `yandex-export` + +The Yandex.Cloud Compute Image Exporter post-processor exports the resultant image +from a yandex build as a qcow2 file to Yandex Object Storage. + +The exporter uses the same Yandex.Cloud folder and +authentication credentials as the yandex build that produced the image. +A temporary VM is started in the folder using these credentials. The VM +mounts the built image as a secondary disk, then dumps the image in qcow2 format. +The VM then uploads the file to the provided Yandex Object Storage `paths` using the same +credentials. + +As such, assigned Service Account must have write permissions to the Yandex Object Storage +`paths`. A new temporary static access keys from assigned Service Account used to upload +image. + + +## Configuration + + +### Required: + +@include 'post-processor/yandex-export/Config-required.mdx' + +### Optional: + +@include 'post-processor/yandex-export/Config-not-required.mdx' + + +## Basic Example + +The following example builds a Compute image in the folder with id `b1g8jvfcgmitdrslcn86`, with an +Service Account whose keyfile is `account.json`. After the image build, a temporary VM +will be created to export the image as a qcow2 file to +`s3://packer-export/my-exported-image.qcow2` and +`s3://packer-export/image-number-two.qcow2`. `keep_input_artifact` is true, so the +source Compute image won't be deleted after the export. + +In order for this example to work, the service account associated with builder +must have write access to both `s3://packer-export/my-exported-image.qcow2` and +`s3://packer-export/image-number-two.qcow2` and get permission to modify temporary instance +(create new disk, attach to instance, etc). + +```json +{ + "builders": [ + { + "type": "yandex", + "folder_id": "b1g8jvfcgmitdrslcn86", + "subnet_id": "e9bp6l8sa4q39yourxzq", + "zone": "ru-central1-a" + + "source_image_family": "ubuntu-1604-lts", + "ssh_username": "ubuntu", + "use_ipv4_nat": true, + } + ], + "post-processors": [ + { + "type": "yandex-export", + "folder_id": "b1g8jvfcgmitdrslcn86", + "subnet_id": "e9bp6l8sa4q39yourxzq", + + "service_account_id": "ajeu0363240rrnn7xgen", + + "paths": [ + "s3://packer-export-bucket/my-exported-image.qcow2", + "s3://packer-export-bucket/image-number-two.qcow2" + ], + "keep_input_artifact": true + } + ] +}``` diff --git a/website/pages/partials/post-processor/yandex-export/Config-not-required.mdx b/website/pages/partials/post-processor/yandex-export/Config-not-required.mdx new file mode 100644 index 000000000..9facc490e --- /dev/null +++ b/website/pages/partials/post-processor/yandex-export/Config-not-required.mdx @@ -0,0 +1,17 @@ + + +- `disk_size` (int) - The size of the disk in GB. This defaults to `100`, which is 100GB. + +- `disk_type` (string) - Specify disk type for the launched instance. Defaults to `network-ssd`. + +- `platform_id` (string) - Identifier of the hardware platform configuration for the instance. This defaults to `standard-v2`. + +- `subnet_id` (string) - The Yandex VPC subnet id to use for + the launched instance. Note, the zone of the subnet must match the + zone in which the VM is launched. + +- `zone` (string) - The name of the zone to launch the instance. This defaults to `ru-central1-a`. + +- `token` (string) - OAuth token to use to authenticate to Yandex.Cloud. Alternatively you may set + value by environment variable YC_TOKEN. + \ No newline at end of file diff --git a/website/pages/partials/post-processor/yandex-export/Config-required.mdx b/website/pages/partials/post-processor/yandex-export/Config-required.mdx new file mode 100644 index 000000000..65944cc23 --- /dev/null +++ b/website/pages/partials/post-processor/yandex-export/Config-required.mdx @@ -0,0 +1,10 @@ + + +- `paths` ([]string) - Paths to Yandex Object Storage where exported image will be uploaded + +- `folder_id` (string) - The folder ID that will be used to launch a temporary instance. + Alternatively you may set value by environment variable YC_FOLDER_ID. + +- `service_account_id` (string) - Service Account ID with proper permission to modify an instance, create and attach disk and + make upload to specific Yandex Object Storage paths + \ No newline at end of file