//go:generate struct-markdown //go:generate mapstructure-to-hcl2 -type Config package yandexexport import ( "context" "fmt" "io/ioutil" "log" "math" "strings" "github.com/c2h5oh/datasize" "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/packer-plugin-sdk/common" "github.com/hashicorp/packer-plugin-sdk/communicator" "github.com/hashicorp/packer-plugin-sdk/multistep" "github.com/hashicorp/packer-plugin-sdk/multistep/commonsteps" "github.com/hashicorp/packer-plugin-sdk/packer" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/packerbuilderdata" "github.com/hashicorp/packer-plugin-sdk/template/config" "github.com/hashicorp/packer-plugin-sdk/template/interpolate" "github.com/hashicorp/packer/builder/file" "github.com/hashicorp/packer/builder/yandex" "github.com/hashicorp/packer/post-processor/artifice" "github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1" "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1" ycsdk "github.com/yandex-cloud/go-sdk" ) const ( defaultStorageEndpoint = "storage.yandexcloud.net" defaultStorageRegion = "ru-central1" defaultSourceImageFamily = "ubuntu-1604-lts" ) type Config struct { common.PackerConfig `mapstructure:",squash"` yandex.AccessConfig `mapstructure:",squash"` yandex.CommonConfig `mapstructure:",squash"` ExchangeConfig `mapstructure:",squash"` communicator.SSH `mapstructure:",squash"` communicator.Config `mapstructure:"-"` // List of paths to Yandex Object Storage where exported image will be uploaded. // Please be aware that use of space char inside path not supported. // Also this param support [build](/docs/templates/legacy_json_templates/engine) template function. // Check available template data for [Yandex](/docs/builders/yandex#build-template-data) builder. // Paths to Yandex Object Storage where exported image will be uploaded. Paths []string `mapstructure:"paths" required:"true"` // The ID of the folder containing the source image. Default `standard-images`. SourceImageFolderID string `mapstructure:"source_image_folder_id" required:"false"` // The source image family to start export process. Default `ubuntu-1604-lts`. // Image must contains utils or supported package manager: `apt` or `yum` - // requires `root` or `sudo` without password. // Utils: `qemu-img`, `aws`. The `qemu-img` utility requires `root` user or // `sudo` access without password. SourceImageFamily string `mapstructure:"source_image_family" required:"false"` // The source image ID to use to create the new image from. Just one of a source_image_id or // source_image_family must be specified. SourceImageID string `mapstructure:"source_image_id" required:"false"` // The extra size of the source disk in GB. This defaults to `0GB`. // Requires `losetup` utility on the instance. // > **Careful!** Increases payment cost. // > See [perfomance](https://cloud.yandex.com/docs/compute/concepts/disk#performance). SourceDiskExtraSize int `mapstructure:"source_disk_extra_size" required:"false"` ctx interpolate.Context } type PostProcessor struct { config Config runner multistep.Runner } func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() } func (p *PostProcessor) Configure(raws ...interface{}) error { err := config.Decode(&p.config, &config.DecodeOpts{ PluginType: BuilderId, Interpolate: true, InterpolateContext: &p.config.ctx, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ "paths", }, }, }, raws...) if err != nil { return err } // Accumulate any errors var errs *packersdk.MultiError errs = packersdk.MultiErrorAppend(errs, p.config.AccessConfig.Prepare(&p.config.ctx)...) // Set defaults. if p.config.DiskSizeGb == 0 { p.config.DiskSizeGb = 100 } if p.config.SSH.SSHUsername == "" { p.config.SSH.SSHUsername = "ubuntu" } p.config.Config = communicator.Config{ Type: "ssh", SSH: p.config.SSH, } errs = packersdk.MultiErrorAppend(errs, p.config.Config.Prepare(&p.config.ctx)...) if p.config.SourceImageID == "" { if p.config.SourceImageFamily == "" { p.config.SourceImageFamily = defaultSourceImageFamily } if p.config.SourceImageFolderID == "" { p.config.SourceImageFolderID = yandex.StandardImagesFolderID } } if p.config.SourceDiskExtraSize < 0 { errs = packer.MultiErrorAppend(errs, fmt.Errorf("source_disk_extra_size must be greater than zero")) } errs = p.config.CommonConfig.Prepare(errs) errs = p.config.ExchangeConfig.Prepare(errs) if len(p.config.Paths) == 0 { errs = packersdk.MultiErrorAppend( errs, fmt.Errorf("paths must be specified")) } // Validate templates in 'paths' for _, path := range p.config.Paths { if err = interpolate.Validate(path, &p.config.ctx); err != nil { errs = packersdk.MultiErrorAppend( errs, fmt.Errorf("Error parsing one of 'paths' template: %s", err)) } } if len(errs.Errors) > 0 { return errs } // Due to the fact that now it's impossible to go to the object storage // through the internal network - we need access // to the global Internet: either through ipv4 or ipv6 // TODO: delete this when access appears if p.config.UseIPv4Nat == false && p.config.UseIPv6 == false { log.Printf("[DEBUG] Force use IPv4") p.config.UseIPv4Nat = true } p.config.Preemptible = true //? safety if p.config.Labels == nil { p.config.Labels = make(map[string]string) } if _, ok := p.config.Labels["role"]; !ok { p.config.Labels["role"] = "exporter" } if _, ok := p.config.Labels["target"]; !ok { p.config.Labels["target"] = "object-storage" } return nil } func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifact packersdk.Artifact) (packersdk.Artifact, bool, bool, error) { imageID := "" switch artifact.BuilderId() { case yandex.BuilderID, artifice.BuilderId: imageID = artifact.State("ImageID").(string) case file.BuilderId: fileName := artifact.Files()[0] if content, err := ioutil.ReadFile(fileName); err == nil { imageID = strings.TrimSpace(string(content)) } else { return nil, false, false, err } default: err := fmt.Errorf( "Unknown artifact type: %s\nCan only export from Yandex Cloud builder artifact or File builder or Artifice post-processor artifact.", artifact.BuilderId()) return nil, false, false, err } // prepare and render values var generatedData map[interface{}]interface{} stateData := artifact.State("generated_data") if stateData != nil { // Make sure it's not a nil map so we can assign to it later. generatedData = stateData.(map[interface{}]interface{}) } // If stateData has a nil map generatedData will be nil // and we need to make sure it's not if generatedData == nil { generatedData = make(map[interface{}]interface{}) } p.config.ctx.Data = generatedData var err error // Render this key since we didn't in the configure phase for i, path := range p.config.Paths { p.config.Paths[i], err = interpolate.Render(path, &p.config.ctx) if err != nil { return nil, false, false, fmt.Errorf("Error rendering one of 'path' template: %s", err) } } log.Printf("Rendered path items: %v", p.config.Paths) ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", imageID, p.config.Paths)) driver, err := yandex.NewDriverYC(ui, &p.config.AccessConfig) if err != nil { return nil, false, false, err } imageDescription, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{ ImageId: imageID, }) if err != nil { return nil, false, false, err } p.config.DiskConfig.DiskSizeGb = chooseBetterDiskSize(ctx, int(imageDescription.GetMinDiskSize()), p.config.DiskConfig.DiskSizeGb) // Set up exporter instance configuration. exporterName := strings.ToLower(fmt.Sprintf("%s-exporter", artifact.Id())) yandexConfig := ycSaneDefaults(&p.config, nil) if yandexConfig.InstanceConfig.InstanceName == "" { yandexConfig.InstanceConfig.InstanceName = exporterName } if yandexConfig.DiskName == "" { yandexConfig.DiskName = exporterName } ui.Say(fmt.Sprintf("Validating service_account_id: '%s'...", yandexConfig.ServiceAccountID)) if err := validateServiceAccount(ctx, driver.SDK(), yandexConfig.ServiceAccountID); err != nil { return nil, false, false, err } // Set up the state. state := new(multistep.BasicStateBag) state.Put("config", &yandexConfig) state.Put("driver", driver) state.Put("sdk", driver.SDK()) state.Put("ui", ui) // Build the steps. steps := []multistep.Step{ &StepCreateS3Keys{ ServiceAccountID: p.config.ServiceAccountID, Paths: p.config.Paths, }, &yandex.StepCreateSSHKey{ Debug: p.config.PackerDebug, DebugKeyPath: fmt.Sprintf("yc_export_pp_%s.pem", p.config.PackerBuildName), }, &yandex.StepCreateInstance{ Debug: p.config.PackerDebug, SerialLogFile: yandexConfig.SerialLogFile, GeneratedData: &packerbuilderdata.GeneratedData{State: state}, }, new(yandex.StepInstanceInfo), &communicator.StepConnect{ Config: &yandexConfig.Communicator, Host: yandex.CommHost, SSHConfig: yandexConfig.Communicator.SSHConfigFunc(), }, &StepAttachDisk{ CommonConfig: p.config.CommonConfig, ImageID: imageID, ExtraSize: p.config.SourceDiskExtraSize, }, new(StepUploadSecrets), new(StepPrepareTools), &StepDump{ ExtraSize: p.config.SourceDiskExtraSize != 0, SizeLimit: imageDescription.GetMinDiskSize(), }, &StepUploadToS3{ Paths: p.config.Paths, }, &yandex.StepTeardownInstance{ SerialLogFile: yandexConfig.SerialLogFile, }, &commonsteps.StepCleanupTempKeys{Comm: &yandexConfig.Communicator}, } // Run the steps. p.runner = commonsteps.NewRunner(steps, p.config.PackerConfig, ui) p.runner.Run(ctx, state) if rawErr, ok := state.GetOk("error"); ok { return nil, false, false, rawErr.(error) } result := &Artifact{ paths: p.config.Paths, urls: formUrls(p.config.Paths), } return result, false, false, nil } func ycSaneDefaults(c *Config, md map[string]string) yandex.Config { yandexConfig := yandex.Config{ CommonConfig: c.CommonConfig, AccessConfig: c.AccessConfig, Communicator: c.Config, } if yandexConfig.Metadata == nil { yandexConfig.Metadata = md } else { for k, v := range md { yandexConfig.Metadata[k] = v } } yandexConfig.SourceImageFamily = c.SourceImageFamily yandexConfig.SourceImageFolderID = c.SourceImageFolderID yandexConfig.SourceImageID = c.SourceImageID yandexConfig.ServiceAccountID = c.ServiceAccountID return yandexConfig } func formUrls(paths []string) []string { result := []string{} for _, path := range paths { url := fmt.Sprintf("https://%s/%s", defaultStorageEndpoint, strings.TrimPrefix(path, "s3://")) result = append(result, url) } return result } func validateServiceAccount(ctx context.Context, ycsdk *ycsdk.SDK, serviceAccountID string) error { _, err := ycsdk.IAM().ServiceAccount().Get(ctx, &iam.GetServiceAccountRequest{ ServiceAccountId: serviceAccountID, }) return err } func chooseBetterDiskSize(ctx context.Context, minSizeBytes, oldSizeGB int) int { max := math.Max(float64(minSizeBytes), float64((datasize.GB * datasize.ByteSize(oldSizeGB)).Bytes())) return int(math.Ceil(datasize.ByteSize(max).GBytes())) }