Merge pull request #9124 from GennadySpb/yandex-export-post-processor
[WIP] Yandex export post processor
This commit is contained in:
Megan Marsh
GitHub
commit
6a682aca2e
|
|
@ -13,6 +13,8 @@
|
|||
* **New Core Feature** provisioners now support a `max_retries` option that can
|
||||
be used for retrying a provisioner on error [GH-9061]
|
||||
|
||||
* **New Post-Processor**: `yandex-export` Upload built image in Yandex Object Storage.
|
||||
|
||||
### IMPROVEMENTS:
|
||||
* builder/azure-arm: Add `boot_diag_storage_account` option for enabling boot
|
||||
diagnostics on a virtual machine [GH-9053]
|
||||
|
|
|
|||
|
|
@ -75,5 +75,6 @@
|
|||
/post-processor/checksum/ v.tolstov@selfip.ru
|
||||
/post-processor/exoscale-import/ @falzm @mcorbin
|
||||
/post-processor/googlecompute-export/ crunkleton@google.com
|
||||
/post-processor/yandex-export/ @GennadySpb
|
||||
/post-processor/vsphere-template/ nelson@bennu.cl
|
||||
/post-processor/ucloud-import/ @shawnmssu
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ func (a *Artifact) Id() string {
|
|||
}
|
||||
|
||||
func (*Artifact) Files() []string {
|
||||
return nil
|
||||
return []string{""}
|
||||
}
|
||||
|
||||
//revive:enable:var-naming
|
||||
|
|
|
|||
|
|
@ -54,11 +54,11 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
|
|||
|
||||
// Build the steps
|
||||
steps := []multistep.Step{
|
||||
&stepCreateSSHKey{
|
||||
&StepCreateSSHKey{
|
||||
Debug: b.config.PackerDebug,
|
||||
DebugKeyPath: fmt.Sprintf("yc_%s.pem", b.config.PackerBuildName),
|
||||
},
|
||||
&stepCreateInstance{
|
||||
&StepCreateInstance{
|
||||
Debug: b.config.PackerDebug,
|
||||
SerialLogFile: b.config.SerialLogFile,
|
||||
},
|
||||
|
|
@ -72,7 +72,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
|
|||
&common.StepCleanupTempKeys{
|
||||
Comm: &b.config.Communicator,
|
||||
},
|
||||
&stepTeardownInstance{},
|
||||
&StepTeardownInstance{},
|
||||
&stepCreateImage{},
|
||||
}
|
||||
|
||||
|
|
@ -93,6 +93,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
|
|||
artifact := &Artifact{
|
||||
image: image.(*compute.Image),
|
||||
config: &b.config,
|
||||
driver: driver,
|
||||
StateData: map[string]interface{}{"generated_data": state.Get("generated_data")},
|
||||
}
|
||||
return artifact, nil
|
||||
|
|
|
|||
|
|
@ -16,4 +16,5 @@ type Driver interface {
|
|||
DeleteInstance(ctx context.Context, instanceID string) error
|
||||
DeleteSubnet(ctx context.Context, subnetID string) error
|
||||
DeleteNetwork(ctx context.Context, networkID string) error
|
||||
GetInstanceMetadata(ctx context.Context, instanceID string, key string) (string, error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -232,5 +232,22 @@ func (d *driverYC) DeleteDisk(ctx context.Context, diskID string) error {
|
|||
|
||||
_, err = op.Response()
|
||||
return err
|
||||
|
||||
}
|
||||
|
||||
func (d *driverYC) GetInstanceMetadata(ctx context.Context, instanceID string, key string) (string, error) {
|
||||
instance, err := d.sdk.Compute().Instance().Get(ctx, &compute.GetInstanceRequest{
|
||||
InstanceId: instanceID,
|
||||
View: compute.InstanceView_FULL,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for k, v := range instance.GetMetadata() {
|
||||
if k == key {
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("Instance metadata key, %s, not found.", key)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import (
|
|||
|
||||
const StandardImagesFolderID = "standard-images"
|
||||
|
||||
type stepCreateInstance struct {
|
||||
type StepCreateInstance struct {
|
||||
Debug bool
|
||||
SerialLogFile string
|
||||
}
|
||||
|
|
@ -106,7 +106,7 @@ func getImage(ctx context.Context, c *Config, d Driver) (*Image, error) {
|
|||
return &Image{}, errors.New("neither source_image_name nor source_image_family defined in config")
|
||||
}
|
||||
|
||||
func (s *stepCreateInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
|
||||
func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
|
||||
sdk := state.Get("sdk").(*ycsdk.SDK)
|
||||
ui := state.Get("ui").(packer.Ui)
|
||||
config := state.Get("config").(*Config)
|
||||
|
|
@ -265,7 +265,7 @@ runcmd:
|
|||
return multistep.ActionContinue
|
||||
}
|
||||
|
||||
func (s *stepCreateInstance) Cleanup(state multistep.StateBag) {
|
||||
func (s *StepCreateInstance) Cleanup(state multistep.StateBag) {
|
||||
config := state.Get("config").(*Config)
|
||||
driver := state.Get("driver").(Driver)
|
||||
ui := state.Get("ui").(packer.Ui)
|
||||
|
|
@ -339,7 +339,7 @@ func (s *stepCreateInstance) Cleanup(state multistep.StateBag) {
|
|||
}
|
||||
}
|
||||
|
||||
func (s *stepCreateInstance) writeSerialLogFile(ctx context.Context, state multistep.StateBag) error {
|
||||
func (s *StepCreateInstance) writeSerialLogFile(ctx context.Context, state multistep.StateBag) error {
|
||||
sdk := state.Get("sdk").(*ycsdk.SDK)
|
||||
ui := state.Get("ui").(packer.Ui)
|
||||
|
||||
|
|
|
|||
|
|
@ -15,12 +15,12 @@ import (
|
|||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type stepCreateSSHKey struct {
|
||||
type StepCreateSSHKey struct {
|
||||
Debug bool
|
||||
DebugKeyPath string
|
||||
}
|
||||
|
||||
func (s *stepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
|
||||
func (s *StepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
|
||||
ui := state.Get("ui").(packer.Ui)
|
||||
config := state.Get("config").(*Config)
|
||||
|
||||
|
|
@ -96,5 +96,5 @@ func (s *stepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) mult
|
|||
return multistep.ActionContinue
|
||||
}
|
||||
|
||||
func (s *stepCreateSSHKey) Cleanup(state multistep.StateBag) {
|
||||
func (s *StepCreateSSHKey) Cleanup(state multistep.StateBag) {
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,9 +11,9 @@ import (
|
|||
ycsdk "github.com/yandex-cloud/go-sdk"
|
||||
)
|
||||
|
||||
type stepTeardownInstance struct{}
|
||||
type StepTeardownInstance struct{}
|
||||
|
||||
func (s *stepTeardownInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
|
||||
func (s *StepTeardownInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
|
||||
sdk := state.Get("sdk").(*ycsdk.SDK)
|
||||
ui := state.Get("ui").(packer.Ui)
|
||||
c := state.Get("config").(*Config)
|
||||
|
|
@ -52,6 +52,6 @@ func (s *stepTeardownInstance) Run(ctx context.Context, state multistep.StateBag
|
|||
return multistep.ActionContinue
|
||||
}
|
||||
|
||||
func (s *stepTeardownInstance) Cleanup(state multistep.StateBag) {
|
||||
func (s *StepTeardownInstance) Cleanup(state multistep.StateBag) {
|
||||
// no cleanup
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
package yandex
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/packer/common/retry"
|
||||
"github.com/hashicorp/packer/helper/multistep"
|
||||
"github.com/hashicorp/packer/packer"
|
||||
)
|
||||
|
||||
const CloudInitScriptStatusKey = "cloud-init-status"
|
||||
const StartupScriptStatusError = "cloud-init-error"
|
||||
const StartupScriptStatusDone = "cloud-init-done"
|
||||
|
||||
type StepWaitCloudInitScript int
|
||||
|
||||
// Run reads the instance metadata and looks for the log entry
|
||||
// indicating the cloud-init script finished.
|
||||
func (s *StepWaitCloudInitScript) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
|
||||
_ = state.Get("config").(*Config)
|
||||
driver := state.Get("driver").(Driver)
|
||||
ui := state.Get("ui").(packer.Ui)
|
||||
instanceID := state.Get("instance_id").(string)
|
||||
|
||||
ui.Say("Waiting for any running cloud-init script to finish...")
|
||||
|
||||
// Keep checking the serial port output to see if the cloud-init script is done.
|
||||
err := retry.Config{
|
||||
ShouldRetry: func(error) bool {
|
||||
return true
|
||||
},
|
||||
RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 2}).Linear,
|
||||
}.Run(ctx, func(ctx context.Context) error {
|
||||
status, err := driver.GetInstanceMetadata(ctx, instanceID, CloudInitScriptStatusKey)
|
||||
|
||||
if err != nil {
|
||||
err := fmt.Errorf("Error getting cloud-init script status: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if status == StartupScriptStatusError {
|
||||
err = errors.New("Cloud-init script error.")
|
||||
return err
|
||||
}
|
||||
|
||||
done := status == StartupScriptStatusDone
|
||||
if !done {
|
||||
ui.Say("Cloud-init script not finished yet. Waiting...")
|
||||
return errors.New("Cloud-init script not done.")
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
err := fmt.Errorf("Error waiting for cloud-init script to finish: %s", err)
|
||||
state.Put("error", err)
|
||||
ui.Error(err.Error())
|
||||
return multistep.ActionHalt
|
||||
}
|
||||
ui.Say("Cloud-init script has finished running.")
|
||||
return multistep.ActionContinue
|
||||
}
|
||||
|
||||
// Cleanup.
|
||||
func (s *StepWaitCloudInitScript) Cleanup(state multistep.StateBag) {}
|
||||
|
|
@ -83,6 +83,7 @@ import (
|
|||
vagrantcloudpostprocessor "github.com/hashicorp/packer/post-processor/vagrant-cloud"
|
||||
vspherepostprocessor "github.com/hashicorp/packer/post-processor/vsphere"
|
||||
vspheretemplatepostprocessor "github.com/hashicorp/packer/post-processor/vsphere-template"
|
||||
yandexexportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-export"
|
||||
ansibleprovisioner "github.com/hashicorp/packer/provisioner/ansible"
|
||||
ansiblelocalprovisioner "github.com/hashicorp/packer/provisioner/ansible-local"
|
||||
azuredtlartifactprovisioner "github.com/hashicorp/packer/provisioner/azure-dtlartifact"
|
||||
|
|
@ -202,6 +203,7 @@ var PostProcessors = map[string]packer.PostProcessor{
|
|||
"vagrant-cloud": new(vagrantcloudpostprocessor.PostProcessor),
|
||||
"vsphere": new(vspherepostprocessor.PostProcessor),
|
||||
"vsphere-template": new(vspheretemplatepostprocessor.PostProcessor),
|
||||
"yandex-export": new(yandexexportpostprocessor.PostProcessor),
|
||||
}
|
||||
|
||||
var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
package yandexexport
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const BuilderId = "packer.post-processor.yandex-export"
|
||||
|
||||
type Artifact struct {
|
||||
paths []string
|
||||
}
|
||||
|
||||
func (*Artifact) BuilderId() string {
|
||||
return BuilderId
|
||||
}
|
||||
|
||||
func (*Artifact) Id() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (a *Artifact) Files() []string {
|
||||
pathsCopy := make([]string, len(a.paths))
|
||||
copy(pathsCopy, a.paths)
|
||||
return pathsCopy
|
||||
}
|
||||
|
||||
func (a *Artifact) String() string {
|
||||
return fmt.Sprintf("Exported artifacts in: %s", a.paths)
|
||||
}
|
||||
|
||||
func (*Artifact) State(name string) interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Artifact) Destroy() error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
//go:generate struct-markdown
|
||||
//go:generate mapstructure-to-hcl2 -type Config
|
||||
|
||||
package yandexexport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/hcl/v2/hcldec"
|
||||
"github.com/hashicorp/packer/builder/yandex"
|
||||
"github.com/hashicorp/packer/common"
|
||||
"github.com/hashicorp/packer/helper/config"
|
||||
"github.com/hashicorp/packer/helper/multistep"
|
||||
"github.com/hashicorp/packer/packer"
|
||||
"github.com/hashicorp/packer/template/interpolate"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
common.PackerConfig `mapstructure:",squash"`
|
||||
|
||||
// Paths to Yandex Object Storage where exported image will be uploaded
|
||||
Paths []string `mapstructure:"paths" required:"true"`
|
||||
// The folder ID that will be used to launch a temporary instance.
|
||||
// Alternatively you may set value by environment variable YC_FOLDER_ID.
|
||||
FolderID string `mapstructure:"folder_id" required:"true"`
|
||||
// Service Account ID with proper permission to modify an instance, create and attach disk and
|
||||
// make upload to specific Yandex Object Storage paths
|
||||
ServiceAccountID string `mapstructure:"service_account_id" required:"true"`
|
||||
// The size of the disk in GB. This defaults to `100`, which is 100GB.
|
||||
DiskSizeGb int `mapstructure:"disk_size" required:"false"`
|
||||
// Specify disk type for the launched instance. Defaults to `network-ssd`.
|
||||
DiskType string `mapstructure:"disk_type" required:"false"`
|
||||
// Identifier of the hardware platform configuration for the instance. This defaults to `standard-v2`.
|
||||
PlatformID string `mapstructure:"platform_id" required:"false"`
|
||||
// The Yandex VPC subnet id to use for
|
||||
// the launched instance. Note, the zone of the subnet must match the
|
||||
// zone in which the VM is launched.
|
||||
SubnetID string `mapstructure:"subnet_id" required:"false"`
|
||||
// The name of the zone to launch the instance. This defaults to `ru-central1-a`.
|
||||
Zone string `mapstructure:"zone" required:"false"`
|
||||
// OAuth token to use to authenticate to Yandex.Cloud. Alternatively you may set
|
||||
// value by environment variable YC_TOKEN.
|
||||
Token string `mapstructure:"token" required:"false"`
|
||||
|
||||
ctx interpolate.Context
|
||||
}
|
||||
|
||||
type PostProcessor struct {
|
||||
config Config
|
||||
runner multistep.Runner
|
||||
}
|
||||
|
||||
func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() }
|
||||
|
||||
func (p *PostProcessor) Configure(raws ...interface{}) error {
|
||||
err := config.Decode(&p.config, &config.DecodeOpts{
|
||||
Interpolate: true,
|
||||
InterpolateContext: &p.config.ctx,
|
||||
}, raws...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errs := new(packer.MultiError)
|
||||
|
||||
if len(p.config.Paths) == 0 {
|
||||
errs = packer.MultiErrorAppend(
|
||||
errs, fmt.Errorf("paths must be specified"))
|
||||
}
|
||||
|
||||
// provision config by OS environment variables
|
||||
if p.config.Token == "" {
|
||||
p.config.Token = os.Getenv("YC_TOKEN")
|
||||
}
|
||||
|
||||
if p.config.FolderID == "" {
|
||||
p.config.FolderID = os.Getenv("YC_FOLDER_ID")
|
||||
}
|
||||
|
||||
// Set defaults.
|
||||
if p.config.DiskSizeGb == 0 {
|
||||
p.config.DiskSizeGb = 100
|
||||
}
|
||||
|
||||
if p.config.DiskType == "" {
|
||||
p.config.DiskType = "network-ssd"
|
||||
}
|
||||
|
||||
if p.config.PlatformID == "" {
|
||||
p.config.PlatformID = "standard-v2"
|
||||
}
|
||||
|
||||
if p.config.Zone == "" {
|
||||
p.config.Zone = "ru-central1-a"
|
||||
}
|
||||
|
||||
if len(errs.Errors) > 0 {
|
||||
return errs
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) {
|
||||
if artifact.BuilderId() != yandex.BuilderID {
|
||||
err := fmt.Errorf(
|
||||
"Unknown artifact type: %s\nCan only export from Yandex Cloud builder artifacts.",
|
||||
artifact.BuilderId())
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
builderID := artifact.State("ImageID").(string)
|
||||
|
||||
ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", builderID, p.config.Paths))
|
||||
|
||||
// Set up exporter instance configuration.
|
||||
exporterName := fmt.Sprintf("%s-exporter", artifact.Id())
|
||||
exporterMetadata := map[string]string{
|
||||
"image_id": builderID,
|
||||
"name": exporterName,
|
||||
"paths": strings.Join(p.config.Paths, " "),
|
||||
"user-data": CloudInitScript,
|
||||
"zone": p.config.Zone,
|
||||
}
|
||||
|
||||
yandexConfig := ycSaneDefaults()
|
||||
yandexConfig.Token = p.config.Token
|
||||
yandexConfig.DiskName = exporterName
|
||||
yandexConfig.InstanceName = exporterName
|
||||
yandexConfig.DiskSizeGb = p.config.DiskSizeGb
|
||||
yandexConfig.Metadata = exporterMetadata
|
||||
yandexConfig.SubnetID = p.config.SubnetID
|
||||
yandexConfig.FolderID = p.config.FolderID
|
||||
yandexConfig.Zone = p.config.Zone
|
||||
|
||||
if p.config.ServiceAccountID != "" {
|
||||
yandexConfig.ServiceAccountID = p.config.ServiceAccountID
|
||||
}
|
||||
if p.config.PlatformID != "" {
|
||||
yandexConfig.ServiceAccountID = p.config.ServiceAccountID
|
||||
}
|
||||
|
||||
driver, err := yandex.NewDriverYC(ui, &yandexConfig)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
// Set up the state.
|
||||
state := new(multistep.BasicStateBag)
|
||||
state.Put("config", &yandexConfig)
|
||||
state.Put("driver", driver)
|
||||
state.Put("sdk", driver.SDK())
|
||||
state.Put("ui", ui)
|
||||
|
||||
// Build the steps.
|
||||
steps := []multistep.Step{
|
||||
&yandex.StepCreateSSHKey{
|
||||
Debug: p.config.PackerDebug,
|
||||
DebugKeyPath: fmt.Sprintf("yc_pp_%s.pem", p.config.PackerBuildName),
|
||||
},
|
||||
&yandex.StepCreateInstance{
|
||||
Debug: p.config.PackerDebug,
|
||||
},
|
||||
new(yandex.StepWaitCloudInitScript),
|
||||
new(yandex.StepTeardownInstance),
|
||||
}
|
||||
|
||||
// Run the steps.
|
||||
p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)
|
||||
p.runner.Run(ctx, state)
|
||||
|
||||
result := &Artifact{paths: p.config.Paths}
|
||||
|
||||
return result, false, false, nil
|
||||
}
|
||||
|
||||
func ycSaneDefaults() yandex.Config {
|
||||
return yandex.Config{
|
||||
DiskType: "network-ssd",
|
||||
InstanceCores: 2,
|
||||
InstanceMemory: 2,
|
||||
Labels: map[string]string{
|
||||
"role": "exporter",
|
||||
"target": "object-storage",
|
||||
},
|
||||
PlatformID: "standard-v2",
|
||||
Preemptible: true,
|
||||
SourceImageFamily: "ubuntu-1604-lts",
|
||||
SourceImageFolderID: yandex.StandardImagesFolderID,
|
||||
UseIPv4Nat: true,
|
||||
Zone: "ru-central1-a",
|
||||
StateTimeout: 3 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
// Code generated by "mapstructure-to-hcl2 -type Config"; DO NOT EDIT.
|
||||
package yandexexport
|
||||
|
||||
import (
|
||||
"github.com/hashicorp/hcl/v2/hcldec"
|
||||
"github.com/zclconf/go-cty/cty"
|
||||
)
|
||||
|
||||
// FlatConfig is an auto-generated flat version of Config.
|
||||
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
|
||||
type FlatConfig struct {
|
||||
PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name"`
|
||||
PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type"`
|
||||
PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug"`
|
||||
PackerForce *bool `mapstructure:"packer_force" cty:"packer_force"`
|
||||
PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error"`
|
||||
PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables"`
|
||||
PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables"`
|
||||
Paths []string `mapstructure:"paths" required:"true" cty:"paths"`
|
||||
FolderID *string `mapstructure:"folder_id" required:"true" cty:"folder_id"`
|
||||
ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id"`
|
||||
DiskSizeGb *int `mapstructure:"disk_size" required:"false" cty:"disk_size"`
|
||||
DiskType *string `mapstructure:"disk_type" required:"false" cty:"disk_type"`
|
||||
PlatformID *string `mapstructure:"platform_id" required:"false" cty:"platform_id"`
|
||||
SubnetID *string `mapstructure:"subnet_id" required:"false" cty:"subnet_id"`
|
||||
Zone *string `mapstructure:"zone" required:"false" cty:"zone"`
|
||||
Token *string `mapstructure:"token" required:"false" cty:"token"`
|
||||
}
|
||||
|
||||
// FlatMapstructure returns a new FlatConfig.
|
||||
// FlatConfig is an auto-generated flat version of Config.
|
||||
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
|
||||
func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
|
||||
return new(FlatConfig)
|
||||
}
|
||||
|
||||
// HCL2Spec returns the hcl spec of a Config.
|
||||
// This spec is used by HCL to read the fields of Config.
|
||||
// The decoded values from this spec will then be applied to a FlatConfig.
|
||||
func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
|
||||
s := map[string]hcldec.Spec{
|
||||
"packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false},
|
||||
"packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false},
|
||||
"packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false},
|
||||
"packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false},
|
||||
"packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false},
|
||||
"packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
|
||||
"packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
|
||||
"paths": &hcldec.AttrSpec{Name: "paths", Type: cty.List(cty.String), Required: false},
|
||||
"folder_id": &hcldec.AttrSpec{Name: "folder_id", Type: cty.String, Required: false},
|
||||
"service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false},
|
||||
"disk_size": &hcldec.AttrSpec{Name: "disk_size", Type: cty.Number, Required: false},
|
||||
"disk_type": &hcldec.AttrSpec{Name: "disk_type", Type: cty.String, Required: false},
|
||||
"platform_id": &hcldec.AttrSpec{Name: "platform_id", Type: cty.String, Required: false},
|
||||
"subnet_id": &hcldec.AttrSpec{Name: "subnet_id", Type: cty.String, Required: false},
|
||||
"zone": &hcldec.AttrSpec{Name: "zone", Type: cty.String, Required: false},
|
||||
"token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false},
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
package yandexexport
|
||||
|
||||
var CloudInitScript string = `#!/usr/bin/env bash
|
||||
GetMetadata () {
|
||||
echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2> /dev/null)"
|
||||
}
|
||||
|
||||
GetInstanceId () {
|
||||
echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2> /dev/null)"
|
||||
}
|
||||
|
||||
GetServiceAccountId () {
|
||||
yc compute instance get ${INSTANCE_ID} | grep service_account | cut -f2 -d' '
|
||||
}
|
||||
|
||||
InstallYc () {
|
||||
curl -s https://storage.yandexcloud.net/yandexcloud-yc/install.sh | sudo bash -s -- -n -i /usr/local
|
||||
}
|
||||
|
||||
InstallAwsCli () {
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
|
||||
unzip -o awscliv2.zip > /dev/null
|
||||
sudo ./aws/install
|
||||
}
|
||||
|
||||
InstallPackages () {
|
||||
sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils
|
||||
}
|
||||
|
||||
InstallTools () {
|
||||
InstallPackages
|
||||
InstallYc
|
||||
InstallAwsCli
|
||||
}
|
||||
|
||||
IMAGE_ID=$(GetMetadata image_id)
|
||||
INSTANCE_ID=$(GetInstanceId)
|
||||
DISKNAME=${INSTANCE_ID}-toexport
|
||||
PATHS=$(GetMetadata paths)
|
||||
ZONE=$(GetMetadata zone)
|
||||
|
||||
Exit () {
|
||||
for i in ${PATHS}; do
|
||||
LOGDEST="${i}.exporter.log"
|
||||
echo "Uploading exporter log to ${LOGDEST}..."
|
||||
aws s3 --endpoint-url=https://storage.yandexcloud.net cp /var/log/syslog ${LOGDEST}
|
||||
done
|
||||
exit $1
|
||||
}
|
||||
|
||||
InstallTools
|
||||
|
||||
echo "####### Export configuration #######"
|
||||
echo "Image ID - ${IMAGE_ID}"
|
||||
echo "Instance ID - ${INSTANCE_ID}"
|
||||
echo "Instance zone - ${ZONE}"
|
||||
echo "Disk name - ${DISKNAME}"
|
||||
echo "Export paths - ${PATHS}"
|
||||
echo "####################################"
|
||||
|
||||
echo "Creating disk from image to be exported..."
|
||||
if ! yc compute disk create --name ${DISKNAME} --source-image-id ${IMAGE_ID} --zone ${ZONE}; then
|
||||
echo "Failed to create disk."
|
||||
Exit 1
|
||||
fi
|
||||
|
||||
echo "Attaching disk..."
|
||||
if ! yc compute instance attach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} --device-name doexport --auto-delete ; then
|
||||
echo "Failed to attach disk."
|
||||
Exit 1
|
||||
fi
|
||||
|
||||
echo "Dumping disk..."
|
||||
if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2 ; then
|
||||
echo "Failed to dump disk to qcow2 image."
|
||||
Exit 1
|
||||
fi
|
||||
|
||||
echo "Detaching disk..."
|
||||
if ! yc compute instance detach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} ; then
|
||||
echo "Failed to detach disk."
|
||||
fi
|
||||
|
||||
echo "Detect Service Account ID..."
|
||||
SERVICE_ACCOUNT_ID=$(GetServiceAccountId)
|
||||
echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}"
|
||||
|
||||
echo "Create static access key..."
|
||||
SEC_json=$(yc iam access-key create --service-account-id ${SERVICE_ACCOUNT_ID} \
|
||||
--description "this key is for export image to storage" --format json)
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to create static access key."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setup env variables to access storage..."
|
||||
eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json} )"
|
||||
|
||||
echo "Check access to storage..."
|
||||
if ! aws s3 --endpoint-url=https://storage.yandexcloud.net ls > /dev/null ; then
|
||||
echo "Failed to access storage."
|
||||
fi
|
||||
|
||||
FAIL=0
|
||||
echo "Deleting disk..."
|
||||
if ! yc compute disk delete --name ${DISKNAME} ; then
|
||||
echo "Failed to delete disk."
|
||||
FAIL=1
|
||||
fi
|
||||
for i in ${PATHS}; do
|
||||
echo "Uploading qcow2 disk image to ${i}..."
|
||||
if ! aws s3 --endpoint-url=https://storage.yandexcloud.net cp disk.qcow2 ${i}; then
|
||||
echo "Failed to upload image to ${i}."
|
||||
FAIL=1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Delete static access key..."
|
||||
if ! yc iam access-key delete ${YC_SK_ID} ; then
|
||||
echo "Failed to delete static access key."
|
||||
fi
|
||||
|
||||
echo "Set metadata key to 'cloud-init-status' to 'cloud-init-done' value"
|
||||
if ! yc compute instance update ${INSTANCE_ID} --metadata cloud-init-status=cloud-init-done ; then
|
||||
echo "Failed to attach disk."
|
||||
Exit 1
|
||||
fi
|
||||
|
||||
Exit ${FAIL}`
|
||||
|
|
@ -256,6 +256,7 @@ export default [
|
|||
'vagrant-cloud',
|
||||
'vsphere',
|
||||
'vsphere-template',
|
||||
'yandex-export',
|
||||
],
|
||||
},
|
||||
'----------',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
---
|
||||
description: >
|
||||
The Yandex.Cloud Compute Image Exporter post-processor exports an image from a
|
||||
Packer
|
||||
|
||||
yandex builder run and uploads it to Yandex Object Storage. The exported
|
||||
|
||||
images can be easily shared and uploaded to other Yandex.Cloud Cloud folders.
|
||||
layout: docs
|
||||
page_title: Yandex.Cloud Compute Image Exporter - Post-Processors
|
||||
sidebar_title: Yandex.Cloud Compute Export
|
||||
---
|
||||
|
||||
# Yandex.Cloud Compute Image Exporter Post-Processor
|
||||
|
||||
Type: `yandex-export`
|
||||
|
||||
The Yandex.Cloud Compute Image Exporter post-processor exports the resultant image
|
||||
from a yandex build as a qcow2 file to Yandex Object Storage.
|
||||
|
||||
The exporter uses the same Yandex.Cloud folder and
|
||||
authentication credentials as the yandex build that produced the image.
|
||||
A temporary VM is started in the folder using these credentials. The VM
|
||||
mounts the built image as a secondary disk, then dumps the image in qcow2 format.
|
||||
The VM then uploads the file to the provided Yandex Object Storage `paths` using the same
|
||||
credentials.
|
||||
|
||||
As such, assigned Service Account must have write permissions to the Yandex Object Storage
|
||||
`paths`. A new temporary static access keys from assigned Service Account used to upload
|
||||
image.
|
||||
|
||||
|
||||
## Configuration
|
||||
|
||||
|
||||
### Required:
|
||||
|
||||
@include 'post-processor/yandex-export/Config-required.mdx'
|
||||
|
||||
### Optional:
|
||||
|
||||
@include 'post-processor/yandex-export/Config-not-required.mdx'
|
||||
|
||||
|
||||
## Basic Example
|
||||
|
||||
The following example builds a Compute image in the folder with id `b1g8jvfcgmitdrslcn86`, with an
|
||||
Service Account whose keyfile is `account.json`. After the image build, a temporary VM
|
||||
will be created to export the image as a qcow2 file to
|
||||
`s3://packer-export/my-exported-image.qcow2` and
|
||||
`s3://packer-export/image-number-two.qcow2`. `keep_input_artifact` is true, so the
|
||||
source Compute image won't be deleted after the export.
|
||||
|
||||
In order for this example to work, the service account associated with builder
|
||||
must have write access to both `s3://packer-export/my-exported-image.qcow2` and
|
||||
`s3://packer-export/image-number-two.qcow2` and get permission to modify temporary instance
|
||||
(create new disk, attach to instance, etc).
|
||||
|
||||
```json
|
||||
{
|
||||
"builders": [
|
||||
{
|
||||
"type": "yandex",
|
||||
"folder_id": "b1g8jvfcgmitdrslcn86",
|
||||
"subnet_id": "e9bp6l8sa4q39yourxzq",
|
||||
"zone": "ru-central1-a"
|
||||
|
||||
"source_image_family": "ubuntu-1604-lts",
|
||||
"ssh_username": "ubuntu",
|
||||
"use_ipv4_nat": true,
|
||||
}
|
||||
],
|
||||
"post-processors": [
|
||||
{
|
||||
"type": "yandex-export",
|
||||
"folder_id": "b1g8jvfcgmitdrslcn86",
|
||||
"subnet_id": "e9bp6l8sa4q39yourxzq",
|
||||
|
||||
"service_account_id": "ajeu0363240rrnn7xgen",
|
||||
|
||||
"paths": [
|
||||
"s3://packer-export-bucket/my-exported-image.qcow2",
|
||||
"s3://packer-export-bucket/image-number-two.qcow2"
|
||||
],
|
||||
"keep_input_artifact": true
|
||||
}
|
||||
]
|
||||
}```
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
<!-- Code generated from the comments of the Config struct in post-processor/yandex-export/post-processor.go; DO NOT EDIT MANUALLY -->
|
||||
|
||||
- `disk_size` (int) - The size of the disk in GB. This defaults to `100`, which is 100GB.
|
||||
|
||||
- `disk_type` (string) - Specify disk type for the launched instance. Defaults to `network-ssd`.
|
||||
|
||||
- `platform_id` (string) - Identifier of the hardware platform configuration for the instance. This defaults to `standard-v2`.
|
||||
|
||||
- `subnet_id` (string) - The Yandex VPC subnet id to use for
|
||||
the launched instance. Note, the zone of the subnet must match the
|
||||
zone in which the VM is launched.
|
||||
|
||||
- `zone` (string) - The name of the zone to launch the instance. This defaults to `ru-central1-a`.
|
||||
|
||||
- `token` (string) - OAuth token to use to authenticate to Yandex.Cloud. Alternatively you may set
|
||||
value by environment variable YC_TOKEN.
|
||||
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
<!-- Code generated from the comments of the Config struct in post-processor/yandex-export/post-processor.go; DO NOT EDIT MANUALLY -->
|
||||
|
||||
- `paths` ([]string) - Paths to Yandex Object Storage where exported image will be uploaded
|
||||
|
||||
- `folder_id` (string) - The folder ID that will be used to launch a temporary instance.
|
||||
Alternatively you may set value by environment variable YC_FOLDER_ID.
|
||||
|
||||
- `service_account_id` (string) - Service Account ID with proper permission to modify an instance, create and attach disk and
|
||||
make upload to specific Yandex Object Storage paths
|
||||
|
||||
Loading…
Reference in New Issue