Add new 'yandex-export' post-processor
This commit is contained in:
Gennady Lipenkov
parent
9489a46f32
commit
d5a6781fb7
|
|
@ -83,6 +83,7 @@ import (
|
|||
vagrantcloudpostprocessor "github.com/hashicorp/packer/post-processor/vagrant-cloud"
|
||||
vspherepostprocessor "github.com/hashicorp/packer/post-processor/vsphere"
|
||||
vspheretemplatepostprocessor "github.com/hashicorp/packer/post-processor/vsphere-template"
|
||||
yandexexportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-export"
|
||||
ansibleprovisioner "github.com/hashicorp/packer/provisioner/ansible"
|
||||
ansiblelocalprovisioner "github.com/hashicorp/packer/provisioner/ansible-local"
|
||||
azuredtlartifactprovisioner "github.com/hashicorp/packer/provisioner/azure-dtlartifact"
|
||||
|
|
@ -202,6 +203,7 @@ var PostProcessors = map[string]packer.PostProcessor{
|
|||
"vagrant-cloud": new(vagrantcloudpostprocessor.PostProcessor),
|
||||
"vsphere": new(vspherepostprocessor.PostProcessor),
|
||||
"vsphere-template": new(vspheretemplatepostprocessor.PostProcessor),
|
||||
"yandex-export": new(yandexexportpostprocessor.PostProcessor),
|
||||
}
|
||||
|
||||
var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
package yandexexport
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const BuilderId = "packer.post-processor.yandex-export"
|
||||
|
||||
type Artifact struct {
|
||||
paths []string
|
||||
}
|
||||
|
||||
func (*Artifact) BuilderId() string {
|
||||
return BuilderId
|
||||
}
|
||||
|
||||
func (*Artifact) Id() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (a *Artifact) Files() []string {
|
||||
pathsCopy := make([]string, len(a.paths))
|
||||
copy(pathsCopy, a.paths)
|
||||
return pathsCopy
|
||||
}
|
||||
|
||||
func (a *Artifact) String() string {
|
||||
return fmt.Sprintf("Exported artifacts in: %s", a.paths)
|
||||
}
|
||||
|
||||
func (*Artifact) State(name string) interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Artifact) Destroy() error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
//go:generate struct-markdown
|
||||
//go:generate mapstructure-to-hcl2 -type Config
|
||||
|
||||
package yandexexport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/hcl/v2/hcldec"
|
||||
"github.com/hashicorp/packer/builder/yandex"
|
||||
"github.com/hashicorp/packer/common"
|
||||
"github.com/hashicorp/packer/helper/config"
|
||||
"github.com/hashicorp/packer/helper/multistep"
|
||||
"github.com/hashicorp/packer/packer"
|
||||
"github.com/hashicorp/packer/template/interpolate"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
common.PackerConfig `mapstructure:",squash"`
|
||||
|
||||
// Paths to Yandex Object Storage where exported image will be uploaded
|
||||
Paths []string `mapstructure:"paths" required:"true"`
|
||||
// The folder ID that will be used to launch a temporary instance.
|
||||
// Alternatively you may set value by environment variable YC_FOLDER_ID.
|
||||
FolderID string `mapstructure:"folder_id" required:"true"`
|
||||
// Service Account ID with proper permission to modify an instance, create and attach disk and
|
||||
// make upload to specific Yandex Object Storage paths
|
||||
ServiceAccountID string `mapstructure:"service_account_id" required:"true"`
|
||||
// The size of the disk in GB. This defaults to `100`, which is 100GB.
|
||||
DiskSizeGb int `mapstructure:"disk_size" required:"false"`
|
||||
// Specify disk type for the launched instance. Defaults to `network-ssd`.
|
||||
DiskType string `mapstructure:"disk_type" required:"false"`
|
||||
// Identifier of the hardware platform configuration for the instance. This defaults to `standard-v2`.
|
||||
PlatformID string `mapstructure:"platform_id" required:"false"`
|
||||
// The Yandex VPC subnet id to use for
|
||||
// the launched instance. Note, the zone of the subnet must match the
|
||||
// zone in which the VM is launched.
|
||||
SubnetID string `mapstructure:"subnet_id" required:"false"`
|
||||
// The name of the zone to launch the instance. This defaults to `ru-central1-a`.
|
||||
Zone string `mapstructure:"zone" required:"false"`
|
||||
// OAuth token to use to authenticate to Yandex.Cloud. Alternatively you may set
|
||||
// value by environment variable YC_TOKEN.
|
||||
Token string `mapstructure:"token" required:"false"`
|
||||
|
||||
ctx interpolate.Context
|
||||
}
|
||||
|
||||
type PostProcessor struct {
|
||||
config Config
|
||||
runner multistep.Runner
|
||||
}
|
||||
|
||||
func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() }
|
||||
|
||||
func (p *PostProcessor) Configure(raws ...interface{}) error {
|
||||
err := config.Decode(&p.config, &config.DecodeOpts{
|
||||
Interpolate: true,
|
||||
InterpolateContext: &p.config.ctx,
|
||||
}, raws...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errs := new(packer.MultiError)
|
||||
|
||||
if len(p.config.Paths) == 0 {
|
||||
errs = packer.MultiErrorAppend(
|
||||
errs, fmt.Errorf("paths must be specified"))
|
||||
}
|
||||
|
||||
// provision config by OS environment variables
|
||||
if p.config.Token == "" {
|
||||
p.config.Token = os.Getenv("YC_TOKEN")
|
||||
}
|
||||
|
||||
if p.config.FolderID == "" {
|
||||
p.config.FolderID = os.Getenv("YC_FOLDER_ID")
|
||||
}
|
||||
|
||||
// Set defaults.
|
||||
if p.config.DiskSizeGb == 0 {
|
||||
p.config.DiskSizeGb = 100
|
||||
}
|
||||
|
||||
if p.config.DiskType == "" {
|
||||
p.config.DiskType = "network-ssd"
|
||||
}
|
||||
|
||||
if p.config.PlatformID == "" {
|
||||
p.config.PlatformID = "standard-v2"
|
||||
}
|
||||
|
||||
if p.config.Zone == "" {
|
||||
p.config.Zone = "ru-central1-a"
|
||||
}
|
||||
|
||||
if len(errs.Errors) > 0 {
|
||||
return errs
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) {
|
||||
if artifact.BuilderId() != yandex.BuilderID {
|
||||
err := fmt.Errorf(
|
||||
"Unknown artifact typs\nCan only export from Yandex Cloud builder artifacts.",
|
||||
artifact.BuilderId())
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
builderID := artifact.State("ImageID").(string)
|
||||
|
||||
ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", builderID, p.config.Paths))
|
||||
|
||||
// Set up exporter instance configuration.
|
||||
exporterName := fmt.Sprintf("%s-exporter", artifact.Id())
|
||||
exporterMetadata := map[string]string{
|
||||
"image_id": builderID,
|
||||
"name": exporterName,
|
||||
"paths": strings.Join(p.config.Paths, " "),
|
||||
"user-data": CloudInitScript,
|
||||
"zone": p.config.Zone,
|
||||
}
|
||||
|
||||
yandexConfig := ycSaneDefaults()
|
||||
yandexConfig.Token = p.config.Token
|
||||
yandexConfig.DiskName = exporterName
|
||||
yandexConfig.InstanceName = exporterName
|
||||
yandexConfig.DiskSizeGb = p.config.DiskSizeGb
|
||||
yandexConfig.Metadata = exporterMetadata
|
||||
yandexConfig.SubnetID = p.config.SubnetID
|
||||
yandexConfig.FolderID = p.config.FolderID
|
||||
yandexConfig.Zone = p.config.Zone
|
||||
|
||||
if p.config.ServiceAccountID != "" {
|
||||
yandexConfig.ServiceAccountID = p.config.ServiceAccountID
|
||||
}
|
||||
if p.config.PlatformID != "" {
|
||||
yandexConfig.ServiceAccountID = p.config.ServiceAccountID
|
||||
}
|
||||
|
||||
driver, err := yandex.NewDriverYC(ui, &yandexConfig)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
|
||||
// Set up the state.
|
||||
state := new(multistep.BasicStateBag)
|
||||
state.Put("config", &yandexConfig)
|
||||
state.Put("driver", driver)
|
||||
state.Put("sdk", driver.SDK())
|
||||
state.Put("ui", ui)
|
||||
|
||||
// Build the steps.
|
||||
steps := []multistep.Step{
|
||||
&yandex.StepCreateSSHKey{
|
||||
Debug: p.config.PackerDebug,
|
||||
DebugKeyPath: fmt.Sprintf("yc_pp_%s.pem", p.config.PackerBuildName),
|
||||
},
|
||||
&yandex.StepCreateInstance{
|
||||
Debug: p.config.PackerDebug,
|
||||
},
|
||||
new(yandex.StepWaitCloudInitScript),
|
||||
new(yandex.StepTeardownInstance),
|
||||
}
|
||||
|
||||
// Run the steps.
|
||||
p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)
|
||||
p.runner.Run(ctx, state)
|
||||
|
||||
result := &Artifact{paths: p.config.Paths}
|
||||
|
||||
return result, false, false, nil
|
||||
}
|
||||
|
||||
func ycSaneDefaults() yandex.Config {
|
||||
return yandex.Config{
|
||||
DiskType: "network-ssd",
|
||||
InstanceCores: 2,
|
||||
InstanceMemory: 2,
|
||||
Labels: map[string]string{
|
||||
"role": "exporter",
|
||||
"target": "object-storage",
|
||||
},
|
||||
PlatformID: "standard-v2",
|
||||
Preemptible: true,
|
||||
SourceImageFamily: "ubuntu-1604-lts",
|
||||
SourceImageFolderID: yandex.StandardImagesFolderID,
|
||||
UseIPv4Nat: true,
|
||||
Zone: "ru-central1-a",
|
||||
StateTimeout: 3 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
// Code generated by "mapstructure-to-hcl2 -type Config"; DO NOT EDIT.
|
||||
package yandexexport
|
||||
|
||||
import (
|
||||
"github.com/hashicorp/hcl/v2/hcldec"
|
||||
"github.com/zclconf/go-cty/cty"
|
||||
)
|
||||
|
||||
// FlatConfig is an auto-generated flat version of Config.
|
||||
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
|
||||
type FlatConfig struct {
|
||||
PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name"`
|
||||
PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type"`
|
||||
PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug"`
|
||||
PackerForce *bool `mapstructure:"packer_force" cty:"packer_force"`
|
||||
PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error"`
|
||||
PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables"`
|
||||
PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables"`
|
||||
Paths []string `mapstructure:"paths" required:"true" cty:"paths"`
|
||||
FolderID *string `mapstructure:"folder_id" required:"true" cty:"folder_id"`
|
||||
ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id"`
|
||||
DiskSizeGb *int `mapstructure:"disk_size" required:"false" cty:"disk_size"`
|
||||
DiskType *string `mapstructure:"disk_type" required:"false" cty:"disk_type"`
|
||||
PlatformID *string `mapstructure:"platform_id" required:"false" cty:"platform_id"`
|
||||
SubnetID *string `mapstructure:"subnet_id" required:"false" cty:"subnet_id"`
|
||||
Zone *string `mapstructure:"zone" required:"false" cty:"zone"`
|
||||
Token *string `mapstructure:"token" required:"false" cty:"token"`
|
||||
}
|
||||
|
||||
// FlatMapstructure returns a new FlatConfig.
|
||||
// FlatConfig is an auto-generated flat version of Config.
|
||||
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
|
||||
func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
|
||||
return new(FlatConfig)
|
||||
}
|
||||
|
||||
// HCL2Spec returns the hcl spec of a Config.
|
||||
// This spec is used by HCL to read the fields of Config.
|
||||
// The decoded values from this spec will then be applied to a FlatConfig.
|
||||
func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
|
||||
s := map[string]hcldec.Spec{
|
||||
"packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false},
|
||||
"packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false},
|
||||
"packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false},
|
||||
"packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false},
|
||||
"packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false},
|
||||
"packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
|
||||
"packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
|
||||
"paths": &hcldec.AttrSpec{Name: "paths", Type: cty.List(cty.String), Required: false},
|
||||
"folder_id": &hcldec.AttrSpec{Name: "folder_id", Type: cty.String, Required: false},
|
||||
"service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false},
|
||||
"disk_size": &hcldec.AttrSpec{Name: "disk_size", Type: cty.Number, Required: false},
|
||||
"disk_type": &hcldec.AttrSpec{Name: "disk_type", Type: cty.String, Required: false},
|
||||
"platform_id": &hcldec.AttrSpec{Name: "platform_id", Type: cty.String, Required: false},
|
||||
"subnet_id": &hcldec.AttrSpec{Name: "subnet_id", Type: cty.String, Required: false},
|
||||
"zone": &hcldec.AttrSpec{Name: "zone", Type: cty.String, Required: false},
|
||||
"token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false},
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
package yandexexport
|
||||
|
||||
var CloudInitScript string = `#!/usr/bin/env bash
|
||||
GetMetadata () {
|
||||
echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2> /dev/null)"
|
||||
}
|
||||
|
||||
GetInstanceId () {
|
||||
echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2> /dev/null)"
|
||||
}
|
||||
|
||||
GetServiceAccountId () {
|
||||
yc compute instance get ${INSTANCE_ID} | grep service_account | cut -f2 -d' '
|
||||
}
|
||||
|
||||
InstallYc () {
|
||||
curl -s https://storage.yandexcloud.net/yandexcloud-yc/install.sh | sudo bash -s -- -n -i /usr/local
|
||||
}
|
||||
|
||||
InstallAwsCli () {
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
|
||||
unzip -o awscliv2.zip > /dev/null
|
||||
sudo ./aws/install
|
||||
}
|
||||
|
||||
InstallPackages () {
|
||||
sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils
|
||||
}
|
||||
|
||||
InstallTools () {
|
||||
InstallPackages
|
||||
InstallYc
|
||||
InstallAwsCli
|
||||
}
|
||||
|
||||
IMAGE_ID=$(GetMetadata image_id)
|
||||
INSTANCE_ID=$(GetInstanceId)
|
||||
DISKNAME=${INSTANCE_ID}-toexport
|
||||
PATHS=$(GetMetadata paths)
|
||||
ZONE=$(GetMetadata zone)
|
||||
|
||||
Exit () {
|
||||
for i in ${PATHS}; do
|
||||
LOGDEST="${i}.exporter.log"
|
||||
echo "Uploading exporter log to ${LOGDEST}..."
|
||||
aws s3 --endpoint-url=https://storage.yandexcloud.net cp /var/log/syslog ${LOGDEST}
|
||||
done
|
||||
exit $1
|
||||
}
|
||||
|
||||
InstallTools
|
||||
|
||||
echo "####### Export configuration #######"
|
||||
echo "Image ID - ${IMAGE_ID}"
|
||||
echo "Instance ID - ${INSTANCE_ID}"
|
||||
echo "Instance zone - ${ZONE}"
|
||||
echo "Disk name - ${DISKNAME}"
|
||||
echo "Export paths - ${PATHS}"
|
||||
echo "####################################"
|
||||
|
||||
echo "Creating disk from image to be exported..."
|
||||
if ! yc compute disk create --name ${DISKNAME} --source-image-id ${IMAGE_ID} --zone ${ZONE}; then
|
||||
echo "Failed to create disk."
|
||||
Exit 1
|
||||
fi
|
||||
|
||||
echo "Attaching disk..."
|
||||
if ! yc compute instance attach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} --device-name doexport --auto-delete ; then
|
||||
echo "Failed to attach disk."
|
||||
Exit 1
|
||||
fi
|
||||
|
||||
echo "Dumping disk..."
|
||||
if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2 ; then
|
||||
echo "Failed to dump disk to qcow2 image."
|
||||
Exit 1
|
||||
fi
|
||||
|
||||
echo "Detaching disk..."
|
||||
if ! yc compute instance detach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} ; then
|
||||
echo "Failed to detach disk."
|
||||
fi
|
||||
|
||||
echo "Detect Service Account ID..."
|
||||
SERVICE_ACCOUNT_ID=$(GetServiceAccountId)
|
||||
echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}"
|
||||
|
||||
echo "Create static access key..."
|
||||
SEC_json=$(yc iam access-key create --service-account-id ${SERVICE_ACCOUNT_ID} \
|
||||
--description "this key is for export image to storage" --format json)
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to create static access key."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setup env variables to access storage..."
|
||||
eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json} )"
|
||||
|
||||
echo "Check access to storage..."
|
||||
if ! aws s3 --endpoint-url=https://storage.yandexcloud.net ls > /dev/null ; then
|
||||
echo "Failed to access storage."
|
||||
fi
|
||||
|
||||
FAIL=0
|
||||
echo "Deleting disk..."
|
||||
if ! yc compute disk delete --name ${DISKNAME} ; then
|
||||
echo "Failed to delete disk."
|
||||
FAIL=1
|
||||
fi
|
||||
for i in ${PATHS}; do
|
||||
echo "Uploading qcow2 disk image to ${i}..."
|
||||
if ! aws s3 --endpoint-url=https://storage.yandexcloud.net cp disk.qcow2 ${i}; then
|
||||
echo "Failed to upload image to ${i}."
|
||||
FAIL=1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Delete static access key..."
|
||||
if ! yc iam access-key delete ${YC_SK_ID} ; then
|
||||
echo "Failed to delete static access key."
|
||||
fi
|
||||
|
||||
echo "Set metadata key to 'cloud-init-status' to 'cloud-init-done' value"
|
||||
if ! yc compute instance update ${INSTANCE_ID} --metadata cloud-init-status=cloud-init-done ; then
|
||||
echo "Failed to attach disk."
|
||||
Exit 1
|
||||
fi
|
||||
|
||||
Exit ${FAIL}`
|
||||
Loading…
Reference in New Issue