iSharkFly-Docs
/
packer-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Yandex/ssh communicator in export (#10352) 

* use ssh for communicate to export
* Update post-processor/yandex-export/step-create-s3-keys.go

Co-authored-by: GennadySpb <lipenkov@gmail.com>
This commit is contained in:
Roman Mingazeev 2020-12-09 18:51:34 +03:00 committed by GitHub
parent 355b93730b
commit aa0efcf73e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 387 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
builder/yandex/builder.go
View File

@ -77,10 +77,10 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
SerialLogFile: b.config.SerialLogFile,
GeneratedData: generatedData,
},
&stepInstanceInfo{},
&StepInstanceInfo{},
&communicator.StepConnect{
Config: &b.config.Communicator,
Host: commHost,
Host: CommHost,
SSHConfig: b.config.Communicator.SSHConfigFunc(),
},
&commonsteps.StepProvision{},

2
builder/yandex/ssh.go
View File

@ -4,7 +4,7 @@ import (
"github.com/hashicorp/packer/packer-plugin-sdk/multistep"
)
func commHost(state multistep.StateBag) (string, error) {
func CommHost(state multistep.StateBag) (string, error) {
ipAddress := state.Get("instance_ip").(string)
return ipAddress, nil
}

8
builder/yandex/step_create_image.go
View File

@ -41,22 +41,22 @@ func (s *stepCreateImage) Run(ctx context.Context, state multistep.StateBag) mul
},
}))
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error creating image: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error creating image: %s", err))
}
ui.Say("Waiting for image to complete...")
if err := op.Wait(ctx); err != nil {
return stepHaltWithError(state, fmt.Errorf("Error waiting for image: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error waiting for image: %s", err))
}
resp, err := op.Response()
if err != nil {
return stepHaltWithError(state, err)
return StepHaltWithError(state, err)
}
image, ok := resp.(*compute.Image)
if !ok {
return stepHaltWithError(state, errors.New("API call response doesn't contain Compute Image"))
return StepHaltWithError(state, errors.New("API call response doesn't contain Compute Image"))
}
log.Printf("Image ID: %s", image.Id)

26
builder/yandex/step_create_instance.go
View File

@ -165,11 +165,11 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
sourceImage, err := getImage(ctx, config, driver)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error getting source image for instance creation: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error getting source image for instance creation: %s", err))
}
if sourceImage.MinDiskSizeGb > config.DiskSizeGb {
return stepHaltWithError(state, fmt.Errorf("Instance DiskSizeGb (%d) should be equal or greater "+
return StepHaltWithError(state, fmt.Errorf("Instance DiskSizeGb (%d) should be equal or greater "+
"than SourceImage disk requirement (%d)", config.DiskSizeGb, sourceImage.MinDiskSizeGb))
}
@ -182,14 +182,14 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
ui.Say("Creating network...")
network, err := createNetwork(ctx, config, driver)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error creating network: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error creating network: %s", err))
}
state.Put("network_id", network.Id)
ui.Say(fmt.Sprintf("Creating subnet in zone %q...", config.Zone))
subnet, err := createSubnet(ctx, config, driver, network.Id)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error creating subnet: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error creating subnet: %s", err))
}
instanceSubnetID = subnet.Id
// save for cleanup
@ -203,7 +203,7 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
ui.Say("Creating disk...")
disk, err := createDisk(ctx, state, config, driver, sourceImage)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error creating disk: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error creating disk: %s", err))
}
// Create an instance based on the configuration
@ -211,7 +211,7 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
instanceMetadata, err := config.createInstanceMetadata(string(config.Communicator.SSHPublicKey))
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error preparing instance metadata: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error preparing instance metadata: %s", err))
}
if config.UseIPv6 {
@ -223,7 +223,7 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
}
instanceMetadata["user-data"], err = MergeCloudUserMetaData(oldUserData, cloudInitIPv6Config)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error merge user data configs: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error merge user data configs: %s", err))
}
}
@ -281,33 +281,33 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag)
op, err := sdk.WrapOperation(sdk.Compute().Instance().Create(ctx, req))
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error create instance: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error create instance: %s", err))
}
opMetadata, err := op.Metadata()
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error get create operation metadata: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error get create operation metadata: %s", err))
}
if cimd, ok := opMetadata.(*compute.CreateInstanceMetadata); ok {
state.Put("instance_id", cimd.InstanceId)
} else {
return stepHaltWithError(state, fmt.Errorf("could not get Instance ID from operation metadata"))
return StepHaltWithError(state, fmt.Errorf("could not get Instance ID from operation metadata"))
}
err = op.Wait(ctx)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error create instance: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error create instance: %s", err))
}
resp, err := op.Response()
if err != nil {
return stepHaltWithError(state, err)
return StepHaltWithError(state, err)
}
instance, ok := resp.(*compute.Instance)
if !ok {
return stepHaltWithError(state, fmt.Errorf("response doesn't contain Instance"))
return StepHaltWithError(state, fmt.Errorf("response doesn't contain Instance"))
}
// instance_id is the generic term used so that users can have access to the

4
builder/yandex/step_create_ssh_key.go
View File

@ -50,7 +50,7 @@ func (s *StepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) mult
priv, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error generating temporary SSH key: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error generating temporary SSH key: %s", err))
}
// ASN.1 DER encoded form
@ -89,7 +89,7 @@ func (s *StepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) mult
ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath))
err := ioutil.WriteFile(s.DebugKeyPath, config.Communicator.SSHPrivateKey, 0600)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error saving debug key: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error saving debug key: %s", err))
}
}

10
builder/yandex/step_instance_info.go
View File

@ -12,9 +12,9 @@ import (
ycsdk "github.com/yandex-cloud/go-sdk"
)
type stepInstanceInfo struct{}
type StepInstanceInfo struct{}
func (s *stepInstanceInfo) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
func (s *StepInstanceInfo) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
sdk := state.Get("sdk").(*ycsdk.SDK)
ui := state.Get("ui").(packersdk.Ui)
c := state.Get("config").(*Config)
@ -30,12 +30,12 @@ func (s *stepInstanceInfo) Run(ctx context.Context, state multistep.StateBag) mu
View: compute.InstanceView_FULL,
})
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error retrieving instance data: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error retrieving instance data: %s", err))
}
instanceIP, err := getInstanceIPAddress(c, instance)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Failed to find instance ip address: %s", err))
return StepHaltWithError(state, fmt.Errorf("Failed to find instance ip address: %s", err))
}
state.Put("instance_ip", instanceIP)
@ -106,6 +106,6 @@ func instanceAddresses(instance *compute.Instance) (ipV4Int, ipV4Ext, ipV6 strin
return
}
func (s *stepInstanceInfo) Cleanup(state multistep.StateBag) {
func (s *StepInstanceInfo) Cleanup(state multistep.StateBag) {
// no cleanup
}

8
builder/yandex/step_teardown_instance.go
View File

@ -37,11 +37,11 @@ func (s *StepTeardownInstance) Run(ctx context.Context, state multistep.StateBag
InstanceId: instanceID,
}))
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error stopping instance: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error stopping instance: %s", err))
}
err = op.Wait(ctx)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error stopping instance: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error stopping instance: %s", err))
}
ui.Say("Deleting instance...")
@ -49,11 +49,11 @@ func (s *StepTeardownInstance) Run(ctx context.Context, state multistep.StateBag
InstanceId: instanceID,
}))
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error deleting instance: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error deleting instance: %s", err))
}
err = op.Wait(ctx)
if err != nil {
return stepHaltWithError(state, fmt.Errorf("Error deleting instance: %s", err))
return StepHaltWithError(state, fmt.Errorf("Error deleting instance: %s", err))
}
ui.Message("Instance has been deleted!")

69
builder/yandex/step_wait_cloudinit_script.go
View File

@ -1,69 +0,0 @@
package yandex
import (
"context"
"errors"
"fmt"
"time"
"github.com/hashicorp/packer/packer-plugin-sdk/multistep"
packersdk "github.com/hashicorp/packer/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/packer-plugin-sdk/retry"
)
const CloudInitScriptStatusKey = "cloud-init-status"
const StartupScriptStatusError = "cloud-init-error"
const StartupScriptStatusDone = "cloud-init-done"
type StepWaitCloudInitScript int
// Run reads the instance metadata and looks for the log entry
// indicating the cloud-init script finished.
func (*StepWaitCloudInitScript) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
_ = state.Get("config").(*Config)
driver := state.Get("driver").(Driver)
ui := state.Get("ui").(packersdk.Ui)
instanceID := state.Get("instance_id").(string)
ui.Say("Waiting for any running cloud-init script to finish...")
// Keep checking the serial port output to see if the cloud-init script is done.
err := retry.Config{
ShouldRetry: func(error) bool {
return true
},
RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 2}).Linear,
}.Run(ctx, func(ctx context.Context) error {
status, err := driver.GetInstanceMetadata(ctx, instanceID, CloudInitScriptStatusKey)
if err != nil {
err := fmt.Errorf("Error getting cloud-init script status: %s", err)
return err
}
if status == StartupScriptStatusError {
err = errors.New("Cloud-init script error.")
return err
}
done := status == StartupScriptStatusDone
if !done {
ui.Say("Cloud-init script not finished yet. Waiting...")
return errors.New("Cloud-init script not done.")
}
return nil
})
if err != nil {
err := fmt.Errorf("Error waiting for cloud-init script to finish: %s", err)
state.Put("error", err)
ui.Error(err.Error())
return multistep.ActionHalt
}
ui.Say("Cloud-init script has finished running.")
return multistep.ActionContinue
}
// Cleanup.
func (s *StepWaitCloudInitScript) Cleanup(state multistep.StateBag) {}

2
builder/yandex/util.go
View File

@ -12,7 +12,7 @@ import (
ycsdk "github.com/yandex-cloud/go-sdk"
)
func stepHaltWithError(state multistep.StateBag, err error) multistep.StepAction {
func StepHaltWithError(state multistep.StateBag, err error) multistep.StepAction {
ui := state.Get("ui").(packersdk.Ui)
state.Put("error", err)
ui.Error(err.Error())

91
post-processor/yandex-export/cloud-init-script.go
View File

@ -8,94 +8,51 @@ GetMetadata() {
curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null
}
GetInstanceId() {
curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2>/dev/null
}
GetServiceAccountId() {
yc compute instance get "${INSTANCE_ID}" | grep service_account | cut -f2 -d' '
}
InstallYc() {
curl -s "${S3_ENDPOINT}/yandexcloud-yc/install.sh" | sudo bash -s -- -n -i /usr/local
}
InstallAwsCli() {
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip -o awscliv2.zip >/dev/null
sudo ./aws/install
}
InstallPackages() {
sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils
sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli
}
InstallTools() {
InstallPackages
InstallYc
InstallAwsCli
WaitFile() {
local RETRIES=60
while [[ ${RETRIES} -gt 0 ]]; do
echo "Wait ${1}"
if [ -f "${1}" ]; then
echo "[${1}] has been found"
return 0
fi
RETRIES=$((RETRIES-1))
sleep 5
done
echo "[${1}] not found"
return 1
}
INSTANCE_ID=$(GetInstanceId)
PATHS=$(GetMetadata paths)
S3_ENDPOINT="https://storage.yandexcloud.net"
export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials"
export AWS_REGION=ru-central1
Exit() {
for i in ${PATHS}; do
LOGDEST="${i}.exporter.log"
echo "Uploading exporter log to ${LOGDEST}..."
aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" cp /var/log/syslog "${LOGDEST}"
aws s3 --endpoint-url="${S3_ENDPOINT}" cp /var/log/syslog "${LOGDEST}"
done
echo "Delete static access key..."
if ! yc iam access-key delete "${YC_SK_ID}"; then
echo "Failed to delete static access key."
FAIL=1
fi
if [ $1 -ne 0 ]; then
echo "Set metadata key 'cloud-init-status' to 'cloud-init-error' value"
if ! yc compute instance update "${INSTANCE_ID}" --metadata cloud-init-status=cloud-init-error; then
echo "Failed to update metadata key 'cloud-init-status'."
exit 111
fi
fi
exit $1
}
InstallTools
InstallPackages
echo "####### Export configuration #######"
echo "Instance ID - ${INSTANCE_ID}"
echo "Export paths - ${PATHS}"
echo "####################################"
echo "Detect Service Account ID..."
SERVICE_ACCOUNT_ID=$(GetServiceAccountId)
echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}"
echo "Create static access key..."
SEC_json=$(yc iam access-key create --service-account-id "${SERVICE_ACCOUNT_ID}" \
--description "this key is for export image to storage" --format json)
if [ $? -ne 0 ]; then
echo "Failed to create static access key."
if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then
echo "Failed wait credentials"
Exit 1
fi
echo "Setup env variables to access storage..."
eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json})"
for i in ${PATHS}; do
bucket=$(echo "${i}" | sed 's/\(s3:\/\/[^\/]*\).*/\1/')
echo "Check access to storage: '${bucket}'..."
if ! aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" ls "${bucket}" >/dev/null; then
echo "Failed to access storage: '${bucket}'."
Exit 1
fi
done
echo "Dumping disk..."
if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then
echo "Failed to dump disk to qcow2 image."
@ -104,18 +61,12 @@ fi
for i in ${PATHS}; do
echo "Uploading qcow2 disk image to ${i}..."
if ! aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" cp disk.qcow2 "${i}"; then
if ! aws s3 --endpoint-url="${S3_ENDPOINT}" cp disk.qcow2 "${i}"; then
echo "Failed to upload image to ${i}."
FAIL=1
fi
done
echo "Set metadata key 'cloud-init-status' to 'cloud-init-done' value"
if ! yc compute instance update "${INSTANCE_ID}" --metadata cloud-init-status=cloud-init-done; then
echo "Failed to update metadata key to 'cloud-init-status'."
Exit 1
fi
Exit ${FAIL}
`
)

21
post-processor/yandex-export/post-processor.go
View File

@ -26,7 +26,10 @@ import (
ycsdk "github.com/yandex-cloud/go-sdk"
)
const defaultStorageEndpoint = "storage.yandexcloud.net"
const (
defaultStorageEndpoint = "storage.yandexcloud.net"
defaultStorageRegion = "ru-central1"
)
type Config struct {
common.PackerConfig `mapstructure:",squash"`
@ -106,6 +109,7 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
// to the global Internet: either through ipv4 or ipv6
// TODO: delete this when access appears
if p.config.UseIPv4Nat == false && p.config.UseIPv6 == false {
log.Printf("[DEBUG] Force use IPv4")
p.config.UseIPv4Nat = true
}
p.config.Preemptible = true //? safety
@ -210,6 +214,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa
err := &packersdk.MultiError{Errors: errs}
return nil, false, false, err
}
ui.Say(fmt.Sprintf("Validating service_account_id: '%s'...", yandexConfig.ServiceAccountID))
if err := validateServiceAccount(ctx, driver.SDK(), yandexConfig.ServiceAccountID); err != nil {
return nil, false, false, err
@ -225,6 +230,10 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa
// Build the steps.
steps := []multistep.Step{
&StepCreateS3Keys{
ServiceAccountID: p.config.ServiceAccountID,
Paths: p.config.Paths,
},
&yandex.StepCreateSSHKey{
Debug: p.config.PackerDebug,
DebugKeyPath: fmt.Sprintf("yc_export_pp_%s.pem", p.config.PackerBuildName),
@ -234,10 +243,18 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa
SerialLogFile: yandexConfig.SerialLogFile,
GeneratedData: &packerbuilderdata.GeneratedData{State: state},
},
new(yandex.StepWaitCloudInitScript),
new(yandex.StepInstanceInfo),
&communicator.StepConnect{
Config: &yandexConfig.Communicator,
Host: yandex.CommHost,
SSHConfig: yandexConfig.Communicator.SSHConfigFunc(),
},
new(StepUploadSecrets),
new(StepWaitCloudInitScript),
&yandex.StepTeardownInstance{
SerialLogFile: yandexConfig.SerialLogFile,
},
&commonsteps.StepCleanupTempKeys{Comm: &yandexConfig.Communicator},
}
// Run the steps.

91
post-processor/yandex-export/scripts/export.sh
View File

@ -4,94 +4,51 @@ GetMetadata() {
curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null
}
GetInstanceId() {
curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2>/dev/null
}
GetServiceAccountId() {
yc compute instance get "${INSTANCE_ID}" | grep service_account | cut -f2 -d' '
}
InstallYc() {
curl -s "${S3_ENDPOINT}/yandexcloud-yc/install.sh" | sudo bash -s -- -n -i /usr/local
}
InstallAwsCli() {
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip -o awscliv2.zip >/dev/null
sudo ./aws/install
}
InstallPackages() {
sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils
sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli
}
InstallTools() {
InstallPackages
InstallYc
InstallAwsCli
WaitFile() {
local RETRIES=60
while [[ ${RETRIES} -gt 0 ]]; do
echo "Wait ${1}"
if [ -f "${1}" ]; then
echo "[${1}] has been found"
return 0
fi
RETRIES=$((RETRIES-1))
sleep 5
done
echo "[${1}] not found"
return 1
}
INSTANCE_ID=$(GetInstanceId)
PATHS=$(GetMetadata paths)
S3_ENDPOINT="https://storage.yandexcloud.net"
export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials"
export AWS_REGION=ru-central1
Exit() {
for i in ${PATHS}; do
LOGDEST="${i}.exporter.log"
echo "Uploading exporter log to ${LOGDEST}..."
aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" cp /var/log/syslog "${LOGDEST}"
aws s3 --endpoint-url="${S3_ENDPOINT}" cp /var/log/syslog "${LOGDEST}"
done
echo "Delete static access key..."
if ! yc iam access-key delete "${YC_SK_ID}"; then
echo "Failed to delete static access key."
FAIL=1
fi
if [ $1 -ne 0 ]; then
echo "Set metadata key 'cloud-init-status' to 'cloud-init-error' value"
if ! yc compute instance update "${INSTANCE_ID}" --metadata cloud-init-status=cloud-init-error; then
echo "Failed to update metadata key 'cloud-init-status'."
exit 111
fi
fi
exit $1
}
InstallTools
InstallPackages
echo "####### Export configuration #######"
echo "Instance ID - ${INSTANCE_ID}"
echo "Export paths - ${PATHS}"
echo "####################################"
echo "Detect Service Account ID..."
SERVICE_ACCOUNT_ID=$(GetServiceAccountId)
echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}"
echo "Create static access key..."
SEC_json=$(yc iam access-key create --service-account-id "${SERVICE_ACCOUNT_ID}" \
--description "this key is for export image to storage" --format json)
if [ $? -ne 0 ]; then
echo "Failed to create static access key."
if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then
echo "Failed wait credentials"
Exit 1
fi
echo "Setup env variables to access storage..."
eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json})"
for i in ${PATHS}; do
bucket=$(echo "${i}" | sed 's/\(s3:\/\/[^\/]*\).*/\1/')
echo "Check access to storage: '${bucket}'..."
if ! aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" ls "${bucket}" >/dev/null; then
echo "Failed to access storage: '${bucket}'."
Exit 1
fi
done
echo "Dumping disk..."
if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then
echo "Failed to dump disk to qcow2 image."
@ -100,16 +57,10 @@ fi
for i in ${PATHS}; do
echo "Uploading qcow2 disk image to ${i}..."
if ! aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" cp disk.qcow2 "${i}"; then
if ! aws s3 --endpoint-url="${S3_ENDPOINT}" cp disk.qcow2 "${i}"; then
echo "Failed to upload image to ${i}."
FAIL=1
fi
done
echo "Set metadata key 'cloud-init-status' to 'cloud-init-done' value"
if ! yc compute instance update "${INSTANCE_ID}" --metadata cloud-init-status=cloud-init-done; then
echo "Failed to update metadata key to 'cloud-init-status'."
Exit 1
fi
Exit ${FAIL}

154
post-processor/yandex-export/step-create-s3-keys.go Normal file
View File

@ -0,0 +1,154 @@
package yandexexport
import (
"context"
"fmt"
"log"
"net/url"
"path/filepath"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/packer/builder/yandex"
"github.com/hashicorp/packer/packer-plugin-sdk/multistep"
"github.com/hashicorp/packer/packer-plugin-sdk/packer"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
)
type StepCreateS3Keys struct {
ServiceAccountID string
Paths []string
}
func (c *StepCreateS3Keys) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
driver := state.Get("driver").(yandex.Driver)
ui := state.Get("ui").(packer.Ui)
ui.Say("Create temporary storage Access Key")
// Create temporary storage Access Key
respWithKey, err := driver.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
ServiceAccountId: c.ServiceAccountID,
Description: "this temporary key is for upload image to storage; created by Packer",
})
if err != nil {
err := fmt.Errorf("Error waiting for cloud-init script to finish: %s", err)
return yandex.StepHaltWithError(state, err)
}
state.Put("s3_secret", respWithKey)
ui.Say("Verify access to paths")
if err := verfiyAccess(respWithKey.GetAccessKey().GetKeyId(), respWithKey.Secret, c.Paths); err != nil {
return yandex.StepHaltWithError(state, err)
}
return multistep.ActionContinue
}
func (s *StepCreateS3Keys) Cleanup(state multistep.StateBag) {
driver := state.Get("driver").(yandex.Driver)
ui := state.Get("ui").(packer.Ui)
if val, ok := state.GetOk("s3_secret"); ok {
ui.Say("S3 secrets have been found")
s3Secret := val.(*awscompatibility.CreateAccessKeyResponse)
ui.Message("Cleanup empty objects...")
cleanUpEmptyObjects(s3Secret.GetAccessKey().GetKeyId(), s3Secret.GetSecret(), s.Paths)
ui.Say("Delete S3 secrets...")
_, err := driver.SDK().IAM().AWSCompatibility().AccessKey().Delete(context.Background(), &awscompatibility.DeleteAccessKeyRequest{
AccessKeyId: s3Secret.GetAccessKey().GetId(),
})
if err != nil {
ui.Error(err.Error())
}
}
}
func verfiyAccess(keyID, secret string, paths []string) error {
newSession, err := session.NewSession(&aws.Config{
Endpoint: aws.String(defaultStorageEndpoint),
Region: aws.String(defaultStorageRegion),
Credentials: credentials.NewStaticCredentials(
keyID, secret, "",
),
})
if err != nil {
return err
}
s3Conn := s3.New(newSession)
for _, path := range paths {
u, err := url.Parse(path)
if err != nil {
return err
}
key := u.Path
if strings.HasSuffix(key, "/") {
key = filepath.Join(key, "disk.qcow2")
}
_, err = s3Conn.PutObject(&s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("")),
Bucket: aws.String(u.Host),
Key: aws.String(key),
})
if err != nil {
return err
}
}
return nil
}
func cleanUpEmptyObjects(keyID, secret string, paths []string) {
newSession, err := session.NewSession(&aws.Config{
Endpoint: aws.String(defaultStorageEndpoint),
Region: aws.String(defaultStorageRegion),
Credentials: credentials.NewStaticCredentials(
keyID, secret, "",
),
})
if err != nil {
log.Printf("[WARN] %s", err)
return
}
s3Conn := s3.New(newSession)
for _, path := range paths {
u, err := url.Parse(path)
if err != nil {
log.Printf("[WARN] %s", err)
continue
}
key := u.Path
if strings.HasSuffix(key, "/") {
key = filepath.Join(key, "disk.qcow2")
}
log.Printf("Check object: '%s'", path)
respHead, err := s3Conn.HeadObject(&s3.HeadObjectInput{
Bucket: aws.String(u.Host),
Key: aws.String(key),
})
if err != nil {
log.Printf("[WARN] %s", err)
continue
}
if *respHead.ContentLength > 0 {
continue
}
log.Printf("[DEBUG] Delete object: '%s'", path)
_, err = s3Conn.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(u.Host),
Key: aws.String(key),
})
if err != nil {
log.Printf("[WARN] %s", err)
}
}
}

41
post-processor/yandex-export/step-upload-secrets.go Normal file
View File

@ -0,0 +1,41 @@
package yandexexport
import (
"context"
"fmt"
"strings"
"github.com/hashicorp/packer/builder/yandex"
"github.com/hashicorp/packer/packer-plugin-sdk/multistep"
packersdk "github.com/hashicorp/packer/packer-plugin-sdk/packer"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
)
type StepUploadSecrets struct{}
// Run reads the instance metadata and looks for the log entry
// indicating the cloud-init script finished.
func (s *StepUploadSecrets) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
_ = state.Get("config").(*yandex.Config)
_ = state.Get("driver").(yandex.Driver)
ui := state.Get("ui").(packersdk.Ui)
comm := state.Get("communicator").(packersdk.Communicator)
s3Secret := state.Get("s3_secret").(*awscompatibility.CreateAccessKeyResponse)
ui.Say("Upload secrets..")
creds := fmt.Sprintf(
"[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n",
s3Secret.GetAccessKey().GetKeyId(),
s3Secret.GetSecret())
err := comm.Upload("/tmp/aws-credentials", strings.NewReader(creds), nil)
if err != nil {
return yandex.StepHaltWithError(state, err)
}
ui.Message("Secrets has been uploaded")
return multistep.ActionContinue
}
// Cleanup.
func (s *StepUploadSecrets) Cleanup(state multistep.StateBag) {}

99
post-processor/yandex-export/step-wait-cloud-init.go Normal file
View File

@ -0,0 +1,99 @@
package yandexexport
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/hashicorp/packer/builder/yandex"
"github.com/hashicorp/packer/packer-plugin-sdk/multistep"
packersdk "github.com/hashicorp/packer/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/packer-plugin-sdk/retry"
)
type StepWaitCloudInitScript int
type cloudInitStatus struct {
V1 struct {
Errors []interface{}
}
}
type cloudInitError struct {
Err error
}
func (e *cloudInitError) Error() string {
return e.Err.Error()
}
// Run reads the instance metadata and looks for the log entry
// indicating the cloud-init script finished.
func (s *StepWaitCloudInitScript) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
ui := state.Get("ui").(packersdk.Ui)
comm := state.Get("communicator").(packersdk.Communicator)
ui.Say("Waiting for any running cloud-init script to finish...")
ctxWithCancel, cancelCtx := context.WithCancel(ctx)
defer cancelCtx()
go func() {
cmd := &packersdk.RemoteCmd{
Command: "tail -f /var/log/cloud-init-output.log",
}
err := cmd.RunWithUi(ctxWithCancel, comm, ui)
if err != nil && !errors.Is(err, context.Canceled) {
ui.Error(err.Error())
return
}
ui.Message("Init output closed")
}()
// Keep checking the serial port output to see if the cloud-init script is done.
retryConfig := &retry.Config{
ShouldRetry: func(e error) bool {
switch e.(type) {
case *cloudInitError:
return false
}
return true
},
RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 2}).Linear,
}
err := retryConfig.Run(ctx, func(ctx context.Context) error {
buff := bytes.Buffer{}
err := comm.Download("/var/run/cloud-init/result.json", &buff)
if err != nil {
err := fmt.Errorf("Waiting cloud-init script status: %s", err)
return err
}
result := &cloudInitStatus{}
err = json.Unmarshal(buff.Bytes(), result)
if err != nil {
err := fmt.Errorf("Failed parse result: %s", err)
return &cloudInitError{Err: err}
}
if len(result.V1.Errors) != 0 {
err := fmt.Errorf("Result: %v", result.V1.Errors)
return &cloudInitError{Err: err}
}
return nil
})
if err != nil {
err := fmt.Errorf("Error waiting for cloud-init script to finish: %s", err)
return yandex.StepHaltWithError(state, err)
}
ui.Say("Cloud-init script has finished running.")
return multistep.ActionContinue
}
// Cleanup.
func (s *StepWaitCloudInitScript) Cleanup(state multistep.StateBag) {}