diff --git a/builder/yandex/builder.go b/builder/yandex/builder.go index f481b9a77..6934c69d3 100644 --- a/builder/yandex/builder.go +++ b/builder/yandex/builder.go @@ -77,10 +77,10 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook) SerialLogFile: b.config.SerialLogFile, GeneratedData: generatedData, }, - &stepInstanceInfo{}, + &StepInstanceInfo{}, &communicator.StepConnect{ Config: &b.config.Communicator, - Host: commHost, + Host: CommHost, SSHConfig: b.config.Communicator.SSHConfigFunc(), }, &commonsteps.StepProvision{}, diff --git a/builder/yandex/ssh.go b/builder/yandex/ssh.go index ffebeab41..b17232018 100644 --- a/builder/yandex/ssh.go +++ b/builder/yandex/ssh.go @@ -4,7 +4,7 @@ import ( "github.com/hashicorp/packer/packer-plugin-sdk/multistep" ) -func commHost(state multistep.StateBag) (string, error) { +func CommHost(state multistep.StateBag) (string, error) { ipAddress := state.Get("instance_ip").(string) return ipAddress, nil } diff --git a/builder/yandex/step_create_image.go b/builder/yandex/step_create_image.go index a225a522b..234720c3e 100644 --- a/builder/yandex/step_create_image.go +++ b/builder/yandex/step_create_image.go @@ -41,22 +41,22 @@ func (s *stepCreateImage) Run(ctx context.Context, state multistep.StateBag) mul }, })) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error creating image: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error creating image: %s", err)) } ui.Say("Waiting for image to complete...") if err := op.Wait(ctx); err != nil { - return stepHaltWithError(state, fmt.Errorf("Error waiting for image: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error waiting for image: %s", err)) } resp, err := op.Response() if err != nil { - return stepHaltWithError(state, err) + return StepHaltWithError(state, err) } image, ok := resp.(*compute.Image) if !ok { - return stepHaltWithError(state, errors.New("API call response doesn't contain Compute Image")) + return StepHaltWithError(state, errors.New("API call response doesn't contain Compute Image")) } log.Printf("Image ID: %s", image.Id) diff --git a/builder/yandex/step_create_instance.go b/builder/yandex/step_create_instance.go index b61abba06..5217bb9e5 100644 --- a/builder/yandex/step_create_instance.go +++ b/builder/yandex/step_create_instance.go @@ -165,11 +165,11 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) sourceImage, err := getImage(ctx, config, driver) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error getting source image for instance creation: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error getting source image for instance creation: %s", err)) } if sourceImage.MinDiskSizeGb > config.DiskSizeGb { - return stepHaltWithError(state, fmt.Errorf("Instance DiskSizeGb (%d) should be equal or greater "+ + return StepHaltWithError(state, fmt.Errorf("Instance DiskSizeGb (%d) should be equal or greater "+ "than SourceImage disk requirement (%d)", config.DiskSizeGb, sourceImage.MinDiskSizeGb)) } @@ -182,14 +182,14 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) ui.Say("Creating network...") network, err := createNetwork(ctx, config, driver) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error creating network: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error creating network: %s", err)) } state.Put("network_id", network.Id) ui.Say(fmt.Sprintf("Creating subnet in zone %q...", config.Zone)) subnet, err := createSubnet(ctx, config, driver, network.Id) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error creating subnet: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error creating subnet: %s", err)) } instanceSubnetID = subnet.Id // save for cleanup @@ -203,7 +203,7 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) ui.Say("Creating disk...") disk, err := createDisk(ctx, state, config, driver, sourceImage) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error creating disk: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error creating disk: %s", err)) } // Create an instance based on the configuration @@ -211,7 +211,7 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) instanceMetadata, err := config.createInstanceMetadata(string(config.Communicator.SSHPublicKey)) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error preparing instance metadata: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error preparing instance metadata: %s", err)) } if config.UseIPv6 { @@ -223,7 +223,7 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) } instanceMetadata["user-data"], err = MergeCloudUserMetaData(oldUserData, cloudInitIPv6Config) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error merge user data configs: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error merge user data configs: %s", err)) } } @@ -281,33 +281,33 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) op, err := sdk.WrapOperation(sdk.Compute().Instance().Create(ctx, req)) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error create instance: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error create instance: %s", err)) } opMetadata, err := op.Metadata() if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error get create operation metadata: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error get create operation metadata: %s", err)) } if cimd, ok := opMetadata.(*compute.CreateInstanceMetadata); ok { state.Put("instance_id", cimd.InstanceId) } else { - return stepHaltWithError(state, fmt.Errorf("could not get Instance ID from operation metadata")) + return StepHaltWithError(state, fmt.Errorf("could not get Instance ID from operation metadata")) } err = op.Wait(ctx) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error create instance: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error create instance: %s", err)) } resp, err := op.Response() if err != nil { - return stepHaltWithError(state, err) + return StepHaltWithError(state, err) } instance, ok := resp.(*compute.Instance) if !ok { - return stepHaltWithError(state, fmt.Errorf("response doesn't contain Instance")) + return StepHaltWithError(state, fmt.Errorf("response doesn't contain Instance")) } // instance_id is the generic term used so that users can have access to the diff --git a/builder/yandex/step_create_ssh_key.go b/builder/yandex/step_create_ssh_key.go index 193287463..95cf810da 100644 --- a/builder/yandex/step_create_ssh_key.go +++ b/builder/yandex/step_create_ssh_key.go @@ -50,7 +50,7 @@ func (s *StepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) mult priv, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error generating temporary SSH key: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error generating temporary SSH key: %s", err)) } // ASN.1 DER encoded form @@ -89,7 +89,7 @@ func (s *StepCreateSSHKey) Run(_ context.Context, state multistep.StateBag) mult ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath)) err := ioutil.WriteFile(s.DebugKeyPath, config.Communicator.SSHPrivateKey, 0600) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error saving debug key: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error saving debug key: %s", err)) } } diff --git a/builder/yandex/step_instance_info.go b/builder/yandex/step_instance_info.go index e93758d07..27191054a 100644 --- a/builder/yandex/step_instance_info.go +++ b/builder/yandex/step_instance_info.go @@ -12,9 +12,9 @@ import ( ycsdk "github.com/yandex-cloud/go-sdk" ) -type stepInstanceInfo struct{} +type StepInstanceInfo struct{} -func (s *stepInstanceInfo) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { +func (s *StepInstanceInfo) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { sdk := state.Get("sdk").(*ycsdk.SDK) ui := state.Get("ui").(packersdk.Ui) c := state.Get("config").(*Config) @@ -30,12 +30,12 @@ func (s *stepInstanceInfo) Run(ctx context.Context, state multistep.StateBag) mu View: compute.InstanceView_FULL, }) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error retrieving instance data: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error retrieving instance data: %s", err)) } instanceIP, err := getInstanceIPAddress(c, instance) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Failed to find instance ip address: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Failed to find instance ip address: %s", err)) } state.Put("instance_ip", instanceIP) @@ -106,6 +106,6 @@ func instanceAddresses(instance *compute.Instance) (ipV4Int, ipV4Ext, ipV6 strin return } -func (s *stepInstanceInfo) Cleanup(state multistep.StateBag) { +func (s *StepInstanceInfo) Cleanup(state multistep.StateBag) { // no cleanup } diff --git a/builder/yandex/step_teardown_instance.go b/builder/yandex/step_teardown_instance.go index 2845fba85..fc5189d02 100644 --- a/builder/yandex/step_teardown_instance.go +++ b/builder/yandex/step_teardown_instance.go @@ -37,11 +37,11 @@ func (s *StepTeardownInstance) Run(ctx context.Context, state multistep.StateBag InstanceId: instanceID, })) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error stopping instance: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error stopping instance: %s", err)) } err = op.Wait(ctx) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error stopping instance: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error stopping instance: %s", err)) } ui.Say("Deleting instance...") @@ -49,11 +49,11 @@ func (s *StepTeardownInstance) Run(ctx context.Context, state multistep.StateBag InstanceId: instanceID, })) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error deleting instance: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error deleting instance: %s", err)) } err = op.Wait(ctx) if err != nil { - return stepHaltWithError(state, fmt.Errorf("Error deleting instance: %s", err)) + return StepHaltWithError(state, fmt.Errorf("Error deleting instance: %s", err)) } ui.Message("Instance has been deleted!") diff --git a/builder/yandex/step_wait_cloudinit_script.go b/builder/yandex/step_wait_cloudinit_script.go deleted file mode 100644 index 75c4faa37..000000000 --- a/builder/yandex/step_wait_cloudinit_script.go +++ /dev/null @@ -1,69 +0,0 @@ -package yandex - -import ( - "context" - "errors" - "fmt" - "time" - - "github.com/hashicorp/packer/packer-plugin-sdk/multistep" - packersdk "github.com/hashicorp/packer/packer-plugin-sdk/packer" - "github.com/hashicorp/packer/packer-plugin-sdk/retry" -) - -const CloudInitScriptStatusKey = "cloud-init-status" -const StartupScriptStatusError = "cloud-init-error" -const StartupScriptStatusDone = "cloud-init-done" - -type StepWaitCloudInitScript int - -// Run reads the instance metadata and looks for the log entry -// indicating the cloud-init script finished. -func (*StepWaitCloudInitScript) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { - _ = state.Get("config").(*Config) - driver := state.Get("driver").(Driver) - ui := state.Get("ui").(packersdk.Ui) - instanceID := state.Get("instance_id").(string) - - ui.Say("Waiting for any running cloud-init script to finish...") - - // Keep checking the serial port output to see if the cloud-init script is done. - err := retry.Config{ - ShouldRetry: func(error) bool { - return true - }, - RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 2}).Linear, - }.Run(ctx, func(ctx context.Context) error { - status, err := driver.GetInstanceMetadata(ctx, instanceID, CloudInitScriptStatusKey) - - if err != nil { - err := fmt.Errorf("Error getting cloud-init script status: %s", err) - return err - } - - if status == StartupScriptStatusError { - err = errors.New("Cloud-init script error.") - return err - } - - done := status == StartupScriptStatusDone - if !done { - ui.Say("Cloud-init script not finished yet. Waiting...") - return errors.New("Cloud-init script not done.") - } - - return nil - }) - - if err != nil { - err := fmt.Errorf("Error waiting for cloud-init script to finish: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - ui.Say("Cloud-init script has finished running.") - return multistep.ActionContinue -} - -// Cleanup. -func (s *StepWaitCloudInitScript) Cleanup(state multistep.StateBag) {} diff --git a/builder/yandex/util.go b/builder/yandex/util.go index fbe74ec3e..d81873f0d 100644 --- a/builder/yandex/util.go +++ b/builder/yandex/util.go @@ -12,7 +12,7 @@ import ( ycsdk "github.com/yandex-cloud/go-sdk" ) -func stepHaltWithError(state multistep.StateBag, err error) multistep.StepAction { +func StepHaltWithError(state multistep.StateBag, err error) multistep.StepAction { ui := state.Get("ui").(packersdk.Ui) state.Put("error", err) ui.Error(err.Error()) diff --git a/post-processor/yandex-export/cloud-init-script.go b/post-processor/yandex-export/cloud-init-script.go index 84a17e494..a6bb81068 100644 --- a/post-processor/yandex-export/cloud-init-script.go +++ b/post-processor/yandex-export/cloud-init-script.go @@ -8,94 +8,51 @@ GetMetadata() { curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null } -GetInstanceId() { - curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2>/dev/null -} - -GetServiceAccountId() { - yc compute instance get "${INSTANCE_ID}" | grep service_account | cut -f2 -d' ' -} - -InstallYc() { - curl -s "${S3_ENDPOINT}/yandexcloud-yc/install.sh" | sudo bash -s -- -n -i /usr/local -} - -InstallAwsCli() { - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -o awscliv2.zip >/dev/null - sudo ./aws/install -} - InstallPackages() { - sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils + sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli } -InstallTools() { - InstallPackages - InstallYc - InstallAwsCli +WaitFile() { + local RETRIES=60 + while [[ ${RETRIES} -gt 0 ]]; do + echo "Wait ${1}" + if [ -f "${1}" ]; then + echo "[${1}] has been found" + return 0 + fi + RETRIES=$((RETRIES-1)) + sleep 5 + done + echo "[${1}] not found" + return 1 } -INSTANCE_ID=$(GetInstanceId) PATHS=$(GetMetadata paths) S3_ENDPOINT="https://storage.yandexcloud.net" +export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials" +export AWS_REGION=ru-central1 Exit() { for i in ${PATHS}; do LOGDEST="${i}.exporter.log" echo "Uploading exporter log to ${LOGDEST}..." - aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" cp /var/log/syslog "${LOGDEST}" + aws s3 --endpoint-url="${S3_ENDPOINT}" cp /var/log/syslog "${LOGDEST}" done - echo "Delete static access key..." - if ! yc iam access-key delete "${YC_SK_ID}"; then - echo "Failed to delete static access key." - FAIL=1 - fi - - if [ $1 -ne 0 ]; then - echo "Set metadata key 'cloud-init-status' to 'cloud-init-error' value" - if ! yc compute instance update "${INSTANCE_ID}" --metadata cloud-init-status=cloud-init-error; then - echo "Failed to update metadata key 'cloud-init-status'." - exit 111 - fi - fi - exit $1 } -InstallTools +InstallPackages echo "####### Export configuration #######" -echo "Instance ID - ${INSTANCE_ID}" echo "Export paths - ${PATHS}" echo "####################################" -echo "Detect Service Account ID..." -SERVICE_ACCOUNT_ID=$(GetServiceAccountId) -echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}" - -echo "Create static access key..." -SEC_json=$(yc iam access-key create --service-account-id "${SERVICE_ACCOUNT_ID}" \ - --description "this key is for export image to storage" --format json) - -if [ $? -ne 0 ]; then - echo "Failed to create static access key." +if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then + echo "Failed wait credentials" Exit 1 fi -echo "Setup env variables to access storage..." -eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json})" - -for i in ${PATHS}; do - bucket=$(echo "${i}" | sed 's/\(s3:\/\/[^\/]*\).*/\1/') - echo "Check access to storage: '${bucket}'..." - if ! aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" ls "${bucket}" >/dev/null; then - echo "Failed to access storage: '${bucket}'." - Exit 1 - fi -done - echo "Dumping disk..." if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then echo "Failed to dump disk to qcow2 image." @@ -104,18 +61,12 @@ fi for i in ${PATHS}; do echo "Uploading qcow2 disk image to ${i}..." - if ! aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" cp disk.qcow2 "${i}"; then + if ! aws s3 --endpoint-url="${S3_ENDPOINT}" cp disk.qcow2 "${i}"; then echo "Failed to upload image to ${i}." FAIL=1 fi done -echo "Set metadata key 'cloud-init-status' to 'cloud-init-done' value" -if ! yc compute instance update "${INSTANCE_ID}" --metadata cloud-init-status=cloud-init-done; then - echo "Failed to update metadata key to 'cloud-init-status'." - Exit 1 -fi - Exit ${FAIL} ` ) diff --git a/post-processor/yandex-export/post-processor.go b/post-processor/yandex-export/post-processor.go index 352158f86..6d76adb53 100644 --- a/post-processor/yandex-export/post-processor.go +++ b/post-processor/yandex-export/post-processor.go @@ -26,7 +26,10 @@ import ( ycsdk "github.com/yandex-cloud/go-sdk" ) -const defaultStorageEndpoint = "storage.yandexcloud.net" +const ( + defaultStorageEndpoint = "storage.yandexcloud.net" + defaultStorageRegion = "ru-central1" +) type Config struct { common.PackerConfig `mapstructure:",squash"` @@ -106,6 +109,7 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { // to the global Internet: either through ipv4 or ipv6 // TODO: delete this when access appears if p.config.UseIPv4Nat == false && p.config.UseIPv6 == false { + log.Printf("[DEBUG] Force use IPv4") p.config.UseIPv4Nat = true } p.config.Preemptible = true //? safety @@ -210,6 +214,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa err := &packersdk.MultiError{Errors: errs} return nil, false, false, err } + ui.Say(fmt.Sprintf("Validating service_account_id: '%s'...", yandexConfig.ServiceAccountID)) if err := validateServiceAccount(ctx, driver.SDK(), yandexConfig.ServiceAccountID); err != nil { return nil, false, false, err @@ -225,6 +230,10 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa // Build the steps. steps := []multistep.Step{ + &StepCreateS3Keys{ + ServiceAccountID: p.config.ServiceAccountID, + Paths: p.config.Paths, + }, &yandex.StepCreateSSHKey{ Debug: p.config.PackerDebug, DebugKeyPath: fmt.Sprintf("yc_export_pp_%s.pem", p.config.PackerBuildName), @@ -234,10 +243,18 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa SerialLogFile: yandexConfig.SerialLogFile, GeneratedData: &packerbuilderdata.GeneratedData{State: state}, }, - new(yandex.StepWaitCloudInitScript), + new(yandex.StepInstanceInfo), + &communicator.StepConnect{ + Config: &yandexConfig.Communicator, + Host: yandex.CommHost, + SSHConfig: yandexConfig.Communicator.SSHConfigFunc(), + }, + new(StepUploadSecrets), + new(StepWaitCloudInitScript), &yandex.StepTeardownInstance{ SerialLogFile: yandexConfig.SerialLogFile, }, + &commonsteps.StepCleanupTempKeys{Comm: &yandexConfig.Communicator}, } // Run the steps. diff --git a/post-processor/yandex-export/scripts/export.sh b/post-processor/yandex-export/scripts/export.sh index 8194c686f..de23d6a08 100644 --- a/post-processor/yandex-export/scripts/export.sh +++ b/post-processor/yandex-export/scripts/export.sh @@ -4,94 +4,51 @@ GetMetadata() { curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2>/dev/null } -GetInstanceId() { - curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2>/dev/null -} - -GetServiceAccountId() { - yc compute instance get "${INSTANCE_ID}" | grep service_account | cut -f2 -d' ' -} - -InstallYc() { - curl -s "${S3_ENDPOINT}/yandexcloud-yc/install.sh" | sudo bash -s -- -n -i /usr/local -} - -InstallAwsCli() { - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -o awscliv2.zip >/dev/null - sudo ./aws/install -} - InstallPackages() { - sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils + sudo apt-get update -qq && sudo apt-get install -y qemu-utils awscli } -InstallTools() { - InstallPackages - InstallYc - InstallAwsCli +WaitFile() { + local RETRIES=60 + while [[ ${RETRIES} -gt 0 ]]; do + echo "Wait ${1}" + if [ -f "${1}" ]; then + echo "[${1}] has been found" + return 0 + fi + RETRIES=$((RETRIES-1)) + sleep 5 + done + echo "[${1}] not found" + return 1 } -INSTANCE_ID=$(GetInstanceId) PATHS=$(GetMetadata paths) S3_ENDPOINT="https://storage.yandexcloud.net" +export AWS_SHARED_CREDENTIALS_FILE="/tmp/aws-credentials" +export AWS_REGION=ru-central1 Exit() { for i in ${PATHS}; do LOGDEST="${i}.exporter.log" echo "Uploading exporter log to ${LOGDEST}..." - aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" cp /var/log/syslog "${LOGDEST}" + aws s3 --endpoint-url="${S3_ENDPOINT}" cp /var/log/syslog "${LOGDEST}" done - echo "Delete static access key..." - if ! yc iam access-key delete "${YC_SK_ID}"; then - echo "Failed to delete static access key." - FAIL=1 - fi - - if [ $1 -ne 0 ]; then - echo "Set metadata key 'cloud-init-status' to 'cloud-init-error' value" - if ! yc compute instance update "${INSTANCE_ID}" --metadata cloud-init-status=cloud-init-error; then - echo "Failed to update metadata key 'cloud-init-status'." - exit 111 - fi - fi - exit $1 } -InstallTools +InstallPackages echo "####### Export configuration #######" -echo "Instance ID - ${INSTANCE_ID}" echo "Export paths - ${PATHS}" echo "####################################" -echo "Detect Service Account ID..." -SERVICE_ACCOUNT_ID=$(GetServiceAccountId) -echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}" - -echo "Create static access key..." -SEC_json=$(yc iam access-key create --service-account-id "${SERVICE_ACCOUNT_ID}" \ - --description "this key is for export image to storage" --format json) - -if [ $? -ne 0 ]; then - echo "Failed to create static access key." +if ! WaitFile "${AWS_SHARED_CREDENTIALS_FILE}"; then + echo "Failed wait credentials" Exit 1 fi -echo "Setup env variables to access storage..." -eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json})" - -for i in ${PATHS}; do - bucket=$(echo "${i}" | sed 's/\(s3:\/\/[^\/]*\).*/\1/') - echo "Check access to storage: '${bucket}'..." - if ! aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" ls "${bucket}" >/dev/null; then - echo "Failed to access storage: '${bucket}'." - Exit 1 - fi -done - echo "Dumping disk..." if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2; then echo "Failed to dump disk to qcow2 image." @@ -100,16 +57,10 @@ fi for i in ${PATHS}; do echo "Uploading qcow2 disk image to ${i}..." - if ! aws s3 --region ru-central1 --endpoint-url="${S3_ENDPOINT}" cp disk.qcow2 "${i}"; then + if ! aws s3 --endpoint-url="${S3_ENDPOINT}" cp disk.qcow2 "${i}"; then echo "Failed to upload image to ${i}." FAIL=1 fi done -echo "Set metadata key 'cloud-init-status' to 'cloud-init-done' value" -if ! yc compute instance update "${INSTANCE_ID}" --metadata cloud-init-status=cloud-init-done; then - echo "Failed to update metadata key to 'cloud-init-status'." - Exit 1 -fi - Exit ${FAIL} diff --git a/post-processor/yandex-export/step-create-s3-keys.go b/post-processor/yandex-export/step-create-s3-keys.go new file mode 100644 index 000000000..640f448ef --- /dev/null +++ b/post-processor/yandex-export/step-create-s3-keys.go @@ -0,0 +1,154 @@ +package yandexexport + +import ( + "context" + "fmt" + "log" + "net/url" + "path/filepath" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/hashicorp/packer/builder/yandex" + "github.com/hashicorp/packer/packer-plugin-sdk/multistep" + "github.com/hashicorp/packer/packer-plugin-sdk/packer" + "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility" +) + +type StepCreateS3Keys struct { + ServiceAccountID string + Paths []string +} + +func (c *StepCreateS3Keys) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { + driver := state.Get("driver").(yandex.Driver) + ui := state.Get("ui").(packer.Ui) + + ui.Say("Create temporary storage Access Key") + // Create temporary storage Access Key + respWithKey, err := driver.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{ + ServiceAccountId: c.ServiceAccountID, + Description: "this temporary key is for upload image to storage; created by Packer", + }) + if err != nil { + err := fmt.Errorf("Error waiting for cloud-init script to finish: %s", err) + return yandex.StepHaltWithError(state, err) + } + state.Put("s3_secret", respWithKey) + + ui.Say("Verify access to paths") + if err := verfiyAccess(respWithKey.GetAccessKey().GetKeyId(), respWithKey.Secret, c.Paths); err != nil { + return yandex.StepHaltWithError(state, err) + } + return multistep.ActionContinue +} + +func (s *StepCreateS3Keys) Cleanup(state multistep.StateBag) { + driver := state.Get("driver").(yandex.Driver) + ui := state.Get("ui").(packer.Ui) + + if val, ok := state.GetOk("s3_secret"); ok { + ui.Say("S3 secrets have been found") + s3Secret := val.(*awscompatibility.CreateAccessKeyResponse) + + ui.Message("Cleanup empty objects...") + cleanUpEmptyObjects(s3Secret.GetAccessKey().GetKeyId(), s3Secret.GetSecret(), s.Paths) + + ui.Say("Delete S3 secrets...") + _, err := driver.SDK().IAM().AWSCompatibility().AccessKey().Delete(context.Background(), &awscompatibility.DeleteAccessKeyRequest{ + AccessKeyId: s3Secret.GetAccessKey().GetId(), + }) + + if err != nil { + ui.Error(err.Error()) + } + } +} + +func verfiyAccess(keyID, secret string, paths []string) error { + newSession, err := session.NewSession(&aws.Config{ + Endpoint: aws.String(defaultStorageEndpoint), + Region: aws.String(defaultStorageRegion), + Credentials: credentials.NewStaticCredentials( + keyID, secret, "", + ), + }) + if err != nil { + return err + } + + s3Conn := s3.New(newSession) + + for _, path := range paths { + u, err := url.Parse(path) + if err != nil { + return err + } + key := u.Path + if strings.HasSuffix(key, "/") { + key = filepath.Join(key, "disk.qcow2") + } + _, err = s3Conn.PutObject(&s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("")), + Bucket: aws.String(u.Host), + Key: aws.String(key), + }) + if err != nil { + return err + } + } + + return nil +} + +func cleanUpEmptyObjects(keyID, secret string, paths []string) { + newSession, err := session.NewSession(&aws.Config{ + Endpoint: aws.String(defaultStorageEndpoint), + Region: aws.String(defaultStorageRegion), + Credentials: credentials.NewStaticCredentials( + keyID, secret, "", + ), + }) + if err != nil { + log.Printf("[WARN] %s", err) + return + } + s3Conn := s3.New(newSession) + + for _, path := range paths { + u, err := url.Parse(path) + if err != nil { + log.Printf("[WARN] %s", err) + continue + } + key := u.Path + if strings.HasSuffix(key, "/") { + key = filepath.Join(key, "disk.qcow2") + } + + log.Printf("Check object: '%s'", path) + respHead, err := s3Conn.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(u.Host), + Key: aws.String(key), + }) + if err != nil { + log.Printf("[WARN] %s", err) + continue + } + if *respHead.ContentLength > 0 { + continue + } + + log.Printf("[DEBUG] Delete object: '%s'", path) + _, err = s3Conn.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(u.Host), + Key: aws.String(key), + }) + if err != nil { + log.Printf("[WARN] %s", err) + } + } +} diff --git a/post-processor/yandex-export/step-upload-secrets.go b/post-processor/yandex-export/step-upload-secrets.go new file mode 100644 index 000000000..0ec9792d5 --- /dev/null +++ b/post-processor/yandex-export/step-upload-secrets.go @@ -0,0 +1,41 @@ +package yandexexport + +import ( + "context" + "fmt" + "strings" + + "github.com/hashicorp/packer/builder/yandex" + "github.com/hashicorp/packer/packer-plugin-sdk/multistep" + packersdk "github.com/hashicorp/packer/packer-plugin-sdk/packer" + "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility" +) + +type StepUploadSecrets struct{} + +// Run reads the instance metadata and looks for the log entry +// indicating the cloud-init script finished. +func (s *StepUploadSecrets) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { + _ = state.Get("config").(*yandex.Config) + _ = state.Get("driver").(yandex.Driver) + ui := state.Get("ui").(packersdk.Ui) + comm := state.Get("communicator").(packersdk.Communicator) + s3Secret := state.Get("s3_secret").(*awscompatibility.CreateAccessKeyResponse) + + ui.Say("Upload secrets..") + creds := fmt.Sprintf( + "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n", + s3Secret.GetAccessKey().GetKeyId(), + s3Secret.GetSecret()) + + err := comm.Upload("/tmp/aws-credentials", strings.NewReader(creds), nil) + if err != nil { + return yandex.StepHaltWithError(state, err) + } + ui.Message("Secrets has been uploaded") + + return multistep.ActionContinue +} + +// Cleanup. +func (s *StepUploadSecrets) Cleanup(state multistep.StateBag) {} diff --git a/post-processor/yandex-export/step-wait-cloud-init.go b/post-processor/yandex-export/step-wait-cloud-init.go new file mode 100644 index 000000000..2ec50ab6a --- /dev/null +++ b/post-processor/yandex-export/step-wait-cloud-init.go @@ -0,0 +1,99 @@ +package yandexexport + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/hashicorp/packer/builder/yandex" + "github.com/hashicorp/packer/packer-plugin-sdk/multistep" + packersdk "github.com/hashicorp/packer/packer-plugin-sdk/packer" + "github.com/hashicorp/packer/packer-plugin-sdk/retry" +) + +type StepWaitCloudInitScript int + +type cloudInitStatus struct { + V1 struct { + Errors []interface{} + } +} + +type cloudInitError struct { + Err error +} + +func (e *cloudInitError) Error() string { + return e.Err.Error() +} + +// Run reads the instance metadata and looks for the log entry +// indicating the cloud-init script finished. +func (s *StepWaitCloudInitScript) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { + ui := state.Get("ui").(packersdk.Ui) + comm := state.Get("communicator").(packersdk.Communicator) + + ui.Say("Waiting for any running cloud-init script to finish...") + + ctxWithCancel, cancelCtx := context.WithCancel(ctx) + + defer cancelCtx() + + go func() { + cmd := &packersdk.RemoteCmd{ + Command: "tail -f /var/log/cloud-init-output.log", + } + + err := cmd.RunWithUi(ctxWithCancel, comm, ui) + if err != nil && !errors.Is(err, context.Canceled) { + ui.Error(err.Error()) + return + } + ui.Message("Init output closed") + }() + + // Keep checking the serial port output to see if the cloud-init script is done. + retryConfig := &retry.Config{ + ShouldRetry: func(e error) bool { + switch e.(type) { + case *cloudInitError: + return false + } + return true + }, + RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 2}).Linear, + } + + err := retryConfig.Run(ctx, func(ctx context.Context) error { + buff := bytes.Buffer{} + err := comm.Download("/var/run/cloud-init/result.json", &buff) + if err != nil { + err := fmt.Errorf("Waiting cloud-init script status: %s", err) + return err + } + result := &cloudInitStatus{} + err = json.Unmarshal(buff.Bytes(), result) + if err != nil { + err := fmt.Errorf("Failed parse result: %s", err) + return &cloudInitError{Err: err} + } + if len(result.V1.Errors) != 0 { + err := fmt.Errorf("Result: %v", result.V1.Errors) + return &cloudInitError{Err: err} + } + return nil + }) + + if err != nil { + err := fmt.Errorf("Error waiting for cloud-init script to finish: %s", err) + return yandex.StepHaltWithError(state, err) + } + ui.Say("Cloud-init script has finished running.") + return multistep.ActionContinue +} + +// Cleanup. +func (s *StepWaitCloudInitScript) Cleanup(state multistep.StateBag) {}