diff --git a/post-processor/googlecompute-export/post-processor.go b/post-processor/googlecompute-export/post-processor.go index eb1fc786f..7631402a3 100644 --- a/post-processor/googlecompute-export/post-processor.go +++ b/post-processor/googlecompute-export/post-processor.go @@ -15,10 +15,19 @@ import ( type Config struct { common.PackerConfig `mapstructure:",squash"` - Paths []string `mapstructure:"paths"` - KeepOriginalImage bool `mapstructure:"keep_input_artifact"` + AccountFile string `mapstructure:"account_file"` - ctx interpolate.Context + DiskSizeGb int64 `mapstructure:"disk_size"` + DiskType string `mapstructure:"disk_type"` + KeepOriginalImage bool `mapstructure:"keep_input_artifact"` + MachineType string `mapstructure:"machine_type"` + Network string `mapstructure:"network"` + Paths []string `mapstructure:"paths"` + Subnetwork string `mapstructure:"subnetwork"` + Zone string `mapstructure:"zone"` + + Account googlecompute.AccountFile + ctx interpolate.Context } type PostProcessor struct { @@ -35,12 +44,38 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { return err } + errs := new(packer.MultiError) + + if len(p.config.Paths) == 0 { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("paths must be specified")) + } + + // Set defaults. + if p.config.DiskSizeGb == 0 { + p.config.DiskSizeGb = 200 + } + + if p.config.DiskType == "" { + p.config.DiskType = "pd-ssd" + } + + if p.config.MachineType == "" { + p.config.MachineType = "n1-highcpu-4" + } + + if p.config.Network == "" && p.config.Subnetwork == "" { + p.config.Network = "default" + } + + if len(errs.Errors) > 0 { + return errs + } + return nil } func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - ui.Say("Starting googlecompute-export...") - ui.Say(fmt.Sprintf("Exporting image to destinations: %v", p.config.Paths)) if artifact.BuilderId() != googlecompute.BuilderId { err := fmt.Errorf( "Unknown artifact type: %s\nCan only export from Google Compute Engine builder artifacts.", @@ -48,79 +83,90 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac return nil, p.config.KeepOriginalImage, err } - result := &Artifact{paths: p.config.Paths} + builderAccountFile := artifact.State("AccountFilePath").(string) + builderImageName := artifact.State("ImageName").(string) + builderProjectId := artifact.State("ProjectId").(string) + builderZone := artifact.State("BuildZone").(string) - if len(p.config.Paths) > 0 { - accountKeyFilePath := artifact.State("AccountFilePath").(string) - imageName := artifact.State("ImageName").(string) - imageSizeGb := artifact.State("ImageSizeGb").(int64) - projectId := artifact.State("ProjectId").(string) - zone := artifact.State("BuildZone").(string) + ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", builderImageName, p.config.Paths)) - // Set up instance configuration. - instanceName := fmt.Sprintf("%s-exporter", artifact.Id()) - metadata := map[string]string{ - "image_name": imageName, - "name": instanceName, - "paths": strings.Join(p.config.Paths, " "), - "startup-script": StartupScript, - "zone": zone, - } - exporterConfig := googlecompute.Config{ - InstanceName: instanceName, - SourceImageProjectId: "debian-cloud", - SourceImage: "debian-8-jessie-v20160629", - DiskName: instanceName, - DiskSizeGb: imageSizeGb + 10, - DiskType: "pd-standard", - Metadata: metadata, - MachineType: "n1-standard-4", - Zone: zone, - Network: "default", - RawStateTimeout: "5m", - Scopes: []string{ - "https://www.googleapis.com/auth/userinfo.email", - "https://www.googleapis.com/auth/compute", - "https://www.googleapis.com/auth/devstorage.full_control", - }, - } - exporterConfig.CalcTimeout() + if p.config.Zone == "" { + p.config.Zone = builderZone + } - // Set up credentials and GCE driver. - if accountKeyFilePath != "" { - err := googlecompute.ProcessAccountFile(&exporterConfig.Account, accountKeyFilePath) - if err != nil { - return nil, p.config.KeepOriginalImage, err - } - } - driver, err := googlecompute.NewDriverGCE(ui, projectId, &exporterConfig.Account) + // Set up credentials for GCE driver. + if builderAccountFile != "" { + err := googlecompute.ProcessAccountFile(&p.config.Account, builderAccountFile) if err != nil { return nil, p.config.KeepOriginalImage, err } - - // Set up the state. - state := new(multistep.BasicStateBag) - state.Put("config", &exporterConfig) - state.Put("driver", driver) - state.Put("ui", ui) - - // Build the steps. - steps := []multistep.Step{ - &googlecompute.StepCreateSSHKey{ - Debug: p.config.PackerDebug, - DebugKeyPath: fmt.Sprintf("gce_%s.pem", p.config.PackerBuildName), - }, - &googlecompute.StepCreateInstance{ - Debug: p.config.PackerDebug, - }, - new(googlecompute.StepWaitStartupScript), - new(googlecompute.StepTeardownInstance), - } - - // Run the steps. - p.runner = common.NewRunner(steps, p.config.PackerConfig, ui) - p.runner.Run(state) } + if p.config.AccountFile != "" { + err := googlecompute.ProcessAccountFile(&p.config.Account, p.config.AccountFile) + if err != nil { + return nil, p.config.KeepOriginalImage, err + } + } + + // Set up exporter instance configuration. + exporterName := fmt.Sprintf("%s-exporter", artifact.Id()) + exporterMetadata := map[string]string{ + "image_name": builderImageName, + "name": exporterName, + "paths": strings.Join(p.config.Paths, " "), + "startup-script": StartupScript, + "zone": p.config.Zone, + } + exporterConfig := googlecompute.Config{ + DiskName: exporterName, + DiskSizeGb: p.config.DiskSizeGb, + DiskType: p.config.DiskType, + InstanceName: exporterName, + MachineType: p.config.MachineType, + Metadata: exporterMetadata, + Network: p.config.Network, + RawStateTimeout: "5m", + SourceImageFamily: "debian-9-worker", + SourceImageProjectId: "compute-image-tools", + Subnetwork: p.config.Subnetwork, + Zone: p.config.Zone, + Scopes: []string{ + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/userinfo.email", + }, + } + exporterConfig.CalcTimeout() + + driver, err := googlecompute.NewDriverGCE(ui, builderProjectId, &p.config.Account) + if err != nil { + return nil, p.config.KeepOriginalImage, err + } + + // Set up the state. + state := new(multistep.BasicStateBag) + state.Put("config", &exporterConfig) + state.Put("driver", driver) + state.Put("ui", ui) + + // Build the steps. + steps := []multistep.Step{ + &googlecompute.StepCreateSSHKey{ + Debug: p.config.PackerDebug, + DebugKeyPath: fmt.Sprintf("gce_%s.pem", p.config.PackerBuildName), + }, + &googlecompute.StepCreateInstance{ + Debug: p.config.PackerDebug, + }, + new(googlecompute.StepWaitStartupScript), + new(googlecompute.StepTeardownInstance), + } + + // Run the steps. + p.runner = common.NewRunner(steps, p.config.PackerConfig, ui) + p.runner.Run(state) + + result := &Artifact{paths: p.config.Paths} return result, p.config.KeepOriginalImage, nil } diff --git a/post-processor/googlecompute-export/startup.go b/post-processor/googlecompute-export/startup.go index abd9f752f..779c97405 100644 --- a/post-processor/googlecompute-export/startup.go +++ b/post-processor/googlecompute-export/startup.go @@ -1,6 +1,6 @@ package googlecomputeexport -var StartupScript string = `#!/bin/sh +var StartupScript string = `#!/bin/bash GetMetadata () { echo "$(curl -f -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/attributes/$1 2> /dev/null)" @@ -8,11 +8,11 @@ GetMetadata () { IMAGENAME=$(GetMetadata image_name) NAME=$(GetMetadata name) DISKNAME=${NAME}-toexport -PATHS=$(GetMetadata paths) +PATHS=($(GetMetadata paths)) ZONE=$(GetMetadata zone) Exit () { - for i in ${PATHS}; do + for i in ${PATHS[@]}; do LOGDEST="${i}.exporter.log" echo "Uploading exporter log to ${LOGDEST}..." gsutil -h "Content-Type:text/plain" cp /var/log/daemon.log ${LOGDEST} @@ -40,17 +40,15 @@ if ! gcloud compute instances attach-disk ${NAME} --disk ${DISKNAME} --device-na Exit 1 fi -echo "Dumping disk..." -if ! dd if=/dev/disk/by-id/google-toexport of=disk.raw bs=4096 conv=sparse; then - echo "Failed to dump disk to image." +echo "GCEExport: Running export tool." +gce_export -gcs_path "${PATHS[0]}" -disk /dev/disk/by-id/google-toexport -y +if [ $? -ne 0 ]; then + echo "ExportFailed: Failed to export disk source to ${PATHS[0]}." Exit 1 fi -echo "Compressing and tar'ing disk image..." -if ! tar -czf root.tar.gz disk.raw; then - echo "Failed to tar disk image." - Exit 1 -fi +echo "ExportSuccess" +sync echo "Detaching disk..." if ! gcloud compute instances detach-disk ${NAME} --disk ${DISKNAME} --zone ${ZONE}; then @@ -64,10 +62,10 @@ if ! gcloud compute disks delete ${DISKNAME} --zone ${ZONE}; then FAIL=1 fi -for i in ${PATHS}; do - echo "Uploading tar'ed disk image to ${i}..." - if ! gsutil -o GSUtil:parallel_composite_upload_threshold=100M cp root.tar.gz ${i}; then - echo "Failed to upload image to ${i}." +for i in ${PATHS[@]:1}; do + echo "Copying archive image to ${i}..." + if ! gsutil -o GSUtil:parallel_composite_upload_threshold=100M cp ${PATHS[0]} ${i}; then + echo "Failed to copy image to ${i}." FAIL=1 fi done diff --git a/post-processor/googlecompute-import/post-processor.go b/post-processor/googlecompute-import/post-processor.go index 85b80a290..4dfb1fee9 100644 --- a/post-processor/googlecompute-import/post-processor.go +++ b/post-processor/googlecompute-import/post-processor.go @@ -10,7 +10,7 @@ import ( "google.golang.org/api/compute/v1" "google.golang.org/api/storage/v1" - googlecompute "github.com/hashicorp/packer/builder/googlecompute" + "github.com/hashicorp/packer/builder/googlecompute" "github.com/hashicorp/packer/common" "github.com/hashicorp/packer/helper/config" "github.com/hashicorp/packer/packer" diff --git a/website/source/docs/post-processors/googlecompute-export.html.md b/website/source/docs/post-processors/googlecompute-export.html.md index d3e13874d..98061cddf 100644 --- a/website/source/docs/post-processors/googlecompute-export.html.md +++ b/website/source/docs/post-processors/googlecompute-export.html.md @@ -34,9 +34,39 @@ permissions to the GCS `paths`. ### Optional +- `account_file` (string) - The JSON file containing your account + credentials. If specified, this take precedence over `googlecompute` + builder authentication method. + +- `disk_size` (number) - The size of the export instances disk, this disk + is unused for the export but a larger size increase `pd-ssd` read speed. + This defaults to `200`, which is 200GB. + +- `disk_type` (string) - Type of disk used to back export instance, like + `pd-ssd` or `pd-standard`. Defaults to `pd-ssd`. + - `keep_input_artifact` (boolean) - If true, do not delete the Google Compute Engine (GCE) image being exported. +- `machine_type` (string) - The export instance machine type. Defaults + to `"n1-highcpu-4"`. + +- `network` (string) - The Google Compute network id or URL to use for the + export instance. Defaults to `"default"`. If the value is not a URL, it + will be interpolated to + `projects/((network_project_id))/global/networks/((network))`. This value + is not required if a `subnet` is specified. + +- `subnetwork` (string) - The Google Compute subnetwork id or URL to use for + the export instance. Only required if the `network` has been created with + custom subnetting. Note, the region of the subnetwork must match the + `zone` in which the VM is launched. If the value is not a URL, + it will be interpolated to + `projects/((network_project_id))/regions/((region))/subnetworks/((subnetwork))` + +- `zone` (string) - The zone in which to launch the export instance. Defaults + to `googlecompute` builder zone. Example: `"us-central1-a"` + ## Basic Example The following example builds a GCE image in the project, `my-project`, with an