Merge pull request #7222 from Adezandee/gce-post-processors

googlecompute-postprocessors: service account and export configs

builder/googlecompute/driver_gce.go

@@ -34,7 +34,7 @@ type driverGCE struct {
 
 var DriverScopes = []string{"https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/devstorage.full_control"}
 
-func NewDriverGCE(ui packer.Ui, p string, a *AccountFile) (Driver, error) {
+func NewClientGCE(a *AccountFile) (*http.Client, error) {
 	var err error
 
 	var client *http.Client
@@ -78,6 +78,15 @@ func NewDriverGCE(ui packer.Ui, p string, a *AccountFile) (Driver, error) {
 		return nil, err
 	}
 
+	return client, nil
+}
+
+func NewDriverGCE(ui packer.Ui, p string, a *AccountFile) (Driver, error) {
+	client, err := NewClientGCE(a)
+	if err != nil {
+		return nil, err
+	}
+
 	log.Printf("[INFO] Instantiating GCE client...")
 	service, err := compute.New(client)
 	if err != nil {

post-processor/googlecompute-export/post-processor.go

@@ -15,10 +15,19 @@ import (
 type Config struct {
 	common.PackerConfig `mapstructure:",squash"`
 
-	Paths             []string `mapstructure:"paths"`
-	KeepOriginalImage bool     `mapstructure:"keep_input_artifact"`
+	AccountFile string `mapstructure:"account_file"`
 
-	ctx interpolate.Context
+	DiskSizeGb        int64    `mapstructure:"disk_size"`
+	DiskType          string   `mapstructure:"disk_type"`
+	KeepOriginalImage bool     `mapstructure:"keep_input_artifact"`
+	MachineType       string   `mapstructure:"machine_type"`
+	Network           string   `mapstructure:"network"`
+	Paths             []string `mapstructure:"paths"`
+	Subnetwork        string   `mapstructure:"subnetwork"`
+	Zone              string   `mapstructure:"zone"`
+
+	Account googlecompute.AccountFile
+	ctx     interpolate.Context
 }
 
 type PostProcessor struct {
@@ -35,12 +44,38 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 		return err
 	}
 
+	errs := new(packer.MultiError)
+
+	if len(p.config.Paths) == 0 {
+		errs = packer.MultiErrorAppend(
+			errs, fmt.Errorf("paths must be specified"))
+	}
+
+	// Set defaults.
+	if p.config.DiskSizeGb == 0 {
+		p.config.DiskSizeGb = 200
+	}
+
+	if p.config.DiskType == "" {
+		p.config.DiskType = "pd-ssd"
+	}
+
+	if p.config.MachineType == "" {
+		p.config.MachineType = "n1-highcpu-4"
+	}
+
+	if p.config.Network == "" && p.config.Subnetwork == "" {
+		p.config.Network = "default"
+	}
+
+	if len(errs.Errors) > 0 {
+		return errs
+	}
+
 	return nil
 }
 
 func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
-	ui.Say("Starting googlecompute-export...")
-	ui.Say(fmt.Sprintf("Exporting image to destinations: %v", p.config.Paths))
 	if artifact.BuilderId() != googlecompute.BuilderId {
 		err := fmt.Errorf(
 			"Unknown artifact type: %s\nCan only export from Google Compute Engine builder artifacts.",
@@ -48,79 +83,90 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac
 		return nil, p.config.KeepOriginalImage, err
 	}
 
-	result := &Artifact{paths: p.config.Paths}
+	builderAccountFile := artifact.State("AccountFilePath").(string)
+	builderImageName := artifact.State("ImageName").(string)
+	builderProjectId := artifact.State("ProjectId").(string)
+	builderZone := artifact.State("BuildZone").(string)
 
-	if len(p.config.Paths) > 0 {
-		accountKeyFilePath := artifact.State("AccountFilePath").(string)
-		imageName := artifact.State("ImageName").(string)
-		imageSizeGb := artifact.State("ImageSizeGb").(int64)
-		projectId := artifact.State("ProjectId").(string)
-		zone := artifact.State("BuildZone").(string)
+	ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", builderImageName, p.config.Paths))
 
-		// Set up instance configuration.
-		instanceName := fmt.Sprintf("%s-exporter", artifact.Id())
-		metadata := map[string]string{
-			"image_name":     imageName,
-			"name":           instanceName,
-			"paths":          strings.Join(p.config.Paths, " "),
-			"startup-script": StartupScript,
-			"zone":           zone,
-		}
-		exporterConfig := googlecompute.Config{
-			InstanceName:         instanceName,
-			SourceImageProjectId: "debian-cloud",
-			SourceImage:          "debian-8-jessie-v20160629",
-			DiskName:             instanceName,
-			DiskSizeGb:           imageSizeGb + 10,
-			DiskType:             "pd-standard",
-			Metadata:             metadata,
-			MachineType:          "n1-standard-4",
-			Zone:                 zone,
-			Network:              "default",
-			RawStateTimeout:      "5m",
-			Scopes: []string{
-				"https://www.googleapis.com/auth/userinfo.email",
-				"https://www.googleapis.com/auth/compute",
-				"https://www.googleapis.com/auth/devstorage.full_control",
-			},
-		}
-		exporterConfig.CalcTimeout()
+	if p.config.Zone == "" {
+		p.config.Zone = builderZone
+	}
 
-		// Set up credentials and GCE driver.
-		if accountKeyFilePath != "" {
-			err := googlecompute.ProcessAccountFile(&exporterConfig.Account, accountKeyFilePath)
-			if err != nil {
-				return nil, p.config.KeepOriginalImage, err
-			}
-		}
-		driver, err := googlecompute.NewDriverGCE(ui, projectId, &exporterConfig.Account)
+	// Set up credentials for GCE driver.
+	if builderAccountFile != "" {
+		err := googlecompute.ProcessAccountFile(&p.config.Account, builderAccountFile)
 		if err != nil {
 			return nil, p.config.KeepOriginalImage, err
 		}
-
-		// Set up the state.
-		state := new(multistep.BasicStateBag)
-		state.Put("config", &exporterConfig)
-		state.Put("driver", driver)
-		state.Put("ui", ui)
-
-		// Build the steps.
-		steps := []multistep.Step{
-			&googlecompute.StepCreateSSHKey{
-				Debug:        p.config.PackerDebug,
-				DebugKeyPath: fmt.Sprintf("gce_%s.pem", p.config.PackerBuildName),
-			},
-			&googlecompute.StepCreateInstance{
-				Debug: p.config.PackerDebug,
-			},
-			new(googlecompute.StepWaitStartupScript),
-			new(googlecompute.StepTeardownInstance),
-		}
-
-		// Run the steps.
-		p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)
-		p.runner.Run(state)
 	}
+	if p.config.AccountFile != "" {
+		err := googlecompute.ProcessAccountFile(&p.config.Account, p.config.AccountFile)
+		if err != nil {
+			return nil, p.config.KeepOriginalImage, err
+		}
+	}
+
+	// Set up exporter instance configuration.
+	exporterName := fmt.Sprintf("%s-exporter", artifact.Id())
+	exporterMetadata := map[string]string{
+		"image_name":     builderImageName,
+		"name":           exporterName,
+		"paths":          strings.Join(p.config.Paths, " "),
+		"startup-script": StartupScript,
+		"zone":           p.config.Zone,
+	}
+	exporterConfig := googlecompute.Config{
+		DiskName:             exporterName,
+		DiskSizeGb:           p.config.DiskSizeGb,
+		DiskType:             p.config.DiskType,
+		InstanceName:         exporterName,
+		MachineType:          p.config.MachineType,
+		Metadata:             exporterMetadata,
+		Network:              p.config.Network,
+		RawStateTimeout:      "5m",
+		SourceImageFamily:    "debian-9-worker",
+		SourceImageProjectId: "compute-image-tools",
+		Subnetwork:           p.config.Subnetwork,
+		Zone:                 p.config.Zone,
+		Scopes: []string{
+			"https://www.googleapis.com/auth/compute",
+			"https://www.googleapis.com/auth/devstorage.full_control",
+			"https://www.googleapis.com/auth/userinfo.email",
+		},
+	}
+	exporterConfig.CalcTimeout()
+
+	driver, err := googlecompute.NewDriverGCE(ui, builderProjectId, &p.config.Account)
+	if err != nil {
+		return nil, p.config.KeepOriginalImage, err
+	}
+
+	// Set up the state.
+	state := new(multistep.BasicStateBag)
+	state.Put("config", &exporterConfig)
+	state.Put("driver", driver)
+	state.Put("ui", ui)
+
+	// Build the steps.
+	steps := []multistep.Step{
+		&googlecompute.StepCreateSSHKey{
+			Debug:        p.config.PackerDebug,
+			DebugKeyPath: fmt.Sprintf("gce_%s.pem", p.config.PackerBuildName),
+		},
+		&googlecompute.StepCreateInstance{
+			Debug: p.config.PackerDebug,
+		},
+		new(googlecompute.StepWaitStartupScript),
+		new(googlecompute.StepTeardownInstance),
+	}
+
+	// Run the steps.
+	p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)
+	p.runner.Run(state)
+
+	result := &Artifact{paths: p.config.Paths}
 
 	return result, p.config.KeepOriginalImage, nil
 }

post-processor/googlecompute-export/startup.go

@@ -1,6 +1,6 @@
 package googlecomputeexport
 
-var StartupScript string = `#!/bin/sh
+var StartupScript string = `#!/bin/bash
 
 GetMetadata () {
   echo "$(curl -f -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/attributes/$1 2> /dev/null)"
@@ -8,11 +8,11 @@ GetMetadata () {
 IMAGENAME=$(GetMetadata image_name)
 NAME=$(GetMetadata name)
 DISKNAME=${NAME}-toexport
-PATHS=$(GetMetadata paths)
+PATHS=($(GetMetadata paths))
 ZONE=$(GetMetadata zone)
 
 Exit () {
-  for i in ${PATHS}; do
+  for i in ${PATHS[@]}; do
     LOGDEST="${i}.exporter.log"
     echo "Uploading exporter log to ${LOGDEST}..."
     gsutil -h "Content-Type:text/plain" cp /var/log/daemon.log ${LOGDEST}
@@ -40,17 +40,15 @@ if ! gcloud compute instances attach-disk ${NAME} --disk ${DISKNAME} --device-na
   Exit 1
 fi
 
-echo "Dumping disk..."
-if ! dd if=/dev/disk/by-id/google-toexport of=disk.raw bs=4096 conv=sparse; then
-  echo "Failed to dump disk to image."
+echo "GCEExport: Running export tool."
+gce_export -gcs_path "${PATHS[0]}" -disk /dev/disk/by-id/google-toexport -y
+if [ $? -ne 0 ]; then
+  echo "ExportFailed: Failed to export disk source to ${PATHS[0]}."
   Exit 1
 fi
 
-echo "Compressing and tar'ing disk image..."
-if ! tar -czf root.tar.gz disk.raw; then
-  echo "Failed to tar disk image."
-  Exit 1
-fi
+echo "ExportSuccess"
+sync
 
 echo "Detaching disk..."
 if ! gcloud compute instances detach-disk ${NAME} --disk ${DISKNAME} --zone ${ZONE}; then
@@ -64,10 +62,10 @@ if ! gcloud compute disks delete ${DISKNAME} --zone ${ZONE}; then
   FAIL=1
 fi
 
-for i in ${PATHS}; do
-  echo "Uploading tar'ed disk image to ${i}..."
-  if ! gsutil -o GSUtil:parallel_composite_upload_threshold=100M cp root.tar.gz ${i}; then
-    echo "Failed to upload image to ${i}."
+for i in ${PATHS[@]:1}; do
+  echo "Copying archive image to ${i}..."
+  if ! gsutil -o GSUtil:parallel_composite_upload_threshold=100M cp ${PATHS[0]} ${i}; then
+    echo "Failed to copy image to ${i}."
     FAIL=1
   fi
 done

post-processor/googlecompute-import/post-processor.go

@@ -13,13 +13,9 @@ import (
 	"github.com/hashicorp/packer/builder/googlecompute"
 	"github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/helper/config"
-	"github.com/hashicorp/packer/helper/multistep"
 	"github.com/hashicorp/packer/packer"
 	"github.com/hashicorp/packer/post-processor/compress"
 	"github.com/hashicorp/packer/template/interpolate"
-
-	"golang.org/x/oauth2"
-	"golang.org/x/oauth2/jwt"
 )
 
 type Config struct {
@@ -38,12 +34,12 @@ type Config struct {
 	KeepOriginalImage    bool              `mapstructure:"keep_input_artifact"`
 	SkipClean            bool              `mapstructure:"skip_clean"`
 
-	ctx interpolate.Context
+	Account googlecompute.AccountFile
+	ctx     interpolate.Context
 }
 
 type PostProcessor struct {
 	config Config
-	runner multistep.Runner
 }
 
 func (p *PostProcessor) Configure(raws ...interface{}) error {
@@ -60,24 +56,29 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 		return err
 	}
 
+	errs := new(packer.MultiError)
+
 	// Set defaults
 	if p.config.GCSObjectName == "" {
 		p.config.GCSObjectName = "packer-import-{{timestamp}}.tar.gz"
 	}
 
-	errs := new(packer.MultiError)
-
 	// Check and render gcs_object_name
 	if err = interpolate.Validate(p.config.GCSObjectName, &p.config.ctx); err != nil {
 		errs = packer.MultiErrorAppend(
 			errs, fmt.Errorf("Error parsing gcs_object_name template: %s", err))
 	}
 
+	if p.config.AccountFile != "" {
+		if err := googlecompute.ProcessAccountFile(&p.config.Account, p.config.AccountFile); err != nil {
+			errs = packer.MultiErrorAppend(errs, err)
+		}
+	}
+
 	templates := map[string]*string{
-		"bucket":       &p.config.Bucket,
-		"image_name":   &p.config.ImageName,
-		"project_id":   &p.config.ProjectId,
-		"account_file": &p.config.AccountFile,
+		"bucket":     &p.config.Bucket,
+		"image_name": &p.config.ImageName,
+		"project_id": &p.config.ProjectId,
 	}
 	for key, ptr := range templates {
 		if *ptr == "" {
@@ -94,7 +95,10 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 }
 
 func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
-	var err error
+	client, err := googlecompute.NewClientGCE(&p.config.Account)
+	if err != nil {
+		return nil, false, err
+	}
 
 	if artifact.BuilderId() != compress.BuilderId {
 		err = fmt.Errorf(
@@ -108,18 +112,18 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac
 		return nil, false, fmt.Errorf("Error rendering gcs_object_name template: %s", err)
 	}
 
-	rawImageGcsPath, err := UploadToBucket(p.config.AccountFile, ui, artifact, p.config.Bucket, p.config.GCSObjectName)
+	rawImageGcsPath, err := UploadToBucket(client, ui, artifact, p.config.Bucket, p.config.GCSObjectName)
 	if err != nil {
 		return nil, p.config.KeepOriginalImage, err
 	}
 
-	gceImageArtifact, err := CreateGceImage(p.config.AccountFile, ui, p.config.ProjectId, rawImageGcsPath, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels, p.config.ImageGuestOsFeatures)
+	gceImageArtifact, err := CreateGceImage(client, ui, p.config.ProjectId, rawImageGcsPath, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels, p.config.ImageGuestOsFeatures)
 	if err != nil {
 		return nil, p.config.KeepOriginalImage, err
 	}
 
 	if !p.config.SkipClean {
-		err = DeleteFromBucket(p.config.AccountFile, ui, p.config.Bucket, p.config.GCSObjectName)
+		err = DeleteFromBucket(client, ui, p.config.Bucket, p.config.GCSObjectName)
 		if err != nil {
 			return nil, p.config.KeepOriginalImage, err
 		}
@@ -128,24 +132,7 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac
 	return gceImageArtifact, p.config.KeepOriginalImage, nil
 }
 
-func UploadToBucket(accountFile string, ui packer.Ui, artifact packer.Artifact, bucket string, gcsObjectName string) (string, error) {
-	var client *http.Client
-	var account googlecompute.AccountFile
-
-	err := googlecompute.ProcessAccountFile(&account, accountFile)
-	if err != nil {
-		return "", err
-	}
-
-	var DriverScopes = []string{"https://www.googleapis.com/auth/devstorage.full_control"}
-	conf := jwt.Config{
-		Email:      account.ClientEmail,
-		PrivateKey: []byte(account.PrivateKey),
-		Scopes:     DriverScopes,
-		TokenURL:   "https://accounts.google.com/o/oauth2/token",
-	}
-
-	client = conf.Client(oauth2.NoContext)
+func UploadToBucket(client *http.Client, ui packer.Ui, artifact packer.Artifact, bucket string, gcsObjectName string) (string, error) {
 	service, err := storage.New(client)
 	if err != nil {
 		return "", err
@@ -162,7 +149,7 @@ func UploadToBucket(accountFile string, ui packer.Ui, artifact packer.Artifact,
 	}
 
 	if source == "" {
-		return "", fmt.Errorf("No tar.gz file found in list of articats")
+		return "", fmt.Errorf("No tar.gz file found in list of artifacts")
 	}
 
 	artifactFile, err := os.Open(source)
@@ -178,28 +165,10 @@ func UploadToBucket(accountFile string, ui packer.Ui, artifact packer.Artifact,
 		return "", err
 	}
 
-	return "https://storage.googleapis.com/" + bucket + "/" + gcsObjectName, nil
+	return storageObject.SelfLink, nil
 }
 
-func CreateGceImage(accountFile string, ui packer.Ui, project string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string, imageGuestOsFeatures []string) (packer.Artifact, error) {
-	var client *http.Client
-	var account googlecompute.AccountFile
-
-	err := googlecompute.ProcessAccountFile(&account, accountFile)
-	if err != nil {
-		return nil, err
-	}
-
-	var DriverScopes = []string{"https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/devstorage.full_control"}
-	conf := jwt.Config{
-		Email:      account.ClientEmail,
-		PrivateKey: []byte(account.PrivateKey),
-		Scopes:     DriverScopes,
-		TokenURL:   "https://accounts.google.com/o/oauth2/token",
-	}
-
-	client = conf.Client(oauth2.NoContext)
-
+func CreateGceImage(client *http.Client, ui packer.Ui, project string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string, imageGuestOsFeatures []string) (packer.Artifact, error) {
 	service, err := compute.New(client)
 	if err != nil {
 		return nil, err
@@ -253,24 +222,7 @@ func CreateGceImage(accountFile string, ui packer.Ui, project string, rawImageUR
 	return &Artifact{paths: []string{op.TargetLink}}, nil
 }
 
-func DeleteFromBucket(accountFile string, ui packer.Ui, bucket string, gcsObjectName string) error {
-	var client *http.Client
-	var account googlecompute.AccountFile
-
-	err := googlecompute.ProcessAccountFile(&account, accountFile)
-	if err != nil {
-		return err
-	}
-
-	var DriverScopes = []string{"https://www.googleapis.com/auth/devstorage.full_control"}
-	conf := jwt.Config{
-		Email:      account.ClientEmail,
-		PrivateKey: []byte(account.PrivateKey),
-		Scopes:     DriverScopes,
-		TokenURL:   "https://accounts.google.com/o/oauth2/token",
-	}
-
-	client = conf.Client(oauth2.NoContext)
+func DeleteFromBucket(client *http.Client, ui packer.Ui, bucket string, gcsObjectName string) error {
 	service, err := storage.New(client)
 	if err != nil {
 		return err

website/source/docs/post-processors/googlecompute-export.html.md

@@ -34,9 +34,39 @@ permissions to the GCS `paths`.
 
 ### Optional
 
+-   `account_file` (string) - The JSON file containing your account
+    credentials. If specified, this take precedence over `googlecompute`
+    builder authentication method.
+
+-   `disk_size` (number) - The size of the export instances disk, this disk
+    is unused for the export but a larger size increase `pd-ssd` read speed.
+    This defaults to `200`, which is 200GB.
+
+-   `disk_type` (string) - Type of disk used to back export instance, like
+    `pd-ssd` or `pd-standard`. Defaults to `pd-ssd`.
+
 -   `keep_input_artifact` (boolean) - If true, do not delete the Google Compute
     Engine (GCE) image being exported.
 
+-   `machine_type` (string) - The export instance machine type. Defaults
+    to `"n1-highcpu-4"`.
+
+-   `network` (string) - The Google Compute network id or URL to use for the
+    export instance. Defaults to `"default"`. If the value is not a URL, it
+    will be interpolated to
+    `projects/((network_project_id))/global/networks/((network))`. This value
+    is not required if a `subnet` is specified.
+
+-   `subnetwork` (string) - The Google Compute subnetwork id or URL to use for
+    the export instance. Only required if the `network` has been created with
+    custom subnetting. Note, the region of the subnetwork must match the
+    `zone` in which the VM is launched. If the value is not a URL,
+    it will be interpolated to
+    `projects/((network_project_id))/regions/((region))/subnetworks/((subnetwork))`
+
+-   `zone` (string) - The zone in which to launch the export instance. Defaults
+    to `googlecompute` builder zone. Example: `"us-central1-a"`
+
 ## Basic Example
 
 The following example builds a GCE image in the project, `my-project`, with an