From ff149df30fee608c6e72680e954a3c2d3dfcd3ff Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Mon, 17 Nov 2014 10:06:22 -0800 Subject: [PATCH] Use golang/oauth2, no longer require client_secrets.json, and use Service Account when run from a GCE Instance. --- builder/googlecompute/account.go | 10 --- builder/googlecompute/builder.go | 2 +- builder/googlecompute/config.go | 20 ----- builder/googlecompute/config_test.go | 33 --------- builder/googlecompute/driver_gce.go | 58 +++++++-------- test/README.md | 3 +- test/builder_googlecompute.bats | 8 +- .../builder-googlecompute/minimal.json | 8 +- .../docs/builders/googlecompute.markdown | 74 ++++++++++++------- 9 files changed, 82 insertions(+), 134 deletions(-) diff --git a/builder/googlecompute/account.go b/builder/googlecompute/account.go index 438181965..59baf6044 100644 --- a/builder/googlecompute/account.go +++ b/builder/googlecompute/account.go @@ -13,16 +13,6 @@ type accountFile struct { ClientId string `json:"client_id"` } -// clientSecretsFile represents the structure of the client secrets JSON file. -type clientSecretsFile struct { - Web struct { - AuthURI string `json:"auth_uri"` - ClientEmail string `json:"client_email"` - ClientId string `json:"client_id"` - TokenURI string `json:"token_uri"` - } -} - func loadJSON(result interface{}, path string) error { f, err := os.Open(path) if err != nil { diff --git a/builder/googlecompute/builder.go b/builder/googlecompute/builder.go index e5b3d171a..4b0b32ca5 100644 --- a/builder/googlecompute/builder.go +++ b/builder/googlecompute/builder.go @@ -35,7 +35,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { // representing a GCE machine image. func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { driver, err := NewDriverGCE( - ui, b.config.ProjectId, &b.config.account, &b.config.clientSecrets) + ui, b.config.ProjectId, &b.config.account) if err != nil { return nil, err } diff --git a/builder/googlecompute/config.go b/builder/googlecompute/config.go index 5bed83b24..5c3639402 100644 --- a/builder/googlecompute/config.go +++ b/builder/googlecompute/config.go @@ -17,7 +17,6 @@ type Config struct { common.PackerConfig `mapstructure:",squash"` AccountFile string `mapstructure:"account_file"` - ClientSecretsFile string `mapstructure:"client_secrets_file"` ProjectId string `mapstructure:"project_id"` BucketName string `mapstructure:"bucket_name"` @@ -38,7 +37,6 @@ type Config struct { Zone string `mapstructure:"zone"` account accountFile - clientSecrets clientSecretsFile instanceName string privateKeyBytes []byte sshTimeout time.Duration @@ -106,7 +104,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { // Process Templates templates := map[string]*string{ "account_file": &c.AccountFile, - "client_secrets_file": &c.ClientSecretsFile, "bucket_name": &c.BucketName, "image_name": &c.ImageName, @@ -138,16 +135,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { errs, errors.New("a bucket_name must be specified")) } - if c.AccountFile == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("an account_file must be specified")) - } - - if c.ClientSecretsFile == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("a client_secrets_file must be specified")) - } - if c.ProjectId == "" { errs = packer.MultiErrorAppend( errs, errors.New("a project_id must be specified")) @@ -185,13 +172,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { } } - if c.ClientSecretsFile != "" { - if err := loadJSON(&c.clientSecrets, c.ClientSecretsFile); err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing client secrets file: %s", err)) - } - } - // Check for any errors. if errs != nil && len(errs.Errors) > 0 { return nil, nil, errs diff --git a/builder/googlecompute/config_test.go b/builder/googlecompute/config_test.go index 83a188020..acc898ce7 100644 --- a/builder/googlecompute/config_test.go +++ b/builder/googlecompute/config_test.go @@ -9,7 +9,6 @@ func testConfig(t *testing.T) map[string]interface{} { return map[string]interface{}{ "account_file": testAccountFile(t), "bucket_name": "foo", - "client_secrets_file": testClientSecretsFile(t), "project_id": "hashicorp", "source_image": "foo", "zone": "us-east-1a", @@ -69,22 +68,6 @@ func TestConfigPrepare(t *testing.T) { false, }, - { - "client_secrets_file", - nil, - true, - }, - { - "client_secrets_file", - testClientSecretsFile(t), - false, - }, - { - "client_secrets_file", - "/tmp/i/should/not/exist", - true, - }, - { "private_key_file", "/tmp/i/should/not/exist", @@ -180,22 +163,6 @@ func testAccountFile(t *testing.T) string { return tf.Name() } -func testClientSecretsFile(t *testing.T) string { - tf, err := ioutil.TempFile("", "packer") - if err != nil { - t.Fatalf("err: %s", err) - } - defer tf.Close() - - if _, err := tf.Write([]byte(testClientSecretsContent)); err != nil { - t.Fatalf("err: %s", err) - } - - return tf.Name() -} - // This is just some dummy data that doesn't actually work (it was revoked // a long time ago). const testAccountContent = `{}` - -const testClientSecretsContent = `{"web":{"auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://accounts.google.com/o/oauth2/token","client_email":"774313886706-eorlsj0r4eqkh5e7nvea5fuf59ifr873@developer.gserviceaccount.com","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/774313886706-eorlsj0r4eqkh5e7nvea5fuf59ifr873@developer.gserviceaccount.com","client_id":"774313886706-eorlsj0r4eqkh5e7nvea5fuf59ifr873.apps.googleusercontent.com","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs"}}` diff --git a/builder/googlecompute/driver_gce.go b/builder/googlecompute/driver_gce.go index 0936d36a0..e1f9b44ad 100644 --- a/builder/googlecompute/driver_gce.go +++ b/builder/googlecompute/driver_gce.go @@ -6,9 +6,9 @@ import ( "net/http" "time" - "code.google.com/p/goauth2/oauth" - "code.google.com/p/goauth2/oauth/jwt" "code.google.com/p/google-api-go-client/compute/v1" + "github.com/golang/oauth2" + "github.com/golang/oauth2/google" "github.com/mitchellh/packer/packer" ) @@ -20,39 +20,35 @@ type driverGCE struct { ui packer.Ui } -const DriverScopes string = "https://www.googleapis.com/auth/compute " + - "https://www.googleapis.com/auth/devstorage.full_control" +var DriverScopes = []string{"https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/devstorage.full_control"} + +func NewDriverGCE(ui packer.Ui, p string, a *accountFile) (Driver, error) { + var f *oauth2.Flow + var err error + + // Auth with AccountFile first if provided + if a.PrivateKey != "" { + log.Printf("[INFO] Requesting Google token via AccountFile...") + log.Printf("[INFO] -- Email: %s", a.ClientEmail) + log.Printf("[INFO] -- Scopes: %s", DriverScopes) + log.Printf("[INFO] -- Private Key Length: %d", len(a.PrivateKey)) + + f, err = oauth2.New( + oauth2.JWTClient(a.ClientEmail, []byte(a.PrivateKey)), + oauth2.Scope(DriverScopes...), + google.JWTEndpoint()) + } else { + log.Printf("[INFO] Requesting Google token via GCE Service Role...") + + f, err = oauth2.New(google.ComputeEngineAccount("")) + } -func NewDriverGCE(ui packer.Ui, p string, a *accountFile, c *clientSecretsFile) (Driver, error) { - // Get the token for use in our requests - log.Printf("[INFO] Requesting Google token...") - log.Printf("[INFO] -- Email: %s", a.ClientEmail) - log.Printf("[INFO] -- Scopes: %s", DriverScopes) - log.Printf("[INFO] -- Private Key Length: %d", len(a.PrivateKey)) - log.Printf("[INFO] -- Token URL: %s", c.Web.TokenURI) - jwtTok := jwt.NewToken( - a.ClientEmail, - DriverScopes, - []byte(a.PrivateKey)) - jwtTok.ClaimSet.Aud = c.Web.TokenURI - token, err := jwtTok.Assert(new(http.Client)) if err != nil { - return nil, fmt.Errorf("Error retrieving auth token: %s", err) + return nil, err } - // Instantiate the transport to communicate to Google - transport := &oauth.Transport{ - Config: &oauth.Config{ - ClientId: a.ClientId, - Scope: DriverScopes, - TokenURL: c.Web.TokenURI, - AuthURL: c.Web.AuthURI, - }, - Token: token, - } - - log.Printf("[INFO] Instantiating GCE client...") - service, err := compute.New(transport.Client()) + log.Printf("[INFO] Instantiating GCE client using...") + service, err := compute.New(&http.Client{Transport: f.NewTransport()}) if err != nil { return nil, err } diff --git a/test/README.md b/test/README.md index 79d1d7909..e29fe9b24 100644 --- a/test/README.md +++ b/test/README.md @@ -41,8 +41,7 @@ Set the following self-explanatory environmental variables: Set the following environmental variables: * `GC_BUCKET_NAME` -* `GC_CLIENT_SECRETS_FILE` -* `GC_PRIVATE_KEY_FILE` +* `GC_ACCOUNT_FILE` * `GC_PROJECT_ID` ### Running diff --git a/test/builder_googlecompute.bats b/test/builder_googlecompute.bats index 2c1ad8ab0..d7062a9a8 100755 --- a/test/builder_googlecompute.bats +++ b/test/builder_googlecompute.bats @@ -8,8 +8,7 @@ fixtures builder-googlecompute # Required parameters : ${GC_BUCKET_NAME:?} -: ${GC_CLIENT_SECRETS_FILE:?} -: ${GC_PRIVATE_KEY_FILE:?} +: ${GC_ACCOUNT_FILE:?} : ${GC_PROJECT_ID:?} command -v gcutil >/dev/null 2>&1 || { echo "'gcutil' must be installed" >&2 @@ -17,8 +16,7 @@ command -v gcutil >/dev/null 2>&1 || { } USER_VARS="-var bucket_name=${GC_BUCKET_NAME}" -USER_VARS="${USER_VARS} -var client_secrets_file=${GC_CLIENT_SECRETS_FILE}" -USER_VARS="${USER_VARS} -var private_key_file=${GC_PRIVATE_KEY_FILE}" +USER_VARS="${USER_VARS} -var account_file=${GC_ACCOUNT_FILE}" USER_VARS="${USER_VARS} -var project_id=${GC_PROJECT_ID}" # This tests if GCE has an image that contains the given parameter. @@ -30,7 +28,7 @@ gc_has_image() { teardown() { gcutil --format=names --project=${GC_PROJECT_ID} listimages \ | grep packerbats \ - | xargs -n1 gcutil --project=${GC_PROJECT_ID} --force deleteimage + | xargs -n1 gcutil --project=${GC_PROJECT_ID} deleteimage --force } @test "googlecompute: build minimal.json" { diff --git a/test/fixtures/builder-googlecompute/minimal.json b/test/fixtures/builder-googlecompute/minimal.json index f2a206833..218e98b9a 100644 --- a/test/fixtures/builder-googlecompute/minimal.json +++ b/test/fixtures/builder-googlecompute/minimal.json @@ -1,20 +1,18 @@ { "variables": { "bucket_name": null, - "client_secrets_file": null, - "private_key_file": null, + "account_file": null, "project_id": null }, "builders": [{ "type": "googlecompute", "bucket_name": "{{user `bucket_name`}}", - "client_secrets_file": "{{user `client_secrets_file`}}", - "private_key_file": "{{user `private_key_file`}}", + "account_file": "{{user `account_file`}}", "project_id": "{{user `project_id`}}", "image_name": "packerbats-minimal-{{timestamp}}", - "source_image": "debian-7-wheezy-v20131120", + "source_image": "debian-7-wheezy-v20141108", "zone": "us-central1-a" }] } diff --git a/website/source/docs/builders/googlecompute.markdown b/website/source/docs/builders/googlecompute.markdown index 7e4355eec..56d568ac0 100644 --- a/website/source/docs/builders/googlecompute.markdown +++ b/website/source/docs/builders/googlecompute.markdown @@ -9,19 +9,49 @@ description: |- Type: `googlecompute` -The `googlecompute` Packer builder is able to create -[images](https://developers.google.com/compute/docs/images) -for use with [Google Compute Engine](https://cloud.google.com/products/compute-engine) -(GCE) based on existing images. Google Compute Engine doesn't allow the creation -of images from scratch. +The `googlecompute` Packer builder is able to create [images](https://developers.google.com/compute/docs/images) for use with +[Google Compute Engine](https://cloud.google.com/products/compute-engine)(GCE) based on existing images. Google +Compute Engine doesn't allow the creation of images from scratch. ## Authentication -Authenticating with Google Cloud services requires two separate JSON -files: one which we call the _account file_ and the _client secrets file_. +Authenticating with Google Cloud services requires at most one JSON file, +called the _account file_. The _account file_ is **not** required if you are running +the `googlecompute` Packer builder from a GCE instance with a properly-configured +[Compute Engine Service Account](https://cloud.google.com/compute/docs/authentication. -Both of these files are downloaded directly from the -[Google Developers Console](https://console.developers.google.com). To make +### Running With a Compute Engine Service Account +If you run the `googlecompute` Packer builder from a GCE instance, you can configure that +instance to use a [Compute Engine Service Account](https://cloud.google.com/compute/docs/authentication). This will allow Packer to authenticate +to Google Cloud without having to bake in a separate credential/authentication file. + +To create a GCE instance that uses a service account, provide the required scopes when +launching the intance. + +For `gcloud`, do this via the `--scopes` parameter: + +```sh +gcloud compute --project YOUR_PROJECT instances create "INSTANCE-NAME" ... \ + --scopes "https://www.googleapis.com/auth/compute" \ + "https://www.googleapis.com/auth/devstorage.full_control" \ + ... +``` + +For the [Google Developers Console](https://console.developers.google.com): + +1. Choose "Show advanced options" +2. Tick "Enable Compute Engine service account" +3. Choose "Read Write" for Compute +4. Chose "Full" for "Storage" + +**The service account will be used automatically by Packer as long as there is +no _account file_ specified in the Packer configuration file.** + +### Running Without a Compute Engine Service Account + +The [Google Developers Console](https://console.developers.google.com) allows you to +create and download a credential file that will let you use the `googlecompute` Packer +builder anywhere. To make the process more straightforwarded, it is documented here. 1. Log into the [Google Developers Console](https://console.developers.google.com) @@ -29,27 +59,22 @@ the process more straightforwarded, it is documented here. 2. Under the "APIs & Auth" section, click "Credentials." -3. Click the "Download JSON" button under the "Compute Engine and App Engine" - account in the OAuth section. The file should start with "client\_secrets". - This is your _client secrets file_. +3. Click the "Create new Client ID" button, select "Service account", and click "Create Client ID" -4. Create a new OAuth client ID and select "Service Account" as the type - of account. Once created, a JSON file should be downloaded. This is your +4. Click "Generate new JSON key" for the Service Account you just created. A JSON file will be downloaded automatically. This is your _account file_. ## Basic Example Below is a fully functioning example. It doesn't do anything useful, since no provisioners are defined, but it will effectively repackage an -existing GCE image. The client secrets file and private key file are the -files obtained in the previous section. +existing GCE image. The account file is obtained in the previous section. ```javascript { "type": "googlecompute", "bucket_name": "my-project-packer-images", "account_file": "account.json", - "client_secrets_file": "client_secret.json", "project_id": "my-project", "source_image": "debian-7-wheezy-v20140718", "zone": "us-central1-a" @@ -63,17 +88,8 @@ each category, the available options are alphabetized and described. ### Required: -* `account_file` (string) - The JSON file containing your account credentials. - Instructions for how to retrieve these are above. - * `bucket_name` (string) - The Google Cloud Storage bucket to store the - images that are created. The bucket must already exist in your project. - -* `client_secrets_file` (string) - The client secrets JSON file that - was set up in the section above. - -* `private_key_file` (string) - The client private key file that was - generated in the section above. + images that are created. The bucket must already exist in your project * `project_id` (string) - The project ID that will be used to launch instances and store images. @@ -86,6 +102,10 @@ each category, the available options are alphabetized and described. ### Optional: +* `account_file` (string) - The JSON file containing your account credentials. + Not required if you run Packer on a GCE instance with a service account. + Instructions for creating file or using service accounts are above. + * `disk_size` (integer) - The size of the disk in GB. This defaults to 10, which is 10GB.