Merge pull request #7282 from hashicorp/do_6994

Allow amazon builders to read credentials from Vault.

builder/amazon/chroot/builder.go

@@ -205,6 +205,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	// Setup the state bag and initial state for the steps
 	state := new(multistep.BasicStateBag)
 	state.Put("config", &b.config)
+	state.Put("access_config", &b.config.AccessConfig)
+	state.Put("ami_config", &b.config.AMIConfig)
 	state.Put("ec2", ec2conn)
 	state.Put("awsSession", session)
 	state.Put("hook", hook)

builder/amazon/common/access_config.go

@@ -16,8 +16,21 @@ import (
 	cleanhttp "github.com/hashicorp/go-cleanhttp"
 	commonhelper "github.com/hashicorp/packer/helper/common"
 	"github.com/hashicorp/packer/template/interpolate"
+	vaultapi "github.com/hashicorp/vault/api"
 )
 
+type VaultAWSEngineOptions struct {
+	Name       string `mapstructure:"name"`
+	RoleARN    string `mapstructure:"role_arn"`
+	TTL        string `mapstructure:"ttl"`
+	EngineName string `mapstructure:"engine_name"`
+}
+
+func (v *VaultAWSEngineOptions) Empty() bool {
+	return len(v.Name) == 0 && len(v.RoleARN) == 0 &&
+		len(v.EngineName) == 0 && len(v.TTL) == 0
+}
+
 // AccessConfig is for common configuration related to AWS access
 type AccessConfig struct {
 	AccessKey             string `mapstructure:"access_key"`
@@ -32,6 +45,7 @@ type AccessConfig struct {
 	SkipMetadataApiCheck  bool   `mapstructure:"skip_metadata_api_check"`
 	Token                 string `mapstructure:"token"`
 	session               *session.Session
+	VaultAWSEngine        VaultAWSEngineOptions `mapstructure:"vault_aws_engine"`
 
 	getEC2Connection func() ec2iface.EC2API
 }
@@ -44,6 +58,7 @@ func (c *AccessConfig) Session() (*session.Session, error) {
 	}
 
 	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true)
+
 	staticCreds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
 	if _, err := staticCreds.Get(); err != credentials.ErrStaticCredentialsEmpty {
 		config.WithCredentials(staticCreds)
@@ -122,14 +137,62 @@ func (c *AccessConfig) IsChinaCloud() bool {
 	return strings.HasPrefix(c.SessionRegion(), "cn-")
 }
 
+func (c *AccessConfig) GetCredsFromVault() error {
+	// const EnvVaultAddress = "VAULT_ADDR"
+	// const EnvVaultToken = "VAULT_TOKEN"
+	vaultConfig := vaultapi.DefaultConfig()
+	cli, err := vaultapi.NewClient(vaultConfig)
+	if err != nil {
+		return fmt.Errorf("Error getting Vault client: %s", err)
+	}
+	if c.VaultAWSEngine.EngineName == "" {
+		c.VaultAWSEngine.EngineName = "aws"
+	}
+	path := fmt.Sprintf("/%s/creds/%s", c.VaultAWSEngine.EngineName,
+		c.VaultAWSEngine.Name)
+	secret, err := cli.Logical().Read(path)
+	if err != nil {
+		return fmt.Errorf("Error reading vault secret: %s", err)
+	}
+	if secret == nil {
+		return fmt.Errorf("Vault Secret does not exist at the given path.")
+	}
+
+	c.AccessKey = secret.Data["access_key"].(string)
+	c.SecretKey = secret.Data["secret_key"].(string)
+	token := secret.Data["security_token"]
+	if token != nil {
+		c.Token = token.(string)
+	} else {
+		c.Token = ""
+	}
+
+	return nil
+}
+
 func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
 	var errs []error
 
 	if c.SkipMetadataApiCheck {
 		log.Println("(WARN) skip_metadata_api_check ignored.")
 	}
-	// Either both access and secret key must be set or neither of them should
-	// be.
+
+	// Make sure it's obvious from the config how we're getting credentials:
+	// Vault, Packer config, or environemnt.
+	if !c.VaultAWSEngine.Empty() {
+		if len(c.AccessKey) > 0 {
+			errs = append(errs,
+				fmt.Errorf("If you have set vault_aws_engine, you must not set"+
+					" the access_key or secret_key."))
+		}
+		// Go ahead and grab those credentials from Vault now, so we can set
+		// the keys and token now.
+		err := c.GetCredsFromVault()
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
 	if (len(c.AccessKey) > 0) != (len(c.SecretKey) > 0) {
 		errs = append(errs,
 			fmt.Errorf("`access_key` and `secret_key` must both be either set or not set."))

builder/amazon/common/step_pre_validate.go

@@ -3,9 +3,12 @@ package common
 import (
 	"context"
 	"fmt"
+	"log"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	retry "github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/helper/multistep"
 	"github.com/hashicorp/packer/packer"
 )
@@ -20,6 +23,51 @@ type StepPreValidate struct {
 
 func (s *StepPreValidate) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
 	ui := state.Get("ui").(packer.Ui)
+
+	if accessConfig, ok := state.GetOk("access_config"); ok {
+		accessconf := accessConfig.(*AccessConfig)
+		if !accessconf.VaultAWSEngine.Empty() {
+			// loop over the authentication a few times to give vault-created creds
+			// time to become eventually-consistent
+			ui.Say("You're using Vault-generated AWS credentials. It may take a " +
+				"few moments for them to become available on AWS. Waiting...")
+			err := retry.Retry(0.2, 30, 11, func(_ uint) (bool, error) {
+				ec2conn, err := accessconf.NewEC2Connection()
+				if err != nil {
+					return true, err
+				}
+				_, err = listEC2Regions(ec2conn)
+				if err != nil {
+					if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "AuthFailure" {
+						log.Printf("Waiting for Vault-generated AWS credentials" +
+							" to pass authentication... trying again.")
+						return false, nil
+					}
+					return true, err
+				}
+				return true, nil
+			})
+
+			if err != nil {
+				state.Put("error", fmt.Errorf("Was unable to Authenticate to AWS using Vault-"+
+					"Generated Credentials within the retry timeout."))
+				return multistep.ActionHalt
+			}
+		}
+
+		if amiConfig, ok := state.GetOk("ami_config"); ok {
+			amiconf := amiConfig.(*AMIConfig)
+			if !amiconf.AMISkipRegionValidation {
+				regionsToValidate := append(amiconf.AMIRegions, accessconf.RawRegion)
+				err := accessconf.ValidateRegion(regionsToValidate...)
+				if err != nil {
+					state.Put("error", fmt.Errorf("error validating regions: %v", err))
+					return multistep.ActionHalt
+				}
+			}
+		}
+	}
+
 	if s.ForceDeregister {
 		ui.Say("Force Deregister flag found, skipping prevalidating AMI Name")
 		return multistep.ActionContinue

builder/amazon/ebs/builder.go

@@ -92,13 +92,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	if err != nil {
 		return nil, err
 	}
-	if !b.config.AMISkipRegionValidation {
-		regionsToValidate := append(b.config.AMIRegions, b.config.RawRegion)
-		err := b.config.AccessConfig.ValidateRegion(regionsToValidate...)
-		if err != nil {
-			return nil, fmt.Errorf("error validating regions: %v", err)
-		}
-	}
+
 	ec2conn := ec2.New(session, &aws.Config{
 		HTTPClient: commonhelper.HttpClientWithEnvironmentProxy(),
 	})
@@ -106,6 +100,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	// Setup the state bag and initial state for the steps
 	state := new(multistep.BasicStateBag)
 	state.Put("config", &b.config)
+	state.Put("access_config", &b.config.AccessConfig)
+	state.Put("ami_config", &b.config.AMIConfig)
 	state.Put("ec2", ec2conn)
 	state.Put("awsSession", session)
 	state.Put("hook", hook)

builder/amazon/ebssurrogate/builder.go

@@ -114,6 +114,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	// Setup the state bag and initial state for the steps
 	state := new(multistep.BasicStateBag)
 	state.Put("config", &b.config)
+	state.Put("access_config", &b.config.AccessConfig)
+	state.Put("ami_config", &b.config.AMIConfig)
 	state.Put("ec2", ec2conn)
 	state.Put("awsSession", session)
 	state.Put("hook", hook)

builder/amazon/ebsvolume/builder.go

@@ -103,6 +103,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	// Setup the state bag and initial state for the steps
 	state := new(multistep.BasicStateBag)
 	state.Put("config", &b.config)
+	state.Put("access_config", &b.config.AccessConfig)
 	state.Put("ec2", ec2conn)
 	state.Put("hook", hook)
 	state.Put("ui", ui)

builder/amazon/instance/builder.go

@@ -184,6 +184,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	// Setup the state bag and initial state for the steps
 	state := new(multistep.BasicStateBag)
 	state.Put("config", &b.config)
+	state.Put("access_config", &b.config.AccessConfig)
+	state.Put("ami_config", &b.config.AMIConfig)
 	state.Put("ec2", ec2conn)
 	state.Put("awsSession", session)
 	state.Put("hook", hook)

website/source/docs/builders/amazon-chroot.html.md

@@ -24,7 +24,7 @@ builder is able to build an EBS-backed AMI without launching a new EC2
 instance. This can dramatically speed up AMI builds for organizations who need
 the extra fast build.
 
-~> **This is an advanced builder** If you're just getting started with
+\~> **This is an advanced builder** If you're just getting started with
 Packer, we recommend starting with the [amazon-ebs
 builder](/docs/builders/amazon-ebs.html), which is much easier to use.
 
@@ -154,8 +154,8 @@ each category, the available configuration keys are alphabetized.
     associated with AMIs, which have been deregistered by `force_deregister`.
     Default `false`.
 
--   `insecure_skip_tls_verify` (boolean) - This allows skipping TLS verification of
-    the AWS EC2 endpoint. The default is `false`.
+-   `insecure_skip_tls_verify` (boolean) - This allows skipping TLS
+    verification of the AWS EC2 endpoint. The default is `false`.
 
 -   `kms_key_id` (string) - ID, alias or ARN of the KMS key to use for boot
     volume encryption. This only applies to the main `region`, other regions
@@ -362,6 +362,34 @@ each category, the available configuration keys are alphabetized.
     [template engine](/docs/templates/engine.html), see [Build template
     data](#build-template-data) for more information.
 
+-   `vault_aws_engine` (object) - Get credentials from Hashicorp Vault's aws
+    secrets engine. You must already have created a role to use. For more
+    information about generating credentials via the Vault engine, see the
+    [Vault
+    docs.](https://www.vaultproject.io/api/secret/aws/index.html#generate-credentials)
+    If you set this flag, you must also set the below options:
+    -   `name` (string) - Required. Specifies the name of the role to generate
+        credentials against. This is part of the request URL.
+    -   `engine_name` (string) - The name of the aws secrets engine. In the
+        Vault docs, this is normally referred to as "aws", and Packer will
+        default to "aws" if `engine_name` is not set.
+    -   `role_arn` (string)- The ARN of the role to assume if credential\_type
+        on the Vault role is assumed\_role. Must match one of the allowed role
+        ARNs in the Vault role. Optional if the Vault role only allows a single
+        AWS role ARN; required otherwise.
+    -   `ttl` (string) - Specifies the TTL for the use of the STS token. This
+        is specified as a string with a duration suffix. Valid only when
+        credential\_type is assumed\_role or federation\_token. When not
+        specified, the default\_sts\_ttl set for the role will be used. If that
+        is also not set, then the default value of 3600s will be used. AWS
+        places limits on the maximum TTL allowed. See the AWS documentation on
+        the DurationSeconds parameter for AssumeRole (for assumed\_role
+        credential types) and GetFederationToken (for federation\_token
+        credential types) for more details.
+
+        Example:
+        `json   {   "vault_aws_engine": {       "name": "myrole",       "role_arn": "myarn",       "ttl": "3600s"   }   }`
+
 ## Basic Example
 
 Here is a basic example. It is completely valid except for the access keys:
@@ -457,8 +485,8 @@ services:
 
 ### Ansible provisioner
 
-Running ansible against `amazon-chroot` requires changing the Ansible connection
-to chroot and running Ansible as root/sudo.
+Running ansible against `amazon-chroot` requires changing the Ansible
+connection to chroot and running Ansible as root/sudo.
 
 ### Using Instances with NVMe block devices.
 

website/source/docs/builders/amazon-ebs.html.md

@@ -46,7 +46,9 @@ builder.
 ### Required:
 
 -   `access_key` (string) - The access key used to communicate with AWS. [Learn
-    how to set this](amazon.html#specifying-amazon-credentials)
+    how to set this](amazon.html#specifying-amazon-credentials). This is not
+    required if you are using `use_vault_aws_engine` for authentication
+    instead.
 
 -   `ami_name` (string) - The name of the resulting AMI that will appear when
     managing AMIs in the AWS console or via APIs. This must be unique. To help
@@ -60,7 +62,9 @@ builder.
     to launch the EC2 instance to create the AMI.
 
 -   `secret_key` (string) - The secret key used to communicate with AWS. [Learn
-    how to set this](amazon.html#specifying-amazon-credentials)
+    how to set this](amazon.html#specifying-amazon-credentials). This is not
+    required if you are using `use_vault_aws_engine` for authentication
+    instead.
 
 -   `source_ami` (string) - The initial AMI used as a base for the newly
     created machine. `source_ami_filter` may be used instead to populate this
@@ -505,6 +509,41 @@ builder.
 -   `user_data_file` (string) - Path to a file that will be used for the user
     data when launching the instance.
 
+-   `vault_aws_engine` (object) - Get credentials from Hashicorp Vault's aws
+    secrets engine. You must already have created a role to use. For more
+    information about generating credentials via the Vault engine, see the
+    [Vault
+    docs.](https://www.vaultproject.io/api/secret/aws/index.html#generate-credentials)
+    If you set this flag, you must also set the below options:
+    -   `name` (string) - Required. Specifies the name of the role to generate
+        credentials against. This is part of the request URL.
+    -   `engine_name` (string) - The name of the aws secrets engine. In the
+        Vault docs, this is normally referred to as "aws", and Packer will
+        default to "aws" if `engine_name` is not set.
+    -   `role_arn` (string)- The ARN of the role to assume if credential\_type
+        on the Vault role is assumed\_role. Must match one of the allowed role
+        ARNs in the Vault role. Optional if the Vault role only allows a single
+        AWS role ARN; required otherwise.
+    -   `ttl` (string) - Specifies the TTL for the use of the STS token. This
+        is specified as a string with a duration suffix. Valid only when
+        credential\_type is assumed\_role or federation\_token. When not
+        specified, the default\_sts\_ttl set for the role will be used. If that
+        is also not set, then the default value of 3600s will be used. AWS
+        places limits on the maximum TTL allowed. See the AWS documentation on
+        the DurationSeconds parameter for AssumeRole (for assumed\_role
+        credential types) and GetFederationToken (for federation\_token
+        credential types) for more details.
+
+    ``` json
+    {
+        "vault_aws_engine": {
+            "name": "myrole",
+            "role_arn": "myarn",
+            "ttl": "3600s"
+        }
+    }
+    ```
+
 -   `vpc_id` (string) - If launching into a VPC subnet, Packer needs the VPC ID
     in order to create a temporary security group within the VPC. Requires
     `subnet_id` to be set. If this field is left blank, Packer will try to get

website/source/docs/builders/amazon-ebssurrogate.html.md

@@ -497,6 +497,33 @@ builder.
 -   `user_data_file` (string) - Path to a file that will be used for the user
     data when launching the instance.
 
+-   `vault_aws_engine` (object) - Get credentials from Hashicorp Vault's aws
+    secrets engine. You must already have created a role to use. For more
+    information about generating credentials via the Vault engine, see the
+    [Vault
+    docs.](https://www.vaultproject.io/api/secret/aws/index.html#generate-credentials)
+    If you set this flag, you must also set the below options:
+-   `name` (string) - Required. Specifies the name of the role to generate
+    credentials against. This is part of the request URL.
+-   `engine_name` (string) - The name of the aws secrets engine. In the Vault
+    docs, this is normally referred to as "aws", and Packer will default to
+    "aws" if `engine_name` is not set.
+-   `role_arn` (string)- The ARN of the role to assume if credential\_type on
+    the Vault role is assumed\_role. Must match one of the allowed role ARNs in
+    the Vault role. Optional if the Vault role only allows a single AWS role
+    ARN; required otherwise.
+-   `ttl` (string) - Specifies the TTL for the use of the STS token. This is
+    specified as a string with a duration suffix. Valid only when
+    credential\_type is assumed\_role or federation\_token. When not specified,
+    the default\_sts\_ttl set for the role will be used. If that is also not
+    set, then the default value of 3600s will be used. AWS places limits on the
+    maximum TTL allowed. See the AWS documentation on the DurationSeconds
+    parameter for AssumeRole (for assumed\_role credential types) and
+    GetFederationToken (for federation\_token credential types) for more
+    details.
+
+    Example:
+    `json     {   "vault_aws_engine": {       "name": "myrole",       "role_arn": "myarn",       "ttl": "3600s"   }     }`
 -   `vpc_id` (string) - If launching into a VPC subnet, Packer needs the VPC ID
     in order to create a temporary security group within the VPC. Requires
     `subnet_id` to be set. If this field is left blank, Packer will try to get

website/source/docs/builders/amazon-ebsvolume.html.md

@@ -407,6 +407,43 @@ builder.
 -   `user_data_file` (string) - Path to a file that will be used for the user
     data when launching the instance.
 
+-   `vault_aws_engine` (object) - Get credentials from Hashicorp Vault's aws
+    secrets engine. You must already have created a role to use. For more
+    information about generating credentials via the Vault engine, see the
+    [Vault docs.]
+    (https://www.vaultproject.io/api/secret/aws/index.html#generate-credentials)
+    If you set this
+    flag, you must also set the below options:
+  -   `name` (string) - Required. Specifies the name of the role to generate
+      credentials against. This is part of the request URL.
+  -   `engine_name` (string) - The name of the aws secrets engine. In the Vault
+      docs, this is normally referred to as "aws", and Packer will default to
+      "aws" if `engine_name` is not set.
+  -   `role_arn` (string)- The ARN of the role to assume if credential_type on
+      the Vault role is assumed_role. Must match one of the allowed role ARNs
+      in the Vault role. Optional if the Vault role only allows a single AWS
+      role ARN; required otherwise.
+  -   `ttl` (string) -  Specifies the TTL for the use of the STS token. This is
+      specified as a string with a duration suffix. Valid only when
+      credential_type is assumed_role or federation_token. When not specified,
+      the default_sts_ttl set for the role will be used. If that is also not
+      set, then the default value of 3600s will be used. AWS places limits on
+      the maximum TTL allowed. See the AWS documentation on the DurationSeconds
+      parameter for AssumeRole (for assumed_role credential types) and
+      GetFederationToken (for federation_token credential types) for more
+      details.
+
+      Example:
+    ``` json
+    {
+        "vault_aws_engine": {
+            "name": "myrole",
+            "role_arn": "myarn",
+            "ttl": "3600s"
+        }
+    }
+    ```
+
 -   `vpc_id` (string) - If launching into a VPC subnet, Packer needs the VPC ID
     in order to create a temporary security group within the VPC. Requires
     `subnet_id` to be set. If this field is left blank, Packer will try to get

website/source/docs/builders/amazon-instance.html.md

@@ -489,6 +489,34 @@ builder.
 -   `user_data_file` (string) - Path to a file that will be used for the user
     data when launching the instance.
 
+-   `vault_aws_engine` (object) - Get credentials from Hashicorp Vault's aws
+    secrets engine. You must already have created a role to use. For more
+    information about generating credentials via the Vault engine, see the
+    [Vault
+    docs.](https://www.vaultproject.io/api/secret/aws/index.html#generate-credentials)
+    If you set this flag, you must also set the below options:
+    -   `name` (string) - Required. Specifies the name of the role to generate
+        credentials against. This is part of the request URL.
+    -   `engine_name` (string) - The name of the aws secrets engine. In the
+        Vault docs, this is normally referred to as "aws", and Packer will
+        default to "aws" if `engine_name` is not set.
+    -   `role_arn` (string)- The ARN of the role to assume if credential\_type
+        on the Vault role is assumed\_role. Must match one of the allowed role
+        ARNs in the Vault role. Optional if the Vault role only allows a single
+        AWS role ARN; required otherwise.
+    -   `ttl` (string) - Specifies the TTL for the use of the STS token. This
+        is specified as a string with a duration suffix. Valid only when
+        credential\_type is assumed\_role or federation\_token. When not
+        specified, the default\_sts\_ttl set for the role will be used. If that
+        is also not set, then the default value of 3600s will be used. AWS
+        places limits on the maximum TTL allowed. See the AWS documentation on
+        the DurationSeconds parameter for AssumeRole (for assumed\_role
+        credential types) and GetFederationToken (for federation\_token
+        credential types) for more details.
+
+        Example:
+        `json   {   "vault_aws_engine": {       "name": "myrole",       "role_arn": "myarn",       "ttl": "3600s"   }   }`
+
 -   `vpc_id` (string) - If launching into a VPC subnet, Packer needs the VPC ID
     in order to create a temporary security group within the VPC. Requires
     `subnet_id` to be set. If this field is left blank, Packer will try to get