From 980841b6c005e4666e6e8cae97cb63a7a4026a0c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jul 2013 15:56:28 +0900 Subject: [PATCH 01/36] builder/amazon/instance: boilerplate --- builder/amazon/instance/builder.go | 37 +++++++++++++++++++++++++ builder/amazon/instance/builder_test.go | 15 ++++++++++ 2 files changed, 52 insertions(+) create mode 100644 builder/amazon/instance/builder.go create mode 100644 builder/amazon/instance/builder_test.go diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go new file mode 100644 index 000000000..ae717ef4e --- /dev/null +++ b/builder/amazon/instance/builder.go @@ -0,0 +1,37 @@ +// The instance package contains a packer.Builder implementation that builds +// AMIs for Amazon EC2 backed by instance storage, as opposed to EBS storage. +package instance + +import ( + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" +) + +// The unique ID for this builder +const BuilderId = "mitchellh.amazon.instance" + +// Config is the configuration that is chained through the steps and +// settable from the template. +type Config struct { +} + +type Builder struct { + config Config + runner multistep.Runner +} + +func (b *Builder) Prepare(raws ...interface{}) error { + return nil +} + +func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { + return nil, nil +} + +func (b *Builder) Cancel() { + if b.runner != nil { + log.Println("Cancelling the step runner...") + b.runner.Cancel() + } +} diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go new file mode 100644 index 000000000..ac214051f --- /dev/null +++ b/builder/amazon/instance/builder_test.go @@ -0,0 +1,15 @@ +package instance + +import ( + "github.com/mitchellh/packer/packer" + "testing" +) + +func TestBuilder_ImplementsBuilder(t *testing.T) { + var raw interface{} + raw = &Builder{} + if _, ok := raw.(packer.Builder); !ok { + t.Fatalf("Builder should be a builder") + } +} + From 5aced3f339e5b0e0b4e695ae9b90a9564b720117 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Jul 2013 13:08:19 +0900 Subject: [PATCH 02/36] builder/amazon/common: AccessConfig for standard access config --- builder/amazon/common/access_config.go | 21 +++++++++++++++++++++ builder/amazon/ebs/builder.go | 7 +++---- builder/amazon/instance/builder_test.go | 1 - 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 builder/amazon/common/access_config.go diff --git a/builder/amazon/common/access_config.go b/builder/amazon/common/access_config.go new file mode 100644 index 000000000..c3819875a --- /dev/null +++ b/builder/amazon/common/access_config.go @@ -0,0 +1,21 @@ +package common + +import ( + "github.com/mitchellh/goamz/aws" +) + +// AccessConfig is for common configuration related to AWS access +type AccessConfig struct { + AccessKey string `mapstructure:"access_key"` + SecretKey string `mapstructure:"secret_key"` +} + +// Auth returns a valid aws.Auth object for access to AWS services, or +// an error if the authentication couldn't be resolved. +func (c *AccessConfig) Auth() (aws.Auth, error) { + return aws.GetAuth(c.AccessKey, c.SecretKey) +} + +func (c *AccessConfig) Validate() []error { + return nil +} diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 6063f995a..6b39a37ca 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -11,6 +11,7 @@ import ( "github.com/mitchellh/goamz/aws" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" "github.com/mitchellh/packer/builder/common" "github.com/mitchellh/packer/packer" "log" @@ -22,9 +23,7 @@ import ( const BuilderId = "mitchellh.amazonebs" type config struct { - // Access information - AccessKey string `mapstructure:"access_key"` - SecretKey string `mapstructure:"secret_key"` + awscommon.AccessConfig `mapstructure:",squash"` // Information for the source instance Region string @@ -123,7 +122,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe panic("region not found") } - auth, err := aws.GetAuth(b.config.AccessKey, b.config.SecretKey) + auth, err := b.config.AccessConfig.Auth() if err != nil { return nil, err } diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go index ac214051f..76c72f772 100644 --- a/builder/amazon/instance/builder_test.go +++ b/builder/amazon/instance/builder_test.go @@ -12,4 +12,3 @@ func TestBuilder_ImplementsBuilder(t *testing.T) { t.Fatalf("Builder should be a builder") } } - From e5e306049c11938612cfe10a23e349058571caf1 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Jul 2013 13:23:40 +0900 Subject: [PATCH 03/36] builder/amazon/common: RunConfig for launch info --- builder/amazon/common/access_config.go | 2 +- builder/amazon/common/run_config.go | 68 ++++++++++++++++++++++++++ builder/amazon/ebs/builder.go | 55 ++------------------- 3 files changed, 72 insertions(+), 53 deletions(-) create mode 100644 builder/amazon/common/run_config.go diff --git a/builder/amazon/common/access_config.go b/builder/amazon/common/access_config.go index c3819875a..736d2d190 100644 --- a/builder/amazon/common/access_config.go +++ b/builder/amazon/common/access_config.go @@ -16,6 +16,6 @@ func (c *AccessConfig) Auth() (aws.Auth, error) { return aws.GetAuth(c.AccessKey, c.SecretKey) } -func (c *AccessConfig) Validate() []error { +func (c *AccessConfig) Prepare() []error { return nil } diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go new file mode 100644 index 000000000..bd2186d93 --- /dev/null +++ b/builder/amazon/common/run_config.go @@ -0,0 +1,68 @@ +package common + +import ( + "errors" + "fmt" + "github.com/mitchellh/goamz/aws" + "time" +) + +// RunConfig contains configuration for running an instance from a source +// AMI and details on how to access that launched image. +type RunConfig struct { + Region string + SourceAmi string `mapstructure:"source_ami"` + InstanceType string `mapstructure:"instance_type"` + RawSSHTimeout string `mapstructure:"ssh_timeout"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort int `mapstructure:"ssh_port"` + SecurityGroupId string `mapstructure:"security_group_id"` + SubnetId string `mapstructure:"subnet_id"` + VpcId string `mapstructure:"vpc_id"` + + // Unexported fields that are calculated from others + sshTimeout time.Duration +} + +func (c *RunConfig) Prepare() []error { + // Defaults + if c.SSHPort == 0 { + c.SSHPort = 22 + } + + if c.RawSSHTimeout == "" { + c.RawSSHTimeout = "1m" + } + + // Validation + var err error + errs := make([]error, 0) + if c.SourceAmi == "" { + errs = append(errs, errors.New("A source_ami must be specified")) + } + + if c.InstanceType == "" { + errs = append(errs, errors.New("An instance_type must be specified")) + } + + if c.Region == "" { + errs = append(errs, errors.New("A region must be specified")) + } else if _, ok := aws.Regions[c.Region]; !ok { + errs = append(errs, fmt.Errorf("Unknown region: %s", c.Region)) + } + + if c.SSHUsername == "" { + errs = append(errs, errors.New("An ssh_username must be specified")) + } + + c.sshTimeout, err = time.ParseDuration(c.RawSSHTimeout) + if err != nil { + errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) + } + + return errs +} + +func (c *RunConfig) SSHTimeout() time.Duration { + return c.sshTimeout +} diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 6b39a37ca..2ed179cc2 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -16,7 +16,6 @@ import ( "github.com/mitchellh/packer/packer" "log" "text/template" - "time" ) // The unique ID for this builder @@ -24,25 +23,14 @@ const BuilderId = "mitchellh.amazonebs" type config struct { awscommon.AccessConfig `mapstructure:",squash"` - - // Information for the source instance - Region string - SourceAmi string `mapstructure:"source_ami"` - InstanceType string `mapstructure:"instance_type"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort int `mapstructure:"ssh_port"` - SecurityGroupId string `mapstructure:"security_group_id"` VpcId string `mapstructure:"vpc_id"` SubnetId string `mapstructure:"subnet_id"` + awscommon.RunConfig `mapstructure:",squash"` // Configuration of the resulting AMI AMIName string `mapstructure:"ami_name"` - PackerDebug bool `mapstructure:"packer_debug"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - - // Unexported fields that are calculated from others - sshTimeout time.Duration + PackerDebug bool `mapstructure:"packer_debug"` } type Builder struct { @@ -59,44 +47,7 @@ func (b *Builder) Prepare(raws ...interface{}) error { // Accumulate any errors errs := common.CheckUnusedConfig(md) - if b.config.SSHPort == 0 { - b.config.SSHPort = 22 - } - - if b.config.RawSSHTimeout == "" { - b.config.RawSSHTimeout = "1m" - } - // Accumulate any errors - if b.config.SourceAmi == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("A source_ami must be specified")) - } - - if b.config.InstanceType == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("An instance_type must be specified")) - } - - if b.config.Region == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("A region must be specified")) - } else if _, ok := aws.Regions[b.config.Region]; !ok { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Unknown region: %s", b.config.Region)) - } - - if b.config.SSHUsername == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("An ssh_username must be specified")) - } - - b.config.sshTimeout, err = time.ParseDuration(b.config.RawSSHTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - if b.config.AMIName == "" { errs = packer.MultiErrorAppend( errs, errors.New("ami_name must be specified")) @@ -144,7 +95,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &common.StepConnectSSH{ SSHAddress: sshAddress, SSHConfig: sshConfig, - SSHWaitTimeout: b.config.sshTimeout, + SSHWaitTimeout: b.config.SSHTimeout(), }, &common.StepProvision{}, &stepStopInstance{}, From 4878cec0e4dd898f42f7c79d536f2df912c897e3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Jul 2013 13:24:18 +0900 Subject: [PATCH 04/36] builder/amazon/ebs: validate access config --- builder/amazon/ebs/builder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 2ed179cc2..61df25b15 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -46,6 +46,8 @@ func (b *Builder) Prepare(raws ...interface{}) error { // Accumulate any errors errs := common.CheckUnusedConfig(md) + errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) + errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare()...) // Accumulate any errors if b.config.AMIName == "" { From 56c3cbfb2a1363c708323ec75b7da690ab29c7e7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Jul 2013 13:28:49 +0900 Subject: [PATCH 05/36] builder/common: common config struct for Packer configs --- builder/amazon/ebs/builder.go | 3 +-- builder/common/packer_config.go | 10 ++++++++++ builder/digitalocean/builder.go | 4 ++-- builder/virtualbox/builder.go | 6 ++---- builder/vmware/builder.go | 6 ++---- 5 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 builder/common/packer_config.go diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 61df25b15..ab241e145 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -22,6 +22,7 @@ import ( const BuilderId = "mitchellh.amazonebs" type config struct { + common.PackerConfig `mapstructure:",squash"` awscommon.AccessConfig `mapstructure:",squash"` VpcId string `mapstructure:"vpc_id"` SubnetId string `mapstructure:"subnet_id"` @@ -29,8 +30,6 @@ type config struct { // Configuration of the resulting AMI AMIName string `mapstructure:"ami_name"` - - PackerDebug bool `mapstructure:"packer_debug"` } type Builder struct { diff --git a/builder/common/packer_config.go b/builder/common/packer_config.go new file mode 100644 index 000000000..f51b92e0b --- /dev/null +++ b/builder/common/packer_config.go @@ -0,0 +1,10 @@ +package common + +// PackerConfig is a struct that contains the configuration keys that +// are sent by packer, properly tagged already so mapstructure can load +// them. Embed this structure into your configuration class to get it. +type PackerConfig struct { + PackerBuildName string `mapstructure:"packer_build_name"` + PackerDebug bool `mapstructure:"packer_debug"` + PackerForce bool `mapstructure:"packer_force"` +} diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index ae79f7c24..c4ab8c5bc 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -28,6 +28,8 @@ type snapshotNameData struct { // to use while communicating with DO and describes the image // you are creating type config struct { + common.PackerConfig `mapstructure:",squash"` + ClientID string `mapstructure:"client_id"` APIKey string `mapstructure:"api_key"` RegionID uint `mapstructure:"region_id"` @@ -38,8 +40,6 @@ type config struct { SSHUsername string `mapstructure:"ssh_username"` SSHPort uint `mapstructure:"ssh_port"` - PackerDebug bool `mapstructure:"packer_debug"` - RawSnapshotName string `mapstructure:"snapshot_name"` RawSSHTimeout string `mapstructure:"ssh_timeout"` RawEventDelay string `mapstructure:"event_delay"` diff --git a/builder/virtualbox/builder.go b/builder/virtualbox/builder.go index af6ffd35a..25a2855f8 100644 --- a/builder/virtualbox/builder.go +++ b/builder/virtualbox/builder.go @@ -24,6 +24,8 @@ type Builder struct { } type config struct { + common.PackerConfig `mapstructure:",squash"` + BootCommand []string `mapstructure:"boot_command"` DiskSize uint `mapstructure:"disk_size"` FloppyFiles []string `mapstructure:"floppy_files"` @@ -49,10 +51,6 @@ type config struct { VBoxManage [][]string `mapstructure:"vboxmanage"` VMName string `mapstructure:"vm_name"` - PackerBuildName string `mapstructure:"packer_build_name"` - PackerDebug bool `mapstructure:"packer_debug"` - PackerForce bool `mapstructure:"packer_force"` - RawBootWait string `mapstructure:"boot_wait"` RawShutdownTimeout string `mapstructure:"shutdown_timeout"` RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` diff --git a/builder/vmware/builder.go b/builder/vmware/builder.go index d2e2ba3ed..48d19a3cf 100644 --- a/builder/vmware/builder.go +++ b/builder/vmware/builder.go @@ -25,6 +25,8 @@ type Builder struct { } type config struct { + common.PackerConfig `mapstructure:",squash"` + DiskName string `mapstructure:"vmdk_name"` DiskSize uint `mapstructure:"disk_size"` FloppyFiles []string `mapstructure:"floppy_files"` @@ -50,10 +52,6 @@ type config struct { VNCPortMin uint `mapstructure:"vnc_port_min"` VNCPortMax uint `mapstructure:"vnc_port_max"` - PackerBuildName string `mapstructure:"packer_build_name"` - PackerDebug bool `mapstructure:"packer_debug"` - PackerForce bool `mapstructure:"packer_force"` - RawBootWait string `mapstructure:"boot_wait"` RawShutdownTimeout string `mapstructure:"shutdown_timeout"` RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` From 60ed71b2ff4c09221a267c9fab621fd47e372a87 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Jul 2013 14:23:45 +0900 Subject: [PATCH 06/36] builder/amazon/common: tests for runconfig --- builder/amazon/common/run_config_test.go | 98 ++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 builder/amazon/common/run_config_test.go diff --git a/builder/amazon/common/run_config_test.go b/builder/amazon/common/run_config_test.go new file mode 100644 index 000000000..e64aba275 --- /dev/null +++ b/builder/amazon/common/run_config_test.go @@ -0,0 +1,98 @@ +package common + +import ( + "testing" +) + +func testConfig() *RunConfig { + return &RunConfig{ + Region: "us-east-1", + SourceAmi: "abcd", + InstanceType: "m1.small", + SSHUsername: "root", + } +} + +func TestRunConfigPrepare(t *testing.T) { + c := testConfig() + err := c.Prepare() + if len(err) > 0 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_InstanceType(t *testing.T) { + c := testConfig() + c.InstanceType = "" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_Region(t *testing.T) { + c := testConfig() + c.Region = "" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } + + c.Region = "us-east-12" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } + + c.Region = "us-east-1" + if err := c.Prepare(); len(err) != 0 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SourceAmi(t *testing.T) { + c := testConfig() + c.SourceAmi = "" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SSHPort(t *testing.T) { + c := testConfig() + c.SSHPort = 0 + if err := c.Prepare(); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.SSHPort != 22 { + t.Fatalf("invalid value: %d", c.SSHPort) + } + + c.SSHPort = 44 + if err := c.Prepare(); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.SSHPort != 44 { + t.Fatalf("invalid value: %d", c.SSHPort) + } +} + +func TestRunConfigPrepare_SSHTimeout(t *testing.T) { + c := testConfig() + c.RawSSHTimeout = "" + if err := c.Prepare(); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + c.RawSSHTimeout = "bad" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SSHUsername(t *testing.T) { + c := testConfig() + c.SSHUsername = "" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} From 1750b34f707edc4af8bcfb19f7d3d32e5ed74d1f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Jul 2013 15:33:52 +0900 Subject: [PATCH 07/36] builder/amazon/*: clean up tests --- builder/amazon/common/run_config_test.go | 10 ++ builder/amazon/ebs/builder_test.go | 160 ----------------------- 2 files changed, 10 insertions(+), 160 deletions(-) diff --git a/builder/amazon/common/run_config_test.go b/builder/amazon/common/run_config_test.go index e64aba275..de3840ad5 100644 --- a/builder/amazon/common/run_config_test.go +++ b/builder/amazon/common/run_config_test.go @@ -1,9 +1,19 @@ package common import ( + "os" "testing" ) +func init() { + // Clear out the AWS access key env vars so they don't + // affect our tests. + os.Setenv("AWS_ACCESS_KEY_ID", "") + os.Setenv("AWS_ACCESS_KEY", "") + os.Setenv("AWS_SECRET_ACCESS_KEY", "") + os.Setenv("AWS_SECRET_KEY", "") +} + func testConfig() *RunConfig { return &RunConfig{ Region: "us-east-1", diff --git a/builder/amazon/ebs/builder_test.go b/builder/amazon/ebs/builder_test.go index ea0d326b5..3ad6d17b6 100644 --- a/builder/amazon/ebs/builder_test.go +++ b/builder/amazon/ebs/builder_test.go @@ -2,19 +2,9 @@ package ebs import ( "github.com/mitchellh/packer/packer" - "os" "testing" ) -func init() { - // Clear out the AWS access key env vars so they don't - // affect our tests. - os.Setenv("AWS_ACCESS_KEY_ID", "") - os.Setenv("AWS_ACCESS_KEY", "") - os.Setenv("AWS_SECRET_ACCESS_KEY", "") - os.Setenv("AWS_SECRET_KEY", "") -} - func testConfig() map[string]interface{} { return map[string]interface{}{ "access_key": "foo", @@ -75,30 +65,6 @@ func TestBuilderPrepare_AMIName(t *testing.T) { } } -func TestBuilderPrepare_InstanceType(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["instance_type"] = "foo" - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.InstanceType != "foo" { - t.Errorf("invalid: %s", b.config.InstanceType) - } - - // Test bad - delete(config, "instance_type") - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } -} - func TestBuilderPrepare_InvalidKey(t *testing.T) { var b Builder config := testConfig() @@ -110,129 +76,3 @@ func TestBuilderPrepare_InvalidKey(t *testing.T) { t.Fatal("should have error") } } - -func TestBuilderPrepare_Region(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["region"] = "us-east-1" - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.Region != "us-east-1" { - t.Errorf("invalid: %s", b.config.Region) - } - - // Test bad - delete(config, "region") - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } - - // Test invalid - config["region"] = "i-am-not-real" - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } -} - -func TestBuilderPrepare_SourceAmi(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["source_ami"] = "foo" - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.SourceAmi != "foo" { - t.Errorf("invalid: %s", b.config.SourceAmi) - } - - // Test bad - delete(config, "source_ami") - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } -} - -func TestBuilderPrepare_SSHPort(t *testing.T) { - var b Builder - config := testConfig() - - // Test default - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.SSHPort != 22 { - t.Errorf("invalid: %d", b.config.SSHPort) - } - - // Test set - config["ssh_port"] = 35 - b = Builder{} - err = b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.SSHPort != 35 { - t.Errorf("invalid: %d", b.config.SSHPort) - } -} - -func TestBuilderPrepare_SSHTimeout(t *testing.T) { - var b Builder - config := testConfig() - - // Test with a bad value - config["ssh_timeout"] = "this is not good" - err := b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } - - // Test with a good one - config["ssh_timeout"] = "5s" - err = b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } -} - -func TestBuilderPrepare_SSHUsername(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["ssh_username"] = "foo" - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.SSHUsername != "foo" { - t.Errorf("invalid: %s", b.config.SSHUsername) - } - - // Test bad - delete(config, "ssh_username") - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } -} From b3edb2fba51e1b4f8f3515653fd959c0f4fae7b6 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Jul 2013 15:34:07 +0900 Subject: [PATCH 08/36] fmt --- builder/amazon/common/run_config_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/run_config_test.go b/builder/amazon/common/run_config_test.go index de3840ad5..0c0d40680 100644 --- a/builder/amazon/common/run_config_test.go +++ b/builder/amazon/common/run_config_test.go @@ -16,10 +16,10 @@ func init() { func testConfig() *RunConfig { return &RunConfig{ - Region: "us-east-1", - SourceAmi: "abcd", + Region: "us-east-1", + SourceAmi: "abcd", InstanceType: "m1.small", - SSHUsername: "root", + SSHUsername: "root", } } From 45096d0768a290cad921b527e35868a631e19a68 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 19:40:45 -0700 Subject: [PATCH 09/36] builder/amazon: extract StepKeyPair for both --- .../step_key_pair.go} | 12 ++-- builder/amazon/ebs/builder.go | 2 +- builder/amazon/instance/builder.go | 68 +++++++++++++++++++ builder/amazon/instance/builder_test.go | 16 +++++ 4 files changed, 90 insertions(+), 8 deletions(-) rename builder/amazon/{ebs/step_keypair.go => common/step_key_pair.go} (81%) diff --git a/builder/amazon/ebs/step_keypair.go b/builder/amazon/common/step_key_pair.go similarity index 81% rename from builder/amazon/ebs/step_keypair.go rename to builder/amazon/common/step_key_pair.go index 498dcb65c..3e49e3d32 100644 --- a/builder/amazon/ebs/step_keypair.go +++ b/builder/amazon/common/step_key_pair.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "cgl.tideland.biz/identifier" @@ -10,11 +10,11 @@ import ( "log" ) -type stepKeyPair struct { +type StepKeyPair struct { keyName string } -func (s *stepKeyPair) Run(state map[string]interface{}) multistep.StepAction { +func (s *StepKeyPair) Run(state map[string]interface{}) multistep.StepAction { ec2conn := state["ec2"].(*ec2.EC2) ui := state["ui"].(packer.Ui) @@ -23,9 +23,7 @@ func (s *stepKeyPair) Run(state map[string]interface{}) multistep.StepAction { log.Printf("temporary keypair name: %s", keyName) keyResp, err := ec2conn.CreateKeyPair(keyName) if err != nil { - err := fmt.Errorf("Error creating temporary keypair: %s", err) - state["error"] = err - ui.Error(err.Error()) + state["error"] = fmt.Errorf("Error creating temporary keypair: %s", err) return multistep.ActionHalt } @@ -39,7 +37,7 @@ func (s *stepKeyPair) Run(state map[string]interface{}) multistep.StepAction { return multistep.ActionContinue } -func (s *stepKeyPair) Cleanup(state map[string]interface{}) { +func (s *StepKeyPair) Cleanup(state map[string]interface{}) { // If no key name is set, then we never created it, so just return if s.keyName == "" { return diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index ab241e145..81e839b9f 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -90,7 +90,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ - &stepKeyPair{}, + &awscommon.StepKeyPair{}, &stepSecurityGroup{}, &stepRunSourceInstance{}, &common.StepConnectSSH{ diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index ae717ef4e..8aa17633d 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -3,7 +3,11 @@ package instance import ( + "github.com/mitchellh/goamz/aws" + "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" + "github.com/mitchellh/packer/builder/common" "github.com/mitchellh/packer/packer" "log" ) @@ -14,6 +18,9 @@ const BuilderId = "mitchellh.amazon.instance" // Config is the configuration that is chained through the steps and // settable from the template. type Config struct { + common.PackerConfig `mapstructure:",squash"` + awscommon.AccessConfig `mapstructure:",squash"` + awscommon.RunConfig `mapstructure:",squash"` } type Builder struct { @@ -22,10 +29,71 @@ type Builder struct { } func (b *Builder) Prepare(raws ...interface{}) error { + md, err := common.DecodeConfig(&b.config, raws...) + if err != nil { + return err + } + + // Accumulate any errors + errs := common.CheckUnusedConfig(md) + errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) + errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare()...) + + if errs != nil && len(errs.Errors) > 0 { + return errs + } + + log.Printf("Config: %+v", b.config) return nil } func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { + region, ok := aws.Regions[b.config.Region] + if !ok { + panic("region not found") + } + + auth, err := b.config.AccessConfig.Auth() + if err != nil { + return nil, err + } + + ec2conn := ec2.New(auth, region) + + // Setup the state bag and initial state for the steps + state := make(map[string]interface{}) + state["config"] = b.config + state["ec2"] = ec2conn + state["hook"] = hook + state["ui"] = ui + + // Build the steps + steps := []multistep.Step{ + &awscommon.StepKeyPair{}, + } + + // Run! + if b.config.PackerDebug { + b.runner = &multistep.DebugRunner{ + Steps: steps, + PauseFn: common.MultistepDebugFn(ui), + } + } else { + b.runner = &multistep.BasicRunner{Steps: steps} + } + + b.runner.Run(state) + + // If there was an error, return that + if rawErr, ok := state["error"]; ok { + return nil, rawErr.(error) + } + + // If there are no AMIs, then just return + if _, ok := state["amis"]; !ok { + return nil, nil + } + return nil, nil } diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go index 76c72f772..9e77886d8 100644 --- a/builder/amazon/instance/builder_test.go +++ b/builder/amazon/instance/builder_test.go @@ -5,6 +5,10 @@ import ( "testing" ) +func testConfig() map[string]interface{} { + return map[string]interface{}{} +} + func TestBuilder_ImplementsBuilder(t *testing.T) { var raw interface{} raw = &Builder{} @@ -12,3 +16,15 @@ func TestBuilder_ImplementsBuilder(t *testing.T) { t.Fatalf("Builder should be a builder") } } + +func TestBuilderPrepare_InvalidKey(t *testing.T) { + var b Builder + config := testConfig() + + // Add a random key + config["i_should_not_be_valid"] = true + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } +} From 4f568f4998debd96815939c138d4f5e67ad5cdfe Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 19:50:55 -0700 Subject: [PATCH 10/36] builder/amazon: extract StepSecurityGroup --- .../{ebs => common}/step_security_group.go | 34 ++++++++++--------- builder/amazon/ebs/builder.go | 5 ++- builder/amazon/instance/builder.go | 4 +++ 3 files changed, 26 insertions(+), 17 deletions(-) rename builder/amazon/{ebs => common}/step_security_group.go (71%) diff --git a/builder/amazon/ebs/step_security_group.go b/builder/amazon/common/step_security_group.go similarity index 71% rename from builder/amazon/ebs/step_security_group.go rename to builder/amazon/common/step_security_group.go index 4a056a1b3..5f3be3df9 100644 --- a/builder/amazon/ebs/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "cgl.tideland.biz/identifier" @@ -10,18 +10,20 @@ import ( "log" ) -type stepSecurityGroup struct { - groupId string +type StepSecurityGroup struct { + SecurityGroupId string + SSHPort int + + createdGroupId string } -func (s *stepSecurityGroup) Run(state map[string]interface{}) multistep.StepAction { - config := state["config"].(config) +func (s *StepSecurityGroup) Run(state map[string]interface{}) multistep.StepAction { ec2conn := state["ec2"].(*ec2.EC2) ui := state["ui"].(packer.Ui) - if config.SecurityGroupId != "" { - log.Printf("Using specified security group: %s", config.SecurityGroupId) - state["securityGroupId"] = config.SecurityGroupId + if s.SecurityGroupId != "" { + log.Printf("Using specified security group: %s", s.SecurityGroupId) + state["securityGroupId"] = s.SecurityGroupId return multistep.ActionContinue } @@ -36,14 +38,14 @@ func (s *stepSecurityGroup) Run(state map[string]interface{}) multistep.StepActi } // Set the group ID so we can delete it later - s.groupId = groupResp.Id + s.createdGroupId = groupResp.Id // Authorize the SSH access perms := []ec2.IPPerm{ ec2.IPPerm{ Protocol: "tcp", - FromPort: config.SSHPort, - ToPort: config.SSHPort, + FromPort: s.SSHPort, + ToPort: s.SSHPort, SourceIPs: []string{"0.0.0.0/0"}, }, } @@ -57,13 +59,13 @@ func (s *stepSecurityGroup) Run(state map[string]interface{}) multistep.StepActi } // Set some state data for use in future steps - state["securityGroupId"] = s.groupId + state["securityGroupId"] = s.createdGroupId return multistep.ActionContinue } -func (s *stepSecurityGroup) Cleanup(state map[string]interface{}) { - if s.groupId == "" { +func (s *StepSecurityGroup) Cleanup(state map[string]interface{}) { + if s.createdGroupId == "" { return } @@ -71,10 +73,10 @@ func (s *stepSecurityGroup) Cleanup(state map[string]interface{}) { ui := state["ui"].(packer.Ui) ui.Say("Deleting temporary security group...") - _, err := ec2conn.DeleteSecurityGroup(ec2.SecurityGroup{Id: s.groupId}) + _, err := ec2conn.DeleteSecurityGroup(ec2.SecurityGroup{Id: s.createdGroupId}) if err != nil { log.Printf("Error deleting security group: %s", err) ui.Error(fmt.Sprintf( - "Error cleaning up security group. Please delete the group manually: %s", s.groupId)) + "Error cleaning up security group. Please delete the group manually: %s", s.createdGroupId)) } } diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 81e839b9f..a0461efb6 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -91,7 +91,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ &awscommon.StepKeyPair{}, - &stepSecurityGroup{}, + &awscommon.StepSecurityGroup{ + SecurityGroupId: b.config.SecurityGroupId, + SSHPort: b.config.SSHPort, + }, &stepRunSourceInstance{}, &common.StepConnectSSH{ SSHAddress: sshAddress, diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 8aa17633d..66680903c 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -70,6 +70,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ &awscommon.StepKeyPair{}, + &awscommon.StepSecurityGroup{ + SecurityGroupId: b.config.SecurityGroupId, + SSHPort: b.config.SSHPort, + }, } // Run! From 30ab70388b7779c00625340440e689178013e3f2 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 19:51:25 -0700 Subject: [PATCH 11/36] builder/amazon/common: panic if SSHPot is 0 --- builder/amazon/common/step_security_group.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builder/amazon/common/step_security_group.go b/builder/amazon/common/step_security_group.go index 5f3be3df9..e5743f202 100644 --- a/builder/amazon/common/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -27,6 +27,10 @@ func (s *StepSecurityGroup) Run(state map[string]interface{}) multistep.StepActi return multistep.ActionContinue } + if s.SSHPort == 0 { + panic("SSHPort must be set to a non-zero value.") + } + // Create the group ui.Say("Creating temporary security group for this instance...") groupName := fmt.Sprintf("packer %s", hex.EncodeToString(identifier.NewUUID().Raw())) From 64ced7ff2c359d9543d7cec895b8de5fdb927496 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 19:58:27 -0700 Subject: [PATCH 12/36] builder/amazon: extract StepRunSourceInstance --- builder/amazon/{ebs => common}/instance.go | 4 +-- .../step_run_source_instance.go | 30 +++++++++++-------- builder/amazon/common/step_security_group.go | 2 +- builder/amazon/ebs/builder.go | 8 +++-- builder/amazon/ebs/step_stop_instance.go | 3 +- builder/amazon/instance/builder.go | 7 ++++- 6 files changed, 34 insertions(+), 20 deletions(-) rename builder/amazon/{ebs => common}/instance.go (89%) rename builder/amazon/{ebs => common}/step_run_source_instance.go (71%) diff --git a/builder/amazon/ebs/instance.go b/builder/amazon/common/instance.go similarity index 89% rename from builder/amazon/ebs/instance.go rename to builder/amazon/common/instance.go index d5a0a8006..53dab50ce 100644 --- a/builder/amazon/ebs/instance.go +++ b/builder/amazon/common/instance.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "fmt" @@ -7,7 +7,7 @@ import ( "time" ) -func waitForState(ec2conn *ec2.EC2, originalInstance *ec2.Instance, pending []string, target string) (i *ec2.Instance, err error) { +func WaitForState(ec2conn *ec2.EC2, originalInstance *ec2.Instance, pending []string, target string) (i *ec2.Instance, err error) { log.Printf("Waiting for instance state to become: %s", target) i = originalInstance diff --git a/builder/amazon/ebs/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go similarity index 71% rename from builder/amazon/ebs/step_run_source_instance.go rename to builder/amazon/common/step_run_source_instance.go index ada21de20..b7c7d4ab5 100644 --- a/builder/amazon/ebs/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "fmt" @@ -8,12 +8,15 @@ import ( "log" ) -type stepRunSourceInstance struct { +type StepRunSourceInstance struct { + ExpectedRootDevice string + InstanceType string + SourceAMI string + instance *ec2.Instance } -func (s *stepRunSourceInstance) Run(state map[string]interface{}) multistep.StepAction { - config := state["config"].(config) +func (s *StepRunSourceInstance) Run(state map[string]interface{}) multistep.StepAction { ec2conn := state["ec2"].(*ec2.EC2) keyName := state["keyPair"].(string) securityGroupId := state["securityGroupId"].(string) @@ -21,8 +24,8 @@ func (s *stepRunSourceInstance) Run(state map[string]interface{}) multistep.Step runOpts := &ec2.RunInstances{ KeyName: keyName, - ImageId: config.SourceAmi, - InstanceType: config.InstanceType, + ImageId: s.SourceAMI, + InstanceType: s.InstanceType, MinCount: 0, MaxCount: 0, SecurityGroups: []ec2.SecurityGroup{ec2.SecurityGroup{Id: securityGroupId}}, @@ -30,16 +33,17 @@ func (s *stepRunSourceInstance) Run(state map[string]interface{}) multistep.Step } ui.Say("Launching a source AWS instance...") - imageResp, err := ec2conn.Images([]string{config.SourceAmi}, ec2.NewFilter()) + imageResp, err := ec2conn.Images([]string{s.SourceAMI}, ec2.NewFilter()) if err != nil { state["error"] = fmt.Errorf("There was a problem with the source AMI: %s", err) return multistep.ActionHalt } - if imageResp.Images[0].RootDeviceType != "ebs" { + if s.ExpectedRootDevice != "" && imageResp.Images[0].RootDeviceType != s.ExpectedRootDevice { state["error"] = fmt.Errorf( - "The provided source AMI is instance-store based. The\n" + - "amazon-ebs bundler can only work with EBS based AMIs.") + "The provided source AMI has an invalid root device type.\n"+ + "Expected '%s', got '%s'.", + s.ExpectedRootDevice, imageResp.Images[0].RootDeviceType) return multistep.ActionHalt } @@ -55,7 +59,7 @@ func (s *stepRunSourceInstance) Run(state map[string]interface{}) multistep.Step log.Printf("instance id: %s", s.instance.InstanceId) ui.Say(fmt.Sprintf("Waiting for instance (%s) to become ready...", s.instance.InstanceId)) - s.instance, err = waitForState(ec2conn, s.instance, []string{"pending"}, "running") + s.instance, err = WaitForState(ec2conn, s.instance, []string{"pending"}, "running") if err != nil { err := fmt.Errorf("Error waiting for instance (%s) to become ready: %s", s.instance.InstanceId, err) state["error"] = err @@ -68,7 +72,7 @@ func (s *stepRunSourceInstance) Run(state map[string]interface{}) multistep.Step return multistep.ActionContinue } -func (s *stepRunSourceInstance) Cleanup(state map[string]interface{}) { +func (s *StepRunSourceInstance) Cleanup(state map[string]interface{}) { if s.instance == nil { return } @@ -83,5 +87,5 @@ func (s *stepRunSourceInstance) Cleanup(state map[string]interface{}) { } pending := []string{"pending", "running", "shutting-down", "stopped", "stopping"} - waitForState(ec2conn, s.instance, pending, "terminated") + WaitForState(ec2conn, s.instance, pending, "terminated") } diff --git a/builder/amazon/common/step_security_group.go b/builder/amazon/common/step_security_group.go index e5743f202..dec317262 100644 --- a/builder/amazon/common/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -12,7 +12,7 @@ import ( type StepSecurityGroup struct { SecurityGroupId string - SSHPort int + SSHPort int createdGroupId string } diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index a0461efb6..03c23bf64 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -93,9 +93,13 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepKeyPair{}, &awscommon.StepSecurityGroup{ SecurityGroupId: b.config.SecurityGroupId, - SSHPort: b.config.SSHPort, + SSHPort: b.config.SSHPort, + }, + &awscommon.StepRunSourceInstance{ + ExpectedRootDevice: "ebs", + InstanceType: b.config.InstanceType, + SourceAMI: b.config.SourceAmi, }, - &stepRunSourceInstance{}, &common.StepConnectSSH{ SSHAddress: sshAddress, SSHConfig: sshConfig, diff --git a/builder/amazon/ebs/step_stop_instance.go b/builder/amazon/ebs/step_stop_instance.go index ac81bda6b..5c608632f 100644 --- a/builder/amazon/ebs/step_stop_instance.go +++ b/builder/amazon/ebs/step_stop_instance.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" "github.com/mitchellh/packer/packer" ) @@ -26,7 +27,7 @@ func (s *stepStopInstance) Run(state map[string]interface{}) multistep.StepActio // Wait for the instance to actual stop ui.Say("Waiting for the instance to stop...") - instance, err = waitForState(ec2conn, instance, []string{"running", "stopping"}, "stopped") + instance, err = awscommon.WaitForState(ec2conn, instance, []string{"running", "stopping"}, "stopped") if err != nil { err := fmt.Errorf("Error waiting for instance to stop: %s", err) state["error"] = err diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 66680903c..908a23a53 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -72,7 +72,12 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepKeyPair{}, &awscommon.StepSecurityGroup{ SecurityGroupId: b.config.SecurityGroupId, - SSHPort: b.config.SSHPort, + SSHPort: b.config.SSHPort, + }, + &awscommon.StepRunSourceInstance{ + ExpectedRootDevice: "instance-store", + InstanceType: b.config.InstanceType, + SourceAMI: b.config.SourceAmi, }, } From 2f9840a4cf710615cd1ce7194d6a5b9f3c3eee76 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 20:03:00 -0700 Subject: [PATCH 13/36] builder/amazon: extract SSH connect funcs --- builder/amazon/common/ssh.go | 33 ++++++++++++++++++++++++++++ builder/amazon/ebs/builder.go | 6 ++--- builder/amazon/ebs/ssh.go | 35 ------------------------------ builder/amazon/instance/builder.go | 5 +++++ 4 files changed, 40 insertions(+), 39 deletions(-) create mode 100644 builder/amazon/common/ssh.go delete mode 100644 builder/amazon/ebs/ssh.go diff --git a/builder/amazon/common/ssh.go b/builder/amazon/common/ssh.go new file mode 100644 index 000000000..77e1d6c53 --- /dev/null +++ b/builder/amazon/common/ssh.go @@ -0,0 +1,33 @@ +package common + +import ( + gossh "code.google.com/p/go.crypto/ssh" + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/packer/communicator/ssh" +) + +func SSHAddress(port int) func(map[string]interface{}) (string, error) { + return func(state map[string]interface{}) (string, error) { + instance := state["instance"].(*ec2.Instance) + return fmt.Sprintf("%s:%d", instance.DNSName, port), nil + } +} + +func SSHConfig(username string) func(map[string]interface{}) (*gossh.ClientConfig, error) { + return func(state map[string]interface{}) (*gossh.ClientConfig, error) { + privateKey := state["privateKey"].(string) + + keyring := new(ssh.SimpleKeychain) + if err := keyring.AddPEMKey(privateKey); err != nil { + return nil, fmt.Errorf("Error setting up SSH config: %s", err) + } + + return &gossh.ClientConfig{ + User: username, + Auth: []gossh.ClientAuth{ + gossh.ClientAuthKeyring(keyring), + }, + }, nil + } +} diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 03c23bf64..48658f636 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -24,8 +24,6 @@ const BuilderId = "mitchellh.amazonebs" type config struct { common.PackerConfig `mapstructure:",squash"` awscommon.AccessConfig `mapstructure:",squash"` - VpcId string `mapstructure:"vpc_id"` - SubnetId string `mapstructure:"subnet_id"` awscommon.RunConfig `mapstructure:",squash"` // Configuration of the resulting AMI @@ -101,8 +99,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SourceAMI: b.config.SourceAmi, }, &common.StepConnectSSH{ - SSHAddress: sshAddress, - SSHConfig: sshConfig, + SSHAddress: awscommon.SSHAddress(b.config.SSHPort), + SSHConfig: awscommon.SSHConfig(b.config.SSHUsername), SSHWaitTimeout: b.config.SSHTimeout(), }, &common.StepProvision{}, diff --git a/builder/amazon/ebs/ssh.go b/builder/amazon/ebs/ssh.go deleted file mode 100644 index 599bf5ad9..000000000 --- a/builder/amazon/ebs/ssh.go +++ /dev/null @@ -1,35 +0,0 @@ -package ebs - -import ( - gossh "code.google.com/p/go.crypto/ssh" - "fmt" - "github.com/mitchellh/goamz/ec2" - "github.com/mitchellh/packer/communicator/ssh" -) - -func sshAddress(state map[string]interface{}) (string, error) { - config := state["config"].(config) - instance := state["instance"].(*ec2.Instance) - if config.VpcId == "" { - return fmt.Sprintf("%s:%d", instance.DNSName, config.SSHPort), nil - } else { - return fmt.Sprintf("%s:%d", instance.PrivateIpAddress, config.SSHPort), nil - } -} - -func sshConfig(state map[string]interface{}) (*gossh.ClientConfig, error) { - config := state["config"].(config) - privateKey := state["privateKey"].(string) - - keyring := new(ssh.SimpleKeychain) - if err := keyring.AddPEMKey(privateKey); err != nil { - return nil, fmt.Errorf("Error setting up SSH config: %s", err) - } - - return &gossh.ClientConfig{ - User: config.SSHUsername, - Auth: []gossh.ClientAuth{ - gossh.ClientAuthKeyring(keyring), - }, - }, nil -} diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 908a23a53..0606de521 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -79,6 +79,11 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe InstanceType: b.config.InstanceType, SourceAMI: b.config.SourceAmi, }, + &common.StepConnectSSH{ + SSHAddress: awscommon.SSHAddress(b.config.SSHPort), + SSHConfig: awscommon.SSHConfig(b.config.SSHUsername), + SSHWaitTimeout: b.config.SSHTimeout(), + }, } // Run! From 0a76d073632fc69819ebda8163048d8439dc5683 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 20:04:28 -0700 Subject: [PATCH 14/36] builder/amazon: provision on instance --- builder/amazon/common/ssh.go | 5 +++++ builder/amazon/instance/builder.go | 1 + 2 files changed, 6 insertions(+) diff --git a/builder/amazon/common/ssh.go b/builder/amazon/common/ssh.go index 77e1d6c53..ea589189f 100644 --- a/builder/amazon/common/ssh.go +++ b/builder/amazon/common/ssh.go @@ -7,6 +7,8 @@ import ( "github.com/mitchellh/packer/communicator/ssh" ) +// SSHAddress returns a function that can be given to the SSH communicator +// for determining the SSH address based on the instance DNS name. func SSHAddress(port int) func(map[string]interface{}) (string, error) { return func(state map[string]interface{}) (string, error) { instance := state["instance"].(*ec2.Instance) @@ -14,6 +16,9 @@ func SSHAddress(port int) func(map[string]interface{}) (string, error) { } } +// SSHConfig returns a function that can be used for the SSH communicator +// config for connecting to the instance created over SSH using the generated +// private key. func SSHConfig(username string) func(map[string]interface{}) (*gossh.ClientConfig, error) { return func(state map[string]interface{}) (*gossh.ClientConfig, error) { privateKey := state["privateKey"].(string) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 0606de521..6826c2c31 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -84,6 +84,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SSHConfig: awscommon.SSHConfig(b.config.SSHUsername), SSHWaitTimeout: b.config.SSHTimeout(), }, + &common.StepProvision{}, } // Run! From 992a9bfb224f7a071db8de5df7bd5680c1ad802f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 20:08:41 -0700 Subject: [PATCH 15/36] builder/amazon/common: extract Artifact --- builder/amazon/{ebs => common}/artifact.go | 36 ++++++++++--------- .../amazon/{ebs => common}/artifact_test.go | 8 ++--- builder/amazon/ebs/builder.go | 7 ++-- 3 files changed, 28 insertions(+), 23 deletions(-) rename builder/amazon/{ebs => common}/artifact.go (54%) rename builder/amazon/{ebs => common}/artifact_test.go (85%) diff --git a/builder/amazon/ebs/artifact.go b/builder/amazon/common/artifact.go similarity index 54% rename from builder/amazon/ebs/artifact.go rename to builder/amazon/common/artifact.go index 4d25d4595..af6d1f2d4 100644 --- a/builder/amazon/ebs/artifact.go +++ b/builder/amazon/common/artifact.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "fmt" @@ -8,35 +8,39 @@ import ( "strings" ) -type artifact struct { +// Artifact is an artifact implementation that contains built AMIs. +type Artifact struct { // A map of regions to AMI IDs. - amis map[string]string + Amis map[string]string + + // BuilderId is the unique ID for the builder that created this AMI + BuilderIdValue string // EC2 connection for performing API stuff. - conn *ec2.EC2 + Conn *ec2.EC2 } -func (*artifact) BuilderId() string { - return BuilderId +func (a *Artifact) BuilderId() string { + return a.BuilderIdValue } -func (*artifact) Files() []string { +func (*Artifact) Files() []string { // We have no files return nil } -func (a *artifact) Id() string { - parts := make([]string, 0, len(a.amis)) - for region, amiId := range a.amis { +func (a *Artifact) Id() string { + parts := make([]string, 0, len(a.Amis)) + for region, amiId := range a.Amis { parts = append(parts, fmt.Sprintf("%s:%s", region, amiId)) } return strings.Join(parts, ",") } -func (a *artifact) String() string { - amiStrings := make([]string, 0, len(a.amis)) - for region, id := range a.amis { +func (a *Artifact) String() string { + amiStrings := make([]string, 0, len(a.Amis)) + for region, id := range a.Amis { single := fmt.Sprintf("%s: %s", region, id) amiStrings = append(amiStrings, single) } @@ -44,12 +48,12 @@ func (a *artifact) String() string { return fmt.Sprintf("AMIs were created:\n\n%s", strings.Join(amiStrings, "\n")) } -func (a *artifact) Destroy() error { +func (a *Artifact) Destroy() error { errors := make([]error, 0) - for _, imageId := range a.amis { + for _, imageId := range a.Amis { log.Printf("Deregistering image ID: %s", imageId) - if _, err := a.conn.DeregisterImage(imageId); err != nil { + if _, err := a.Conn.DeregisterImage(imageId); err != nil { errors = append(errors, err) } diff --git a/builder/amazon/ebs/artifact_test.go b/builder/amazon/common/artifact_test.go similarity index 85% rename from builder/amazon/ebs/artifact_test.go rename to builder/amazon/common/artifact_test.go index dd8ccba9b..be8121074 100644 --- a/builder/amazon/ebs/artifact_test.go +++ b/builder/amazon/common/artifact_test.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "cgl.tideland.biz/asserts" @@ -10,7 +10,7 @@ func TestArtifact_Impl(t *testing.T) { assert := asserts.NewTestingAsserts(t, true) var actual packer.Artifact - assert.Implementor(&artifact{}, &actual, "should be an Artifact") + assert.Implementor(&Artifact{}, &actual, "should be an Artifact") } func TestArtifactId(t *testing.T) { @@ -22,7 +22,7 @@ func TestArtifactId(t *testing.T) { amis["east"] = "foo" amis["west"] = "bar" - a := &artifact{amis, nil} + a := &Artifact{amis, nil} result := a.Id() assert.Equal(result, expected, "should match output") } @@ -39,7 +39,7 @@ west: bar` amis["east"] = "foo" amis["west"] = "bar" - a := &artifact{amis, nil} + a := &Artifact{amis, nil} result := a.String() assert.Equal(result, expected, "should match output") } diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 48658f636..da11e68b9 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -131,9 +131,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } // Build the artifact and return it - artifact := &artifact{ - amis: state["amis"].(map[string]string), - conn: ec2conn, + artifact := &awscommon.Artifact{ + Amis: state["amis"].(map[string]string), + BuilderIdValue: BuilderId, + Conn: ec2conn, } return artifact, nil From e85a01d9da28eeb6aca5bfe9e0dd26d92cb0b7ac Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 20:08:54 -0700 Subject: [PATCH 16/36] fmt --- builder/amazon/ebs/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index da11e68b9..f258e7ab9 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -132,9 +132,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the artifact and return it artifact := &awscommon.Artifact{ - Amis: state["amis"].(map[string]string), + Amis: state["amis"].(map[string]string), BuilderIdValue: BuilderId, - Conn: ec2conn, + Conn: ec2conn, } return artifact, nil From 42f78804fe6e459ca92219c6b54a226b14e45eb0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 20:12:40 -0700 Subject: [PATCH 17/36] plugin/builder-amazon-instance: add the plugin --- config.go | 1 + plugin/builder-amazon-instance/main.go | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 plugin/builder-amazon-instance/main.go diff --git a/config.go b/config.go index 12a78d300..4660fad1c 100644 --- a/config.go +++ b/config.go @@ -20,6 +20,7 @@ const defaultConfig = ` "builders": { "amazon-ebs": "packer-builder-amazon-ebs", + "amazon-instance": "packer-builder-amazon-instance", "digitalocean": "packer-builder-digitalocean", "virtualbox": "packer-builder-virtualbox", "vmware": "packer-builder-vmware" diff --git a/plugin/builder-amazon-instance/main.go b/plugin/builder-amazon-instance/main.go new file mode 100644 index 000000000..fc1c66b40 --- /dev/null +++ b/plugin/builder-amazon-instance/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/mitchellh/packer/builder/amazon/instance" + "github.com/mitchellh/packer/packer/plugin" +) + +func main() { + plugin.ServeBuilder(new(instance.Builder)) +} From 39b3b18d8133ce246bf5b595fd3f99f0b1b773b5 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 21:00:12 -0700 Subject: [PATCH 18/36] builder/amazon/instance: upload x509 cert --- builder/amazon/instance/builder.go | 28 +++++- builder/amazon/instance/builder_test.go | 86 ++++++++++++++++++- .../amazon/instance/step_upload_x509_cert.go | 44 ++++++++++ 3 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 builder/amazon/instance/step_upload_x509_cert.go diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 6826c2c31..ba7224de0 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -3,6 +3,8 @@ package instance import ( + "errors" + "fmt" "github.com/mitchellh/goamz/aws" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" @@ -10,6 +12,7 @@ import ( "github.com/mitchellh/packer/builder/common" "github.com/mitchellh/packer/packer" "log" + "os" ) // The unique ID for this builder @@ -21,6 +24,10 @@ type Config struct { common.PackerConfig `mapstructure:",squash"` awscommon.AccessConfig `mapstructure:",squash"` awscommon.RunConfig `mapstructure:",squash"` + + X509CertPath string `mapstructure:"x509_cert_path"` + X509KeyPath string `mapstructure:"x509_key_path"` + X509UploadPath string `mapstructure:"x509_upload_path"` } type Builder struct { @@ -39,6 +46,24 @@ func (b *Builder) Prepare(raws ...interface{}) error { errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare()...) + if b.config.X509CertPath == "" { + errs = packer.MultiErrorAppend(errs, errors.New("x509_cert_path is required")) + } else if _, err := os.Stat(b.config.X509CertPath); err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("x509_cert_path points to bad file: %s", err)) + } + + if b.config.X509KeyPath == "" { + errs = packer.MultiErrorAppend(errs, errors.New("x509_key_path is required")) + } else if _, err := os.Stat(b.config.X509KeyPath); err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("x509_key_path points to bad file: %s", err)) + } + + if b.config.X509UploadPath == "" { + errs = packer.MultiErrorAppend(errs, errors.New("x509_upload_path is required")) + } + if errs != nil && len(errs.Errors) > 0 { return errs } @@ -62,7 +87,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Setup the state bag and initial state for the steps state := make(map[string]interface{}) - state["config"] = b.config + state["config"] = &b.config state["ec2"] = ec2conn state["hook"] = hook state["ui"] = ui @@ -85,6 +110,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SSHWaitTimeout: b.config.SSHTimeout(), }, &common.StepProvision{}, + &StepUploadX509Cert{}, } // Run! diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go index 9e77886d8..d83ee9c33 100644 --- a/builder/amazon/instance/builder_test.go +++ b/builder/amazon/instance/builder_test.go @@ -2,11 +2,26 @@ package instance import ( "github.com/mitchellh/packer/packer" + "io/ioutil" + "os" "testing" ) func testConfig() map[string]interface{} { - return map[string]interface{}{} + tf, err := ioutil.TempFile("", "packer") + if err != nil { + panic(err) + } + + return map[string]interface{}{ + "instance_type": "m1.small", + "region": "us-east-1", + "source_ami": "foo", + "ssh_username": "bob", + "x509_cert_path": tf.Name(), + "x509_key_path": tf.Name(), + "x509_upload_path": "/foo", + } } func TestBuilder_ImplementsBuilder(t *testing.T) { @@ -28,3 +43,72 @@ func TestBuilderPrepare_InvalidKey(t *testing.T) { t.Fatal("should have error") } } + +func TestBuilderPrepare_X509CertPath(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["x509_cert_path"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["x509_cert_path"] = "i/am/a/file/that/doesnt/exist" + err = b.Prepare(config) + if err == nil { + t.Error("should have error") + } + + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("error tempfile: %s", err) + } + defer os.Remove(tf.Name()) + + config["x509_cert_path"] = tf.Name() + err = b.Prepare(config) + if err != nil { + t.Fatalf("should not have error: %s", err) + } +} + +func TestBuilderPrepare_X509KeyPath(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["x509_key_path"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["x509_key_path"] = "i/am/a/file/that/doesnt/exist" + err = b.Prepare(config) + if err == nil { + t.Error("should have error") + } + + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("error tempfile: %s", err) + } + defer os.Remove(tf.Name()) + + config["x509_key_path"] = tf.Name() + err = b.Prepare(config) + if err != nil { + t.Fatalf("should not have error: %s", err) + } +} + +func TestBuilderPrepare_X509UploadPath(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["x509_upload_path"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } +} diff --git a/builder/amazon/instance/step_upload_x509_cert.go b/builder/amazon/instance/step_upload_x509_cert.go new file mode 100644 index 000000000..22f710e64 --- /dev/null +++ b/builder/amazon/instance/step_upload_x509_cert.go @@ -0,0 +1,44 @@ +package instance + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "os" +) + +type StepUploadX509Cert struct{} + +func (s *StepUploadX509Cert) Run(state map[string]interface{}) multistep.StepAction { + comm := state["communicator"].(packer.Communicator) + config := state["config"].(*Config) + ui := state["ui"].(packer.Ui) + + x509RemoteCertPath := config.X509UploadPath + "/cert.pem" + x509RemoteKeyPath := config.X509UploadPath + "/key.pem" + + ui.Say("Uploading X509 Certificate...") + if err := s.uploadSingle(comm, x509RemoteCertPath, config.X509CertPath); err != nil { + state["error"] = fmt.Errorf("Error uploading X509 cert: %s", err) + return multistep.ActionHalt + } + + if err := s.uploadSingle(comm, x509RemoteKeyPath, config.X509KeyPath); err != nil { + state["error"] = fmt.Errorf("Error uploading X509 cert: %s", err) + return multistep.ActionHalt + } + + return multistep.ActionHalt +} + +func (s *StepUploadX509Cert) Cleanup(map[string]interface{}) {} + +func (s *StepUploadX509Cert) uploadSingle(comm packer.Communicator, dst, src string) error { + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + return comm.Upload(dst, f) +} From 2674bdc96dd961dfe9ddebba6215c35dbc40dc93 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 20 Jul 2013 21:03:29 -0700 Subject: [PATCH 19/36] builder/amazon/instance: after upload x509, should continue --- builder/amazon/instance/step_upload_x509_cert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/instance/step_upload_x509_cert.go b/builder/amazon/instance/step_upload_x509_cert.go index 22f710e64..314e70098 100644 --- a/builder/amazon/instance/step_upload_x509_cert.go +++ b/builder/amazon/instance/step_upload_x509_cert.go @@ -28,7 +28,7 @@ func (s *StepUploadX509Cert) Run(state map[string]interface{}) multistep.StepAct return multistep.ActionHalt } - return multistep.ActionHalt + return multistep.ActionContinue } func (s *StepUploadX509Cert) Cleanup(map[string]interface{}) {} From f94500b324865af3e66bdace2d56557509e44e78 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 21 Jul 2013 11:32:59 -0700 Subject: [PATCH 20/36] builder/amazon/instance: boilerplate for bundle volume --- builder/amazon/instance/builder.go | 1 + builder/amazon/instance/step_bundle_volume.go | 15 +++++++++++++++ builder/amazon/instance/step_upload_x509_cert.go | 2 ++ 3 files changed, 18 insertions(+) create mode 100644 builder/amazon/instance/step_bundle_volume.go diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index ba7224de0..9cb7ce20b 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -111,6 +111,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, &common.StepProvision{}, &StepUploadX509Cert{}, + &StepBundleVolume{}, } // Run! diff --git a/builder/amazon/instance/step_bundle_volume.go b/builder/amazon/instance/step_bundle_volume.go new file mode 100644 index 000000000..e4c2eca48 --- /dev/null +++ b/builder/amazon/instance/step_bundle_volume.go @@ -0,0 +1,15 @@ +package instance + +import ( + "github.com/mitchellh/multistep" + "time" +) + +type StepBundleVolume struct{} + +func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepAction { + time.Sleep(10 * time.Hour) + return multistep.ActionContinue +} + +func (s *StepBundleVolume) Cleanup(map[string]interface{}) {} diff --git a/builder/amazon/instance/step_upload_x509_cert.go b/builder/amazon/instance/step_upload_x509_cert.go index 314e70098..33add850e 100644 --- a/builder/amazon/instance/step_upload_x509_cert.go +++ b/builder/amazon/instance/step_upload_x509_cert.go @@ -20,11 +20,13 @@ func (s *StepUploadX509Cert) Run(state map[string]interface{}) multistep.StepAct ui.Say("Uploading X509 Certificate...") if err := s.uploadSingle(comm, x509RemoteCertPath, config.X509CertPath); err != nil { state["error"] = fmt.Errorf("Error uploading X509 cert: %s", err) + ui.Error(state["error"].(error).Error()) return multistep.ActionHalt } if err := s.uploadSingle(comm, x509RemoteKeyPath, config.X509KeyPath); err != nil { state["error"] = fmt.Errorf("Error uploading X509 cert: %s", err) + ui.Error(state["error"].(error).Error()) return multistep.ActionHalt } From c710323f0416bf6cf6ced736d2bdb4454a128759 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 21 Jul 2013 22:46:11 -0700 Subject: [PATCH 21/36] builder/amazon/*: Fix failing tests from rebase of VPC --- builder/amazon/common/artifact_test.go | 7 +++++-- builder/amazon/common/step_run_source_instance.go | 3 ++- builder/amazon/common/step_security_group.go | 8 +++++++- builder/amazon/ebs/builder.go | 1 + builder/amazon/instance/builder.go | 1 + 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/builder/amazon/common/artifact_test.go b/builder/amazon/common/artifact_test.go index be8121074..28af553c7 100644 --- a/builder/amazon/common/artifact_test.go +++ b/builder/amazon/common/artifact_test.go @@ -22,7 +22,10 @@ func TestArtifactId(t *testing.T) { amis["east"] = "foo" amis["west"] = "bar" - a := &Artifact{amis, nil} + a := &Artifact{ + Amis: amis, + } + result := a.Id() assert.Equal(result, expected, "should match output") } @@ -39,7 +42,7 @@ west: bar` amis["east"] = "foo" amis["west"] = "bar" - a := &Artifact{amis, nil} + a := &Artifact{Amis: amis} result := a.String() assert.Equal(result, expected, "should match output") } diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index b7c7d4ab5..2c7254b49 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -12,6 +12,7 @@ type StepRunSourceInstance struct { ExpectedRootDevice string InstanceType string SourceAMI string + SubnetId string instance *ec2.Instance } @@ -29,7 +30,7 @@ func (s *StepRunSourceInstance) Run(state map[string]interface{}) multistep.Step MinCount: 0, MaxCount: 0, SecurityGroups: []ec2.SecurityGroup{ec2.SecurityGroup{Id: securityGroupId}}, - SubnetId: config.SubnetId, + SubnetId: s.SubnetId, } ui.Say("Launching a source AWS instance...") diff --git a/builder/amazon/common/step_security_group.go b/builder/amazon/common/step_security_group.go index dec317262..d7576f40a 100644 --- a/builder/amazon/common/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -13,6 +13,7 @@ import ( type StepSecurityGroup struct { SecurityGroupId string SSHPort int + VpcId string createdGroupId string } @@ -35,7 +36,12 @@ func (s *StepSecurityGroup) Run(state map[string]interface{}) multistep.StepActi ui.Say("Creating temporary security group for this instance...") groupName := fmt.Sprintf("packer %s", hex.EncodeToString(identifier.NewUUID().Raw())) log.Printf("Temporary group name: %s", groupName) - groupResp, err := ec2conn.CreateSecurityGroup(ec2.SecurityGroup{Name: groupName, Description: "Temporary group for Packer", VpcId: config.VpcId}) + group := ec2.SecurityGroup{ + Name: groupName, + Description: "Temporary group for Packer", + VpcId: s.VpcId, + } + groupResp, err := ec2conn.CreateSecurityGroup(group) if err != nil { ui.Error(err.Error()) return multistep.ActionHalt diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index f258e7ab9..629ded646 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -97,6 +97,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe ExpectedRootDevice: "ebs", InstanceType: b.config.InstanceType, SourceAMI: b.config.SourceAmi, + SubnetId: b.config.SubnetId, }, &common.StepConnectSSH{ SSHAddress: awscommon.SSHAddress(b.config.SSHPort), diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 9cb7ce20b..e3b7e2ef7 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -103,6 +103,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe ExpectedRootDevice: "instance-store", InstanceType: b.config.InstanceType, SourceAMI: b.config.SourceAmi, + SubnetId: b.config.SubnetId, }, &common.StepConnectSSH{ SSHAddress: awscommon.SSHAddress(b.config.SSHPort), From dd1c4d4d2a1c0d802b07571780defdd247256d93 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 21 Jul 2013 22:46:58 -0700 Subject: [PATCH 22/36] fmt --- builder/amazon/common/step_security_group.go | 6 +++--- builder/amazon/ebs/builder.go | 3 ++- builder/amazon/instance/builder.go | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/builder/amazon/common/step_security_group.go b/builder/amazon/common/step_security_group.go index d7576f40a..5b01a68f1 100644 --- a/builder/amazon/common/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -13,7 +13,7 @@ import ( type StepSecurityGroup struct { SecurityGroupId string SSHPort int - VpcId string + VpcId string createdGroupId string } @@ -37,9 +37,9 @@ func (s *StepSecurityGroup) Run(state map[string]interface{}) multistep.StepActi groupName := fmt.Sprintf("packer %s", hex.EncodeToString(identifier.NewUUID().Raw())) log.Printf("Temporary group name: %s", groupName) group := ec2.SecurityGroup{ - Name: groupName, + Name: groupName, Description: "Temporary group for Packer", - VpcId: s.VpcId, + VpcId: s.VpcId, } groupResp, err := ec2conn.CreateSecurityGroup(group) if err != nil { diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 629ded646..46f5231fc 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -92,12 +92,13 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepSecurityGroup{ SecurityGroupId: b.config.SecurityGroupId, SSHPort: b.config.SSHPort, + VpcId: b.config.VpcId, }, &awscommon.StepRunSourceInstance{ ExpectedRootDevice: "ebs", InstanceType: b.config.InstanceType, SourceAMI: b.config.SourceAmi, - SubnetId: b.config.SubnetId, + SubnetId: b.config.SubnetId, }, &common.StepConnectSSH{ SSHAddress: awscommon.SSHAddress(b.config.SSHPort), diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index e3b7e2ef7..196907698 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -98,12 +98,13 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepSecurityGroup{ SecurityGroupId: b.config.SecurityGroupId, SSHPort: b.config.SSHPort, + VpcId: b.config.VpcId, }, &awscommon.StepRunSourceInstance{ ExpectedRootDevice: "instance-store", InstanceType: b.config.InstanceType, SourceAMI: b.config.SourceAmi, - SubnetId: b.config.SubnetId, + SubnetId: b.config.SubnetId, }, &common.StepConnectSSH{ SSHAddress: awscommon.SSHAddress(b.config.SSHPort), From 110fd0e17fd2c72244aea499b47dac92a2b51055 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 21 Jul 2013 22:48:58 -0700 Subject: [PATCH 23/36] builder/amazon/common: SSH into private IP if in VPC --- builder/amazon/common/ssh.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/builder/amazon/common/ssh.go b/builder/amazon/common/ssh.go index ea589189f..96e51cd20 100644 --- a/builder/amazon/common/ssh.go +++ b/builder/amazon/common/ssh.go @@ -11,8 +11,15 @@ import ( // for determining the SSH address based on the instance DNS name. func SSHAddress(port int) func(map[string]interface{}) (string, error) { return func(state map[string]interface{}) (string, error) { + var host string instance := state["instance"].(*ec2.Instance) - return fmt.Sprintf("%s:%d", instance.DNSName, port), nil + if instance.VpcId != "" { + host = instance.PrivateIpAddress + } else { + host = instance.DNSName + } + + return fmt.Sprintf("%s:%d", host, port), nil } } From fd43c27de11ac06a114553ed81c09579526f1148 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 23 Jul 2013 23:19:44 -0500 Subject: [PATCH 24/36] builder/amazon/instance: check for the ami tools --- builder/amazon/instance/step_bundle_volume.go | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/builder/amazon/instance/step_bundle_volume.go b/builder/amazon/instance/step_bundle_volume.go index e4c2eca48..a0142991c 100644 --- a/builder/amazon/instance/step_bundle_volume.go +++ b/builder/amazon/instance/step_bundle_volume.go @@ -1,14 +1,36 @@ package instance import ( + "fmt" "github.com/mitchellh/multistep" - "time" + "github.com/mitchellh/packer/packer" ) type StepBundleVolume struct{} func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepAction { - time.Sleep(10 * time.Hour) + comm := state["communicator"].(packer.Communicator) + ui := state["ui"].(packer.Ui) + + // Verify the AMI tools are available + ui.Say("Checking for EC2 AMI tools...") + cmd := &packer.RemoteCmd{Command: "ec2-ami-tools-version"} + if err := comm.Start(cmd); err != nil { + state["error"] = fmt.Errorf("Error checking for AMI tools: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + cmd.Wait() + + if cmd.ExitStatus != 0 { + state["error"] = fmt.Errorf( + "The EC2 AMI tools could not be detected. These must be manually\n" + + "via a provisioner or some other means and are required for Packer\n" + + "to create an instance-store AMI.") + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + return multistep.ActionContinue } From c1361b0cf57b107117e2c172959d732ff24689c9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 24 Jul 2013 15:41:49 -0500 Subject: [PATCH 25/36] builder/amazon/instance: ec2-bundle-vol is called --- builder/amazon/instance/builder.go | 24 +++++++++++-- builder/amazon/instance/builder_test.go | 28 +++++++++++++++ builder/amazon/instance/step_bundle_volume.go | 36 +++++++++++++++++++ .../amazon/instance/step_upload_x509_cert.go | 3 ++ 4 files changed, 88 insertions(+), 3 deletions(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 196907698..be2ea3dd3 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -13,6 +13,7 @@ import ( "github.com/mitchellh/packer/packer" "log" "os" + "strings" ) // The unique ID for this builder @@ -25,9 +26,11 @@ type Config struct { awscommon.AccessConfig `mapstructure:",squash"` awscommon.RunConfig `mapstructure:",squash"` - X509CertPath string `mapstructure:"x509_cert_path"` - X509KeyPath string `mapstructure:"x509_key_path"` - X509UploadPath string `mapstructure:"x509_upload_path"` + AccountId string `mapstructure:"account_id"` + BundleVolCommand string `mapstructure:"bundle_vol_command"` + X509CertPath string `mapstructure:"x509_cert_path"` + X509KeyPath string `mapstructure:"x509_key_path"` + X509UploadPath string `mapstructure:"x509_upload_path"` } type Builder struct { @@ -46,6 +49,21 @@ func (b *Builder) Prepare(raws ...interface{}) error { errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare()...) + if b.config.AccountId == "" { + errs = packer.MultiErrorAppend(errs, errors.New("account_id is required")) + } else { + b.config.AccountId = strings.Replace(b.config.AccountId, "-", "", -1) + } + + if b.config.BundleVolCommand == "" { + b.config.BundleVolCommand = "ec2-bundle-vol " + + "-k {{.KeyPath}} " + + "-u {{.AccountId}} " + + "-c {{.CertPath}} " + + "-r {{.Architecture}} " + + "-e {{.PrivatePath}}" + } + if b.config.X509CertPath == "" { errs = packer.MultiErrorAppend(errs, errors.New("x509_cert_path is required")) } else if _, err := os.Stat(b.config.X509CertPath); err != nil { diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go index d83ee9c33..5b3a560a5 100644 --- a/builder/amazon/instance/builder_test.go +++ b/builder/amazon/instance/builder_test.go @@ -14,6 +14,7 @@ func testConfig() map[string]interface{} { } return map[string]interface{}{ + "account_id": "foo", "instance_type": "m1.small", "region": "us-east-1", "source_ami": "foo", @@ -32,6 +33,33 @@ func TestBuilder_ImplementsBuilder(t *testing.T) { } } +func TestBuilderPrepare_AccountId(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["account_id"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["account_id"] = "foo" + err = b.Prepare(config) + if err != nil { + t.Errorf("err: %s", err) + } + + config["account_id"] = "0123-0456-7890" + err = b.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + if b.config.AccountId != "012304567890" { + t.Errorf("should strip hyphens: %s", b.config.AccountId) + } +} + func TestBuilderPrepare_InvalidKey(t *testing.T) { var b Builder config := testConfig() diff --git a/builder/amazon/instance/step_bundle_volume.go b/builder/amazon/instance/step_bundle_volume.go index a0142991c..7851f687a 100644 --- a/builder/amazon/instance/step_bundle_volume.go +++ b/builder/amazon/instance/step_bundle_volume.go @@ -1,16 +1,31 @@ package instance import ( + "bytes" "fmt" + "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" + "text/template" ) +type bundleCmdData struct { + AccountId string + Architecture string + CertPath string + KeyPath string + PrivatePath string +} + type StepBundleVolume struct{} func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepAction { comm := state["communicator"].(packer.Communicator) + config := state["config"].(*Config) + instance := state["instance"].(*ec2.Instance) ui := state["ui"].(packer.Ui) + x509RemoteCertPath := state["x509RemoteCertPath"].(string) + x509RemoteKeyPath := state["x509RemoteKeyPath"].(string) // Verify the AMI tools are available ui.Say("Checking for EC2 AMI tools...") @@ -31,6 +46,27 @@ func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepActio return multistep.ActionHalt } + // Bundle the volume + var bundleCmd bytes.Buffer + tData := bundleCmdData{ + AccountId: config.AccountId, + Architecture: instance.Architecture, + CertPath: x509RemoteCertPath, + KeyPath: x509RemoteKeyPath, + PrivatePath: config.X509UploadPath, + } + t := template.Must(template.New("bundleCmd").Parse(config.BundleVolCommand)) + t.Execute(&bundleCmd, tData) + + ui.Say("Bundling the volume...") + cmd = new(packer.RemoteCmd) + cmd.Command = bundleCmd.String() + if err := cmd.StartWithUi(comm, ui); err != nil { + state["error"] = fmt.Errorf("Error bundling volume: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + return multistep.ActionContinue } diff --git a/builder/amazon/instance/step_upload_x509_cert.go b/builder/amazon/instance/step_upload_x509_cert.go index 33add850e..5660bc228 100644 --- a/builder/amazon/instance/step_upload_x509_cert.go +++ b/builder/amazon/instance/step_upload_x509_cert.go @@ -30,6 +30,9 @@ func (s *StepUploadX509Cert) Run(state map[string]interface{}) multistep.StepAct return multistep.ActionHalt } + state["x509RemoteCertPath"] = x509RemoteCertPath + state["x509RemoteKeyPath"] = x509RemoteKeyPath + return multistep.ActionContinue } From 7763411914545c24a5d24a49bc394a13c907eff4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 24 Jul 2013 15:44:58 -0500 Subject: [PATCH 26/36] builder/amazon/instance: detect error bundling --- builder/amazon/instance/step_bundle_volume.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/builder/amazon/instance/step_bundle_volume.go b/builder/amazon/instance/step_bundle_volume.go index 7851f687a..2ea0c782f 100644 --- a/builder/amazon/instance/step_bundle_volume.go +++ b/builder/amazon/instance/step_bundle_volume.go @@ -67,6 +67,14 @@ func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepActio return multistep.ActionHalt } + if cmd.ExitStatus != 0 { + state["error"] = fmt.Errorf( + "Volume bundling failed. Please see the output above for more\n" + + "details on what went wrong.") + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + return multistep.ActionContinue } From d3d7bfe876d45c0b48776b07175ef24689b7f798 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 24 Jul 2013 16:07:33 -0500 Subject: [PATCH 27/36] builder/amazon/instance: sudo the ec2-bundle-vol by default --- builder/amazon/instance/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index be2ea3dd3..b24dae7a9 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -56,7 +56,7 @@ func (b *Builder) Prepare(raws ...interface{}) error { } if b.config.BundleVolCommand == "" { - b.config.BundleVolCommand = "ec2-bundle-vol " + + b.config.BundleVolCommand = "sudo -n ec2-bundle-vol " + "-k {{.KeyPath}} " + "-u {{.AccountId}} " + "-c {{.CertPath}} " + From 63474f47e45ccbb2e1439e7154343933971c9c8a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 24 Jul 2013 22:51:59 -0500 Subject: [PATCH 28/36] builder/amazon/instance: run ec2-bundle-vol in batch mode --- builder/amazon/instance/builder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index b24dae7a9..4fb2b226d 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -61,7 +61,8 @@ func (b *Builder) Prepare(raws ...interface{}) error { "-u {{.AccountId}} " + "-c {{.CertPath}} " + "-r {{.Architecture}} " + - "-e {{.PrivatePath}}" + "-e {{.PrivatePath}} " + + "--batch" } if b.config.X509CertPath == "" { From 877172166b8f2882bc5d96d4f198fba1e8a05b8c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 24 Jul 2013 23:22:16 -0500 Subject: [PATCH 29/36] builder/amazon/instance: bundle volume and keep track of dir --- builder/amazon/instance/builder.go | 27 +++++++++-- builder/amazon/instance/builder_test.go | 48 +++++++++++++++++++ builder/amazon/instance/step_bundle_volume.go | 4 ++ 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 4fb2b226d..ac4631cba 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -26,11 +26,14 @@ type Config struct { awscommon.AccessConfig `mapstructure:",squash"` awscommon.RunConfig `mapstructure:",squash"` - AccountId string `mapstructure:"account_id"` - BundleVolCommand string `mapstructure:"bundle_vol_command"` - X509CertPath string `mapstructure:"x509_cert_path"` - X509KeyPath string `mapstructure:"x509_key_path"` - X509UploadPath string `mapstructure:"x509_upload_path"` + AccountId string `mapstructure:"account_id"` + BundleDestination string `mapstructure:"bundle_destination"` + BundlePrefix string `mapstructure:"bundle_prefix"` + BundleVolCommand string `mapstructure:"bundle_vol_command"` + S3Bucket string `mapstructure:"s3_bucket"` + X509CertPath string `mapstructure:"x509_cert_path"` + X509KeyPath string `mapstructure:"x509_key_path"` + X509UploadPath string `mapstructure:"x509_upload_path"` } type Builder struct { @@ -44,6 +47,14 @@ func (b *Builder) Prepare(raws ...interface{}) error { return err } + if b.config.BundleDestination == "" { + b.config.BundleDestination = "/tmp" + } + + if b.config.BundlePrefix == "" { + b.config.BundlePrefix = "image" + } + // Accumulate any errors errs := common.CheckUnusedConfig(md) errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) @@ -62,9 +73,15 @@ func (b *Builder) Prepare(raws ...interface{}) error { "-c {{.CertPath}} " + "-r {{.Architecture}} " + "-e {{.PrivatePath}} " + + "-d {{.Destination}} " + + "-p {{.Prefix}} " + "--batch" } + if b.config.S3Bucket == "" { + errs = packer.MultiErrorAppend(errs, errors.New("s3_bucket is required")) + } + if b.config.X509CertPath == "" { errs = packer.MultiErrorAppend(errs, errors.New("x509_cert_path is required")) } else if _, err := os.Stat(b.config.X509CertPath); err != nil { diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go index 5b3a560a5..6febf7b8b 100644 --- a/builder/amazon/instance/builder_test.go +++ b/builder/amazon/instance/builder_test.go @@ -17,6 +17,7 @@ func testConfig() map[string]interface{} { "account_id": "foo", "instance_type": "m1.small", "region": "us-east-1", + "s3_bucket": "foo", "source_ami": "foo", "ssh_username": "bob", "x509_cert_path": tf.Name(), @@ -60,6 +61,36 @@ func TestBuilderPrepare_AccountId(t *testing.T) { } } +func TestBuilderPrepare_BundleDestination(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["bundle_destination"] = "" + err := b.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + if b.config.BundleDestination != "/tmp" { + t.Fatalf("bad: %s", b.config.BundleDestination) + } +} + +func TestBuilderPrepare_BundlePrefix(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["bundle_prefix"] = "" + err := b.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + if b.config.BundlePrefix != "image" { + t.Fatalf("bad: %s", b.config.BundlePrefix) + } +} + func TestBuilderPrepare_InvalidKey(t *testing.T) { var b Builder config := testConfig() @@ -72,6 +103,23 @@ func TestBuilderPrepare_InvalidKey(t *testing.T) { } } +func TestBuilderPrepare_S3Bucket(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["s3_bucket"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["s3_bucket"] = "foo" + err = b.Prepare(config) + if err != nil { + t.Errorf("err: %s", err) + } +} + func TestBuilderPrepare_X509CertPath(t *testing.T) { b := &Builder{} config := testConfig() diff --git a/builder/amazon/instance/step_bundle_volume.go b/builder/amazon/instance/step_bundle_volume.go index 2ea0c782f..139268a95 100644 --- a/builder/amazon/instance/step_bundle_volume.go +++ b/builder/amazon/instance/step_bundle_volume.go @@ -13,7 +13,9 @@ type bundleCmdData struct { AccountId string Architecture string CertPath string + Destination string KeyPath string + Prefix string PrivatePath string } @@ -52,7 +54,9 @@ func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepActio AccountId: config.AccountId, Architecture: instance.Architecture, CertPath: x509RemoteCertPath, + Destination: config.BundleDestination, KeyPath: x509RemoteKeyPath, + Prefix: config.BundlePrefix, PrivatePath: config.X509UploadPath, } t := template.Must(template.New("bundleCmd").Parse(config.BundleVolCommand)) From c504beacc61167e1769265a0121678a543ad8bcb Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 24 Jul 2013 23:29:21 -0500 Subject: [PATCH 30/36] builder/amazon/instance: upload bundle --- builder/amazon/instance/builder.go | 47 +++++++++------ builder/amazon/instance/step_upload_bundle.go | 57 +++++++++++++++++++ 2 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 builder/amazon/instance/step_upload_bundle.go diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index ac4631cba..d9b367b2f 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -26,14 +26,15 @@ type Config struct { awscommon.AccessConfig `mapstructure:",squash"` awscommon.RunConfig `mapstructure:",squash"` - AccountId string `mapstructure:"account_id"` - BundleDestination string `mapstructure:"bundle_destination"` - BundlePrefix string `mapstructure:"bundle_prefix"` - BundleVolCommand string `mapstructure:"bundle_vol_command"` - S3Bucket string `mapstructure:"s3_bucket"` - X509CertPath string `mapstructure:"x509_cert_path"` - X509KeyPath string `mapstructure:"x509_key_path"` - X509UploadPath string `mapstructure:"x509_upload_path"` + AccountId string `mapstructure:"account_id"` + BundleDestination string `mapstructure:"bundle_destination"` + BundlePrefix string `mapstructure:"bundle_prefix"` + BundleUploadCommand string `mapstructure:"bundle_upload_command"` + BundleVolCommand string `mapstructure:"bundle_vol_command"` + S3Bucket string `mapstructure:"s3_bucket"` + X509CertPath string `mapstructure:"x509_cert_path"` + X509KeyPath string `mapstructure:"x509_key_path"` + X509UploadPath string `mapstructure:"x509_upload_path"` } type Builder struct { @@ -55,15 +56,15 @@ func (b *Builder) Prepare(raws ...interface{}) error { b.config.BundlePrefix = "image" } - // Accumulate any errors - errs := common.CheckUnusedConfig(md) - errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) - errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare()...) - - if b.config.AccountId == "" { - errs = packer.MultiErrorAppend(errs, errors.New("account_id is required")) - } else { - b.config.AccountId = strings.Replace(b.config.AccountId, "-", "", -1) + if b.config.BundleUploadCommand == "" { + b.config.BundleUploadCommand = "sudo -n ec2-upload-bundle " + + "-b {{.BucketName}} " + + "-m {{.ManifestPath}} " + + "-a {{.AccessKey}} " + + "-s {{.SecretKey}} " + + "-d {{.BundleDirectory}} " + + "--batch " + + "--retry" } if b.config.BundleVolCommand == "" { @@ -78,6 +79,17 @@ func (b *Builder) Prepare(raws ...interface{}) error { "--batch" } + // Accumulate any errors + errs := common.CheckUnusedConfig(md) + errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) + errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare()...) + + if b.config.AccountId == "" { + errs = packer.MultiErrorAppend(errs, errors.New("account_id is required")) + } else { + b.config.AccountId = strings.Replace(b.config.AccountId, "-", "", -1) + } + if b.config.S3Bucket == "" { errs = packer.MultiErrorAppend(errs, errors.New("s3_bucket is required")) } @@ -150,6 +162,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &common.StepProvision{}, &StepUploadX509Cert{}, &StepBundleVolume{}, + &StepUploadBundle{}, } // Run! diff --git a/builder/amazon/instance/step_upload_bundle.go b/builder/amazon/instance/step_upload_bundle.go new file mode 100644 index 000000000..87cfb7e6e --- /dev/null +++ b/builder/amazon/instance/step_upload_bundle.go @@ -0,0 +1,57 @@ +package instance + +import ( + "bytes" + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "text/template" +) + +type uploadCmdData struct { + AccessKey string + BucketName string + BundleDirectory string + ManifestPath string + SecretKey string +} + +type StepUploadBundle struct{} + +func (s *StepUploadBundle) Run(state map[string]interface{}) multistep.StepAction { + comm := state["communicator"].(packer.Communicator) + config := state["config"].(*Config) + ui := state["ui"].(packer.Ui) + + var uploadCmd bytes.Buffer + tData := uploadCmdData{ + AccessKey: config.AccessKey, + BucketName: config.S3Bucket, + BundleDirectory: config.BundleDestination, + ManifestPath: fmt.Sprintf( + "%s/%s.manifest.xml", config.BundleDestination, config.BundlePrefix), + SecretKey: config.SecretKey, + } + t := template.Must(template.New("uploadCmd").Parse(config.BundleUploadCommand)) + t.Execute(&uploadCmd, tData) + + ui.Say("Uploading the bundle...") + cmd := &packer.RemoteCmd{Command: uploadCmd.String()} + if err := cmd.StartWithUi(comm, ui); err != nil { + state["error"] = fmt.Errorf("Error uploading volume: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + if cmd.ExitStatus != 0 { + state["error"] = fmt.Errorf( + "Bundle upload failed. Please see the output above for more\n" + + "details on what went wrong.") + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *StepUploadBundle) Cleanup(state map[string]interface{}) {} From 03a2cc8b2215cc7df13e67be25da76e393dde20e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 24 Jul 2013 23:51:46 -0500 Subject: [PATCH 31/36] builder/amazon/instance: prefix has CreateTime support --- builder/amazon/instance/builder.go | 2 +- builder/amazon/instance/builder_test.go | 2 +- builder/amazon/instance/step_bundle_volume.go | 21 +++++++++++++++++-- builder/amazon/instance/step_upload_bundle.go | 6 +++--- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index d9b367b2f..503af210a 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -53,7 +53,7 @@ func (b *Builder) Prepare(raws ...interface{}) error { } if b.config.BundlePrefix == "" { - b.config.BundlePrefix = "image" + b.config.BundlePrefix = "image-{{.CreateTime}}" } if b.config.BundleUploadCommand == "" { diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go index 6febf7b8b..2e6b3639a 100644 --- a/builder/amazon/instance/builder_test.go +++ b/builder/amazon/instance/builder_test.go @@ -86,7 +86,7 @@ func TestBuilderPrepare_BundlePrefix(t *testing.T) { t.Fatalf("err: %s", err) } - if b.config.BundlePrefix != "image" { + if b.config.BundlePrefix != "image-{{.CreateTime}}" { t.Fatalf("bad: %s", b.config.BundlePrefix) } } diff --git a/builder/amazon/instance/step_bundle_volume.go b/builder/amazon/instance/step_bundle_volume.go index 139268a95..23fb7a415 100644 --- a/builder/amazon/instance/step_bundle_volume.go +++ b/builder/amazon/instance/step_bundle_volume.go @@ -6,7 +6,9 @@ import ( "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" + "strconv" "text/template" + "time" ) type bundleCmdData struct { @@ -19,6 +21,10 @@ type bundleCmdData struct { PrivatePath string } +type bundlePrefixData struct { + CreateTime string +} + type StepBundleVolume struct{} func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepAction { @@ -49,6 +55,13 @@ func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepActio } // Bundle the volume + var bundlePrefix bytes.Buffer + prefixTData := bundlePrefixData{ + CreateTime: strconv.FormatInt(time.Now().UTC().Unix(), 10), + } + t := template.Must(template.New("bundlePrefix").Parse(config.BundlePrefix)) + t.Execute(&bundlePrefix, prefixTData) + var bundleCmd bytes.Buffer tData := bundleCmdData{ AccountId: config.AccountId, @@ -56,10 +69,10 @@ func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepActio CertPath: x509RemoteCertPath, Destination: config.BundleDestination, KeyPath: x509RemoteKeyPath, - Prefix: config.BundlePrefix, + Prefix: bundlePrefix.String(), PrivatePath: config.X509UploadPath, } - t := template.Must(template.New("bundleCmd").Parse(config.BundleVolCommand)) + t = template.Must(template.New("bundleCmd").Parse(config.BundleVolCommand)) t.Execute(&bundleCmd, tData) ui.Say("Bundling the volume...") @@ -79,6 +92,10 @@ func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepActio return multistep.ActionHalt } + // Store the manifest path + state["manifest_path"] = fmt.Sprintf( + "%s/%s.manifest.xml", config.BundleDestination, bundlePrefix.String()) + return multistep.ActionContinue } diff --git a/builder/amazon/instance/step_upload_bundle.go b/builder/amazon/instance/step_upload_bundle.go index 87cfb7e6e..827daf8b0 100644 --- a/builder/amazon/instance/step_upload_bundle.go +++ b/builder/amazon/instance/step_upload_bundle.go @@ -21,6 +21,7 @@ type StepUploadBundle struct{} func (s *StepUploadBundle) Run(state map[string]interface{}) multistep.StepAction { comm := state["communicator"].(packer.Communicator) config := state["config"].(*Config) + manifestPath := state["manifest_path"].(string) ui := state["ui"].(packer.Ui) var uploadCmd bytes.Buffer @@ -28,9 +29,8 @@ func (s *StepUploadBundle) Run(state map[string]interface{}) multistep.StepActio AccessKey: config.AccessKey, BucketName: config.S3Bucket, BundleDirectory: config.BundleDestination, - ManifestPath: fmt.Sprintf( - "%s/%s.manifest.xml", config.BundleDestination, config.BundlePrefix), - SecretKey: config.SecretKey, + ManifestPath: manifestPath, + SecretKey: config.SecretKey, } t := template.Must(template.New("uploadCmd").Parse(config.BundleUploadCommand)) t.Execute(&uploadCmd, tData) From 0552bc7306d9f6f2ce780f98458664e498f162b1 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 25 Jul 2013 00:19:04 -0500 Subject: [PATCH 32/36] builder/amazon/instance: register the AMI --- builder/amazon/instance/builder.go | 23 ++++++- builder/amazon/instance/builder_test.go | 29 +++++++++ builder/amazon/instance/step_bundle_volume.go | 4 +- builder/amazon/instance/step_register_ami.go | 61 +++++++++++++++++++ builder/amazon/instance/step_upload_bundle.go | 4 ++ 5 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 builder/amazon/instance/step_register_ami.go diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 503af210a..2d2a101a5 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -14,6 +14,7 @@ import ( "log" "os" "strings" + "text/template" ) // The unique ID for this builder @@ -27,6 +28,7 @@ type Config struct { awscommon.RunConfig `mapstructure:",squash"` AccountId string `mapstructure:"account_id"` + AMIName string `mapstructure:"ami_name"` BundleDestination string `mapstructure:"bundle_destination"` BundlePrefix string `mapstructure:"bundle_prefix"` BundleUploadCommand string `mapstructure:"bundle_upload_command"` @@ -90,6 +92,17 @@ func (b *Builder) Prepare(raws ...interface{}) error { b.config.AccountId = strings.Replace(b.config.AccountId, "-", "", -1) } + if b.config.AMIName == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("ami_name must be specified")) + } else { + _, err = template.New("ami").Parse(b.config.AMIName) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Failed parsing ami_name: %s", err)) + } + } + if b.config.S3Bucket == "" { errs = packer.MultiErrorAppend(errs, errors.New("s3_bucket is required")) } @@ -163,6 +176,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &StepUploadX509Cert{}, &StepBundleVolume{}, &StepUploadBundle{}, + &StepRegisterAMI{}, } // Run! @@ -187,7 +201,14 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe return nil, nil } - return nil, nil + // Build the artifact and return it + artifact := &awscommon.Artifact{ + Amis: state["amis"].(map[string]string), + BuilderIdValue: BuilderId, + Conn: ec2conn, + } + + return artifact, nil } func (b *Builder) Cancel() { diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go index 2e6b3639a..9633dff84 100644 --- a/builder/amazon/instance/builder_test.go +++ b/builder/amazon/instance/builder_test.go @@ -15,6 +15,7 @@ func testConfig() map[string]interface{} { return map[string]interface{}{ "account_id": "foo", + "ami_name": "foo", "instance_type": "m1.small", "region": "us-east-1", "s3_bucket": "foo", @@ -61,6 +62,34 @@ func TestBuilderPrepare_AccountId(t *testing.T) { } } +func TestBuilderPrepare_AMIName(t *testing.T) { + var b Builder + config := testConfig() + + // Test good + config["ami_name"] = "foo" + err := b.Prepare(config) + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + // Test bad + config["ami_name"] = "foo {{" + b = Builder{} + err = b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + // Test bad + delete(config, "ami_name") + b = Builder{} + err = b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } +} + func TestBuilderPrepare_BundleDestination(t *testing.T) { b := &Builder{} config := testConfig() diff --git a/builder/amazon/instance/step_bundle_volume.go b/builder/amazon/instance/step_bundle_volume.go index 23fb7a415..f5da54dde 100644 --- a/builder/amazon/instance/step_bundle_volume.go +++ b/builder/amazon/instance/step_bundle_volume.go @@ -93,8 +93,10 @@ func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepActio } // Store the manifest path + manifestName := bundlePrefix.String() + ".manifest.xml" + state["manifest_name"] = manifestName state["manifest_path"] = fmt.Sprintf( - "%s/%s.manifest.xml", config.BundleDestination, bundlePrefix.String()) + "%s/%s", config.BundleDestination, manifestName) return multistep.ActionContinue } diff --git a/builder/amazon/instance/step_register_ami.go b/builder/amazon/instance/step_register_ami.go new file mode 100644 index 000000000..52f5c30e6 --- /dev/null +++ b/builder/amazon/instance/step_register_ami.go @@ -0,0 +1,61 @@ +package instance + +import ( + "bytes" + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "strconv" + "text/template" + "time" +) + +type amiNameData struct { + CreateTime string +} + +type StepRegisterAMI struct{} + +func (s *StepRegisterAMI) Run(state map[string]interface{}) multistep.StepAction { + comm := state["communicator"].(packer.Communicator) + config := state["config"].(*Config) + manifestPath := state["remote_manifest_path"].(string) + ui := state["ui"].(packer.Ui) + + // Parse the name of the AMI + amiNameBuf := new(bytes.Buffer) + tData := amiNameData{ + strconv.FormatInt(time.Now().UTC().Unix(), 10), + } + + t := template.Must(template.New("ami").Parse(config.AMIName)) + t.Execute(amiNameBuf, tData) + amiName := amiNameBuf.String() + + ui.Say("Registering the AMI...") + cmd := &packer.RemoteCmd{ + Command: fmt.Sprintf( + "ec2-register %s -n '%s' -O '%s' -W '%s'", + manifestPath, + amiName, + config.AccessKey, + config.SecretKey), + } + if err := cmd.StartWithUi(comm, ui); err != nil { + state["error"] = fmt.Errorf("Error registering AMI: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + if cmd.ExitStatus != 0 { + state["error"] = fmt.Errorf( + "AMI registration failed. Please see the output above for more\n" + + "details on what went wrong.") + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *StepRegisterAMI) Cleanup(map[string]interface{}) {} diff --git a/builder/amazon/instance/step_upload_bundle.go b/builder/amazon/instance/step_upload_bundle.go index 827daf8b0..8c8a9fdb5 100644 --- a/builder/amazon/instance/step_upload_bundle.go +++ b/builder/amazon/instance/step_upload_bundle.go @@ -21,6 +21,7 @@ type StepUploadBundle struct{} func (s *StepUploadBundle) Run(state map[string]interface{}) multistep.StepAction { comm := state["communicator"].(packer.Communicator) config := state["config"].(*Config) + manifestName := state["manifest_name"].(string) manifestPath := state["manifest_path"].(string) ui := state["ui"].(packer.Ui) @@ -51,6 +52,9 @@ func (s *StepUploadBundle) Run(state map[string]interface{}) multistep.StepActio return multistep.ActionHalt } + state["remote_manifest_path"] = fmt.Sprintf( + "%s/%s", config.S3Bucket, manifestName) + return multistep.ActionContinue } From b5fdab407fdf9d33e8f23ce7d3213dbf3f0268e7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 25 Jul 2013 00:45:55 -0500 Subject: [PATCH 33/36] builder/amazon/instance: register AMI using API --- builder/amazon/instance/step_register_ami.go | 29 +++++++++----------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/builder/amazon/instance/step_register_ami.go b/builder/amazon/instance/step_register_ami.go index 52f5c30e6..56ce784b9 100644 --- a/builder/amazon/instance/step_register_ami.go +++ b/builder/amazon/instance/step_register_ami.go @@ -3,6 +3,7 @@ package instance import ( "bytes" "fmt" + "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" "strconv" @@ -17,8 +18,8 @@ type amiNameData struct { type StepRegisterAMI struct{} func (s *StepRegisterAMI) Run(state map[string]interface{}) multistep.StepAction { - comm := state["communicator"].(packer.Communicator) config := state["config"].(*Config) + ec2conn := state["ec2"].(*ec2.EC2) manifestPath := state["remote_manifest_path"].(string) ui := state["ui"].(packer.Ui) @@ -33,27 +34,23 @@ func (s *StepRegisterAMI) Run(state map[string]interface{}) multistep.StepAction amiName := amiNameBuf.String() ui.Say("Registering the AMI...") - cmd := &packer.RemoteCmd{ - Command: fmt.Sprintf( - "ec2-register %s -n '%s' -O '%s' -W '%s'", - manifestPath, - amiName, - config.AccessKey, - config.SecretKey), + registerOpts := &ec2.RegisterImage{ + ImageLocation: manifestPath, + Name: amiName, } - if err := cmd.StartWithUi(comm, ui); err != nil { + + registerResp, err := ec2conn.RegisterImage(registerOpts) + if err != nil { state["error"] = fmt.Errorf("Error registering AMI: %s", err) ui.Error(state["error"].(error).Error()) return multistep.ActionHalt } - if cmd.ExitStatus != 0 { - state["error"] = fmt.Errorf( - "AMI registration failed. Please see the output above for more\n" + - "details on what went wrong.") - ui.Error(state["error"].(error).Error()) - return multistep.ActionHalt - } + // Set the AMI ID in the state + ui.Say(fmt.Sprintf("AMI: %s", registerResp.ImageId)) + amis := make(map[string]string) + amis[config.Region] = registerResp.ImageId + state["amis"] = amis return multistep.ActionContinue } From d46741e4f7f06958b1049d9ff8f9d370b74fc5ff Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 25 Jul 2013 00:56:37 -0500 Subject: [PATCH 34/36] builder/amazon/*: wait for AMI to be ready in common, use it instance --- builder/amazon/common/ami.go | 25 ++++++++++++++++++++ builder/amazon/ebs/step_create_ami.go | 24 +++++-------------- builder/amazon/instance/step_register_ami.go | 10 ++++++++ 3 files changed, 41 insertions(+), 18 deletions(-) create mode 100644 builder/amazon/common/ami.go diff --git a/builder/amazon/common/ami.go b/builder/amazon/common/ami.go new file mode 100644 index 000000000..6a305aa37 --- /dev/null +++ b/builder/amazon/common/ami.go @@ -0,0 +1,25 @@ +package common + +import ( + "github.com/mitchellh/goamz/ec2" + "log" + "time" +) + +// WaitForAMI waits for the given AMI ID to become ready. +func WaitForAMI(c *ec2.EC2, imageId string) error { + for { + imageResp, err := c.Images([]string{imageId}, ec2.NewFilter()) + if err != nil { + return err + } + + if imageResp.Images[0].State == "available" { + return nil + } + + log.Printf("Image in state %s, sleeping 2s before checking again", + imageResp.Images[0].State) + time.Sleep(2 * time.Second) + } +} diff --git a/builder/amazon/ebs/step_create_ami.go b/builder/amazon/ebs/step_create_ami.go index a33021b49..0b08bc7c9 100644 --- a/builder/amazon/ebs/step_create_ami.go +++ b/builder/amazon/ebs/step_create_ami.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" "github.com/mitchellh/packer/packer" - "log" "strconv" "text/template" "time" @@ -57,23 +57,11 @@ func (s *stepCreateAMI) Run(state map[string]interface{}) multistep.StepAction { // Wait for the image to become ready ui.Say("Waiting for AMI to become ready...") - for { - imageResp, err := ec2conn.Images([]string{createResp.ImageId}, ec2.NewFilter()) - if err != nil { - err := fmt.Errorf("Error querying images: %s", err) - state["error"] = err - ui.Error(err.Error()) - return multistep.ActionHalt - } - - if imageResp.Images[0].State == "available" { - break - } - - log.Printf("Image in state %s, sleeping 2s before checking again", - imageResp.Images[0].State) - - time.Sleep(2 * time.Second) + if err := awscommon.WaitForAMI(ec2conn, createResp.ImageId); err != nil { + err := fmt.Errorf("Error waiting for AMI: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt } return multistep.ActionContinue diff --git a/builder/amazon/instance/step_register_ami.go b/builder/amazon/instance/step_register_ami.go index 56ce784b9..d95deb888 100644 --- a/builder/amazon/instance/step_register_ami.go +++ b/builder/amazon/instance/step_register_ami.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" "github.com/mitchellh/packer/packer" "strconv" "text/template" @@ -52,6 +53,15 @@ func (s *StepRegisterAMI) Run(state map[string]interface{}) multistep.StepAction amis[config.Region] = registerResp.ImageId state["amis"] = amis + // Wait for the image to become ready + ui.Say("Waiting for AMI to become ready...") + if err := awscommon.WaitForAMI(ec2conn, registerResp.ImageId); err != nil { + err := fmt.Errorf("Error waiting for AMI: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + return multistep.ActionContinue } From 321d1cce169158d1f7cb6f923aad1c7c89d5abdc Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 25 Jul 2013 10:51:21 -0500 Subject: [PATCH 35/36] website: document the amazon-instance builder --- .../docs/builders/amazon-ebs.html.markdown | 17 +- .../builders/amazon-instance.html.markdown | 217 ++++++++++++++++++ .../source/docs/builders/amazon.html.markdown | 25 ++ website/source/layouts/docs.erb | 2 +- 4 files changed, 249 insertions(+), 12 deletions(-) create mode 100644 website/source/docs/builders/amazon-instance.html.markdown create mode 100644 website/source/docs/builders/amazon.html.markdown diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index e04e7615b..3d067bf8c 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -1,22 +1,17 @@ --- layout: "docs" +page_title: "Amazon AMI Builder (EBS backed)" --- -# Amazon AMI Builder +# AMI Builder (EBS backed) Type: `amazon-ebs` The `amazon-ebs` builder is able to create Amazon AMIs backed by EBS -volumes for use in [EC2](http://aws.amazon.com/ec2/). The builder takes -an initial source AMI, runs any provisioning necesary on the instance, -and snapshots it into a reusable AMI. - -Amazon supports two types of AMIs: EBS-backed and instance-store. Instance -store AMIs are considerably harder to create, requiring many platform-specific -steps that can often take a very long time. EBS-backed AMIs, on the hand, -only require a source AMI to exist. This builder only builds EBS-backed -instances, because they are easier to create, especially across many -platforms running Packer. +volumes for use in [EC2](http://aws.amazon.com/ec2/). For more information +on the difference betwen EBS-backed instances and instance-store backed +instances, see the +["storage for the root device" section in the EC2 documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ComponentsAMIs.html#storage-for-the-root-device). This builder builds an AMI by launching an EC2 instance from a source AMI, provisioning that running machine, and then creating an AMI from that machine. diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown new file mode 100644 index 000000000..8f685136c --- /dev/null +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -0,0 +1,217 @@ +--- +layout: "docs" +page_title: "Amazon AMI Builder (instance-store)" +--- + +# AMI Builder (instance-store) + +Type: `amazon-instance` + +The `amazon-instance` builder is able to create Amazon AMIs backed by +instance storage as the root device. For more information on the difference +between instance storage and EBS-backed instances, see the +["storage for the root device" section in the EC2 documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ComponentsAMIs.html#storage-for-the-root-device). + +This builder builds an AMI by launching an EC2 instance from an existing +instance-storage backed AMI, provisioning that running machine, and then +bundling and creating a new AMI from that machine. +This is all done in your own AWS account. The builder will create temporary +keypairs, security group rules, etc. that provide it temporary access to +the instance while the image is being created. This simplifies configuration +quite a bit. + +The builder does _not_ manage AMIs. Once it creates an AMI and stores it +in your account, it is up to you to use, delete, etc. the AMI. + +## Configuration Reference + +There are many configuration options available for the builder. They are +segmented below into two categories: required and optional parameters. Within +each category, the available configuration keys are alphabetized. + +Required: + +* `access_key` (string) - The access key used to communicate with AWS. + If not specified, Packer will attempt to read this from environmental + variables `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` (in that order). + +* `account_id` (string) - Your AWS account ID. This is required for bundling + the AMI. This is _not the same_ as the access key. You can find your + account ID in the security credentials page of your AWS account. + +* `ami_name` (string) - The name of the resulting AMI that will appear + when managing AMIs in the AWS console or via APIs. This must be unique. + To help make this unique, certain template parameters are available for + this value, which are documented below. + +* `instance_type` (string) - The EC2 instance type to use while building + the AMI, such as "m1.small". + +* `region` (string) - The name of the region, such as "us-east-1", in which + to launch the EC2 instance to create the AMI. + +* `s3_bucket` (string) - The name of the S3 bucket to upload the AMI. + This bucket will be created if it doesn't exist. + +* `secret_key` (string) - The secret key used to communicate with AWS. + If not specified, Packer will attempt to read this from environmental + variables `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` (in that order). + +* `source_ami` (string) - The initial AMI used as a base for the newly + created machine. + +* `ssh_username` (string) - The username to use in order to communicate + over SSH to the running machine. + +* `x509_cert_path` (string) - The local path to a valid X509 certificate for + your AWS account. This is used for bundling the AMI. This X509 certificate + must be registered with your account from the security credentials page + in the AWS console. + +* `x509_key_path` (string) - The local path to the private key for the X509 + certificate specified by `x509_cert_path`. This is used for bundling the AMI. + +Optional: + +* `bundle_destination` (string) - The directory on the running instance + where the bundled AMI will be saved prior to uploading. By default this is + "/tmp". This directory must exist and be writable. + +* `bundle_prefix` (string) - The prefix for files created from bundling + the root volume. By default this is "image-{{.Createtime}}". The `CreateTime` + variable should be used to make sure this is unique, otherwise it can + collide with other created AMIs by Packer in your account. + +* `bundle_upload_command` (string) - The command to use to upload the + bundled volume. See the "custom bundle commands" section below for more + information. + +* `bundle_vol_command` (string) - The command to use to bundle the volume. + See the "custom bundle commands" section below for more information. + +* `security_group_id` (string) - The ID (_not_ the name) of the security + group to assign to the instance. By default this is not set and Packer + will automatically create a new temporary security group to allow SSH + access. Note that if this is specified, you must be sure the security + group allows access to the `ssh_port` given below. + +* `ssh_port` (int) - The port that SSH will be available on. This defaults + to port 22. + +* `ssh_timeout` (string) - The time to wait for SSH to become available + before timing out. The format of this value is a duration such as "5s" + or "5m". The default SSH timeout is "1m", or one minute. + +* `subnet_id` (string) - If using VPC, the ID of the subnet, such as + "subnet-12345def", where Packer will launch the EC2 instance. + +* `vpc_id` (string) - If launching into a VPC subnet, Packer needs the + VPC ID in order to create a temporary security group within the VPC. + +* `x509_upload_path` (string) - The path on the remote machine where the + X509 certificate will be uploaded. This path must already exist and be + writable. X509 certificates are uploaded after provisioning is run, so + it is perfectly okay to create this directory as part of the provisioning + process. + +## Basic Example + +Here is a basic example. It is completely valid except for the access keys: + +
+{
+  "type": "amazon-instance",
+  "access_key": "YOUR KEY HERE",
+  "secret_key": "YOUR SECRET KEY HERE",
+  "region": "us-east-1",
+  "source_ami": "ami-d9d6a6b0",
+  "instance_type": "m1.small",
+  "ssh_username": "ubuntu",
+
+  "account_id": "0123-4567-0890",
+  "s3_bucket": "packer-images",
+  "x509_cert_path": "x509.cert",
+  "x509_key_path": "x509.key",
+  "x509_upload_path": "/tmp",
+
+  "ami_name": "packer-quick-start {{.CreateTime}}"
+}
+
+ +
+Note: Packer can also read the access key and secret +access key from environmental variables. See the configuration reference in +the section above for more information on what environmental variables Packer +will look for. +
+ +## AMI Name Variables + +The AMI name specified by the `ami_name` configuration variable is actually +treated as a [configuration template](/docs/templates/configuration-templates.html). +Packer provides a set of variables that it will replace +within the AMI name. This helps ensure the AMI name is unique, as AWS requires. + +The available variables are shown below: + +* `CreateTime` - This will be replaced with the Unix timestamp of when + the AMI was built. + +## Custom Bundle Commands + +A lot of the process required for creating an instance-store backed AMI +involves commands being run on the actual source instance. Specifically, the +`ec2-bundle-vol` and `ec2-upload-bundle` commands must be used to bundle +the root filesystem and upload it, respectively. + +Each of these commands have a lot of available flags. Instead of exposing each +possible flag as a template configuration option, the instance-store AMI +builder for Packer lets you customize the entire command used to bundle +and upload the AMI. + +These are configured with `bundle_vol_command` and `bundle_upload_command`. +Both of these configurations are +[configuration templates](/docs/templates/configuration-templates.html) +and have support for their own set of template variables. + +### Bundle Volume Command + +The default value for `bundle_vol_command` is shown below. It is split +across multiple lines for convenience of reading. The bundle volume command +is responsible for executing `ec2-bundle-vol` in order to store and image +of the root filesystem to use to create the AMI. + +``` +sudo -n ec2-bundle-vol \ + -k {{.KeyPath}} \ + -u {{.AccountId}} \ + -c {{.CertPath}} \ + -r {{.Architecture}} \ + -e {{.PrivatePath}} \ + -d {{.Destination}} \ + -p {{.Prefix}} \ + --batch +``` + +The available template variables should be self-explanatory based on the +parameters they're used to satisfy the `ec2-bundle-vol` command. + +### Bundle Upload Command + +The default value for `bundle_upload_command` is shown below. It is split +across multiple lines for convenience of reading. The bundle upload command +is responsible for taking the bundled volume and uploading it to S3. + +``` +sudo -n ec2-upload-bundle \ + -b {{.BucketName}} \ + -m {{.ManifestPath}} \ + -a {{.AccessKey}} \ + -s {{.SecretKey}} \ + -d {{.BundleDirectory}} \ + --batch \ + --retry +``` + +The available template variables should be self-explanatory based on the +parameters they're used to satisfy the `ec2-upload-bundle` command. diff --git a/website/source/docs/builders/amazon.html.markdown b/website/source/docs/builders/amazon.html.markdown new file mode 100644 index 000000000..b9dc7d2e7 --- /dev/null +++ b/website/source/docs/builders/amazon.html.markdown @@ -0,0 +1,25 @@ +--- +layout: "docs" +page_title: "Amazon AMI Builder" +--- + +# Amazon AMI Builder + +Packer is able to create Amazon AMIs. To achieve this, Packer comes with +multiple builders depending on the strategy you want to use to build the +AMI. Packer supports the following builders at the moment: + +* [amazon-ebs](/docs/builders/amazon-ebs.html) - Create EBS-backed AMIs + by launching a source instance and re-packaging it into a new AMI after + provisioning. If in doubt, use this builder, which is the easiest to get + started with. + +* [amazon-instance](/docs/builders/amazon-instance.html) - Create + instance-store AMIs by launching and provisioning a source instance, then + rebundling it and uploading it to S3. + +
+Don't know which builder to use? If in doubt, use the +amazon-ebs builder. It is +much easier to use and Amazon generally recommends EBS-backed images nowadays. +
diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index e5acb6b6b..b7abfffed 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -27,7 +27,7 @@
  • Builders

  • -
  • Amazon EC2 (AMI)
  • +
  • Amazon EC2 (AMI)
  • DigitalOcean
  • VirtualBox
  • VMware
  • From fc390422832a724352b50f18642d224a22c7f84f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 25 Jul 2013 10:51:38 -0500 Subject: [PATCH 36/36] builder/amazon/instance: default x509_upload_path --- builder/amazon/instance/builder.go | 8 ++++---- builder/amazon/instance/builder_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 2d2a101a5..83e486a49 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -81,6 +81,10 @@ func (b *Builder) Prepare(raws ...interface{}) error { "--batch" } + if b.config.X509UploadPath == "" { + b.config.X509UploadPath = "/tmp" + } + // Accumulate any errors errs := common.CheckUnusedConfig(md) errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) @@ -121,10 +125,6 @@ func (b *Builder) Prepare(raws ...interface{}) error { errs, fmt.Errorf("x509_key_path points to bad file: %s", err)) } - if b.config.X509UploadPath == "" { - errs = packer.MultiErrorAppend(errs, errors.New("x509_upload_path is required")) - } - if errs != nil && len(errs.Errors) > 0 { return errs } diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go index 9633dff84..f643cd9c3 100644 --- a/builder/amazon/instance/builder_test.go +++ b/builder/amazon/instance/builder_test.go @@ -213,7 +213,7 @@ func TestBuilderPrepare_X509UploadPath(t *testing.T) { config["x509_upload_path"] = "" err := b.Prepare(config) - if err == nil { - t.Fatal("should have error") + if err != nil { + t.Fatalf("should not have error: %s", err) } }