diff --git a/CHANGELOG.md b/CHANGELOG.md index b8dd8da1a..c11f240a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ FEATURES: +* New builder: `amazon-instance` can create instance-storage backed + AMIs. * VMware builder now works with Workstation 9 on Linux. IMPROVEMENTS: diff --git a/builder/amazon/common/access_config.go b/builder/amazon/common/access_config.go new file mode 100644 index 000000000..736d2d190 --- /dev/null +++ b/builder/amazon/common/access_config.go @@ -0,0 +1,21 @@ +package common + +import ( + "github.com/mitchellh/goamz/aws" +) + +// AccessConfig is for common configuration related to AWS access +type AccessConfig struct { + AccessKey string `mapstructure:"access_key"` + SecretKey string `mapstructure:"secret_key"` +} + +// Auth returns a valid aws.Auth object for access to AWS services, or +// an error if the authentication couldn't be resolved. +func (c *AccessConfig) Auth() (aws.Auth, error) { + return aws.GetAuth(c.AccessKey, c.SecretKey) +} + +func (c *AccessConfig) Prepare() []error { + return nil +} diff --git a/builder/amazon/common/ami.go b/builder/amazon/common/ami.go new file mode 100644 index 000000000..6a305aa37 --- /dev/null +++ b/builder/amazon/common/ami.go @@ -0,0 +1,25 @@ +package common + +import ( + "github.com/mitchellh/goamz/ec2" + "log" + "time" +) + +// WaitForAMI waits for the given AMI ID to become ready. +func WaitForAMI(c *ec2.EC2, imageId string) error { + for { + imageResp, err := c.Images([]string{imageId}, ec2.NewFilter()) + if err != nil { + return err + } + + if imageResp.Images[0].State == "available" { + return nil + } + + log.Printf("Image in state %s, sleeping 2s before checking again", + imageResp.Images[0].State) + time.Sleep(2 * time.Second) + } +} diff --git a/builder/amazon/ebs/artifact.go b/builder/amazon/common/artifact.go similarity index 54% rename from builder/amazon/ebs/artifact.go rename to builder/amazon/common/artifact.go index 4d25d4595..af6d1f2d4 100644 --- a/builder/amazon/ebs/artifact.go +++ b/builder/amazon/common/artifact.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "fmt" @@ -8,35 +8,39 @@ import ( "strings" ) -type artifact struct { +// Artifact is an artifact implementation that contains built AMIs. +type Artifact struct { // A map of regions to AMI IDs. - amis map[string]string + Amis map[string]string + + // BuilderId is the unique ID for the builder that created this AMI + BuilderIdValue string // EC2 connection for performing API stuff. - conn *ec2.EC2 + Conn *ec2.EC2 } -func (*artifact) BuilderId() string { - return BuilderId +func (a *Artifact) BuilderId() string { + return a.BuilderIdValue } -func (*artifact) Files() []string { +func (*Artifact) Files() []string { // We have no files return nil } -func (a *artifact) Id() string { - parts := make([]string, 0, len(a.amis)) - for region, amiId := range a.amis { +func (a *Artifact) Id() string { + parts := make([]string, 0, len(a.Amis)) + for region, amiId := range a.Amis { parts = append(parts, fmt.Sprintf("%s:%s", region, amiId)) } return strings.Join(parts, ",") } -func (a *artifact) String() string { - amiStrings := make([]string, 0, len(a.amis)) - for region, id := range a.amis { +func (a *Artifact) String() string { + amiStrings := make([]string, 0, len(a.Amis)) + for region, id := range a.Amis { single := fmt.Sprintf("%s: %s", region, id) amiStrings = append(amiStrings, single) } @@ -44,12 +48,12 @@ func (a *artifact) String() string { return fmt.Sprintf("AMIs were created:\n\n%s", strings.Join(amiStrings, "\n")) } -func (a *artifact) Destroy() error { +func (a *Artifact) Destroy() error { errors := make([]error, 0) - for _, imageId := range a.amis { + for _, imageId := range a.Amis { log.Printf("Deregistering image ID: %s", imageId) - if _, err := a.conn.DeregisterImage(imageId); err != nil { + if _, err := a.Conn.DeregisterImage(imageId); err != nil { errors = append(errors, err) } diff --git a/builder/amazon/ebs/artifact_test.go b/builder/amazon/common/artifact_test.go similarity index 84% rename from builder/amazon/ebs/artifact_test.go rename to builder/amazon/common/artifact_test.go index dd8ccba9b..28af553c7 100644 --- a/builder/amazon/ebs/artifact_test.go +++ b/builder/amazon/common/artifact_test.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "cgl.tideland.biz/asserts" @@ -10,7 +10,7 @@ func TestArtifact_Impl(t *testing.T) { assert := asserts.NewTestingAsserts(t, true) var actual packer.Artifact - assert.Implementor(&artifact{}, &actual, "should be an Artifact") + assert.Implementor(&Artifact{}, &actual, "should be an Artifact") } func TestArtifactId(t *testing.T) { @@ -22,7 +22,10 @@ func TestArtifactId(t *testing.T) { amis["east"] = "foo" amis["west"] = "bar" - a := &artifact{amis, nil} + a := &Artifact{ + Amis: amis, + } + result := a.Id() assert.Equal(result, expected, "should match output") } @@ -39,7 +42,7 @@ west: bar` amis["east"] = "foo" amis["west"] = "bar" - a := &artifact{amis, nil} + a := &Artifact{Amis: amis} result := a.String() assert.Equal(result, expected, "should match output") } diff --git a/builder/amazon/ebs/instance.go b/builder/amazon/common/instance.go similarity index 89% rename from builder/amazon/ebs/instance.go rename to builder/amazon/common/instance.go index d5a0a8006..53dab50ce 100644 --- a/builder/amazon/ebs/instance.go +++ b/builder/amazon/common/instance.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "fmt" @@ -7,7 +7,7 @@ import ( "time" ) -func waitForState(ec2conn *ec2.EC2, originalInstance *ec2.Instance, pending []string, target string) (i *ec2.Instance, err error) { +func WaitForState(ec2conn *ec2.EC2, originalInstance *ec2.Instance, pending []string, target string) (i *ec2.Instance, err error) { log.Printf("Waiting for instance state to become: %s", target) i = originalInstance diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go new file mode 100644 index 000000000..bd2186d93 --- /dev/null +++ b/builder/amazon/common/run_config.go @@ -0,0 +1,68 @@ +package common + +import ( + "errors" + "fmt" + "github.com/mitchellh/goamz/aws" + "time" +) + +// RunConfig contains configuration for running an instance from a source +// AMI and details on how to access that launched image. +type RunConfig struct { + Region string + SourceAmi string `mapstructure:"source_ami"` + InstanceType string `mapstructure:"instance_type"` + RawSSHTimeout string `mapstructure:"ssh_timeout"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort int `mapstructure:"ssh_port"` + SecurityGroupId string `mapstructure:"security_group_id"` + SubnetId string `mapstructure:"subnet_id"` + VpcId string `mapstructure:"vpc_id"` + + // Unexported fields that are calculated from others + sshTimeout time.Duration +} + +func (c *RunConfig) Prepare() []error { + // Defaults + if c.SSHPort == 0 { + c.SSHPort = 22 + } + + if c.RawSSHTimeout == "" { + c.RawSSHTimeout = "1m" + } + + // Validation + var err error + errs := make([]error, 0) + if c.SourceAmi == "" { + errs = append(errs, errors.New("A source_ami must be specified")) + } + + if c.InstanceType == "" { + errs = append(errs, errors.New("An instance_type must be specified")) + } + + if c.Region == "" { + errs = append(errs, errors.New("A region must be specified")) + } else if _, ok := aws.Regions[c.Region]; !ok { + errs = append(errs, fmt.Errorf("Unknown region: %s", c.Region)) + } + + if c.SSHUsername == "" { + errs = append(errs, errors.New("An ssh_username must be specified")) + } + + c.sshTimeout, err = time.ParseDuration(c.RawSSHTimeout) + if err != nil { + errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) + } + + return errs +} + +func (c *RunConfig) SSHTimeout() time.Duration { + return c.sshTimeout +} diff --git a/builder/amazon/common/run_config_test.go b/builder/amazon/common/run_config_test.go new file mode 100644 index 000000000..0c0d40680 --- /dev/null +++ b/builder/amazon/common/run_config_test.go @@ -0,0 +1,108 @@ +package common + +import ( + "os" + "testing" +) + +func init() { + // Clear out the AWS access key env vars so they don't + // affect our tests. + os.Setenv("AWS_ACCESS_KEY_ID", "") + os.Setenv("AWS_ACCESS_KEY", "") + os.Setenv("AWS_SECRET_ACCESS_KEY", "") + os.Setenv("AWS_SECRET_KEY", "") +} + +func testConfig() *RunConfig { + return &RunConfig{ + Region: "us-east-1", + SourceAmi: "abcd", + InstanceType: "m1.small", + SSHUsername: "root", + } +} + +func TestRunConfigPrepare(t *testing.T) { + c := testConfig() + err := c.Prepare() + if len(err) > 0 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_InstanceType(t *testing.T) { + c := testConfig() + c.InstanceType = "" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_Region(t *testing.T) { + c := testConfig() + c.Region = "" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } + + c.Region = "us-east-12" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } + + c.Region = "us-east-1" + if err := c.Prepare(); len(err) != 0 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SourceAmi(t *testing.T) { + c := testConfig() + c.SourceAmi = "" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SSHPort(t *testing.T) { + c := testConfig() + c.SSHPort = 0 + if err := c.Prepare(); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.SSHPort != 22 { + t.Fatalf("invalid value: %d", c.SSHPort) + } + + c.SSHPort = 44 + if err := c.Prepare(); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.SSHPort != 44 { + t.Fatalf("invalid value: %d", c.SSHPort) + } +} + +func TestRunConfigPrepare_SSHTimeout(t *testing.T) { + c := testConfig() + c.RawSSHTimeout = "" + if err := c.Prepare(); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + c.RawSSHTimeout = "bad" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SSHUsername(t *testing.T) { + c := testConfig() + c.SSHUsername = "" + if err := c.Prepare(); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} diff --git a/builder/amazon/common/ssh.go b/builder/amazon/common/ssh.go new file mode 100644 index 000000000..96e51cd20 --- /dev/null +++ b/builder/amazon/common/ssh.go @@ -0,0 +1,45 @@ +package common + +import ( + gossh "code.google.com/p/go.crypto/ssh" + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/packer/communicator/ssh" +) + +// SSHAddress returns a function that can be given to the SSH communicator +// for determining the SSH address based on the instance DNS name. +func SSHAddress(port int) func(map[string]interface{}) (string, error) { + return func(state map[string]interface{}) (string, error) { + var host string + instance := state["instance"].(*ec2.Instance) + if instance.VpcId != "" { + host = instance.PrivateIpAddress + } else { + host = instance.DNSName + } + + return fmt.Sprintf("%s:%d", host, port), nil + } +} + +// SSHConfig returns a function that can be used for the SSH communicator +// config for connecting to the instance created over SSH using the generated +// private key. +func SSHConfig(username string) func(map[string]interface{}) (*gossh.ClientConfig, error) { + return func(state map[string]interface{}) (*gossh.ClientConfig, error) { + privateKey := state["privateKey"].(string) + + keyring := new(ssh.SimpleKeychain) + if err := keyring.AddPEMKey(privateKey); err != nil { + return nil, fmt.Errorf("Error setting up SSH config: %s", err) + } + + return &gossh.ClientConfig{ + User: username, + Auth: []gossh.ClientAuth{ + gossh.ClientAuthKeyring(keyring), + }, + }, nil + } +} diff --git a/builder/amazon/ebs/step_keypair.go b/builder/amazon/common/step_key_pair.go similarity index 81% rename from builder/amazon/ebs/step_keypair.go rename to builder/amazon/common/step_key_pair.go index 498dcb65c..3e49e3d32 100644 --- a/builder/amazon/ebs/step_keypair.go +++ b/builder/amazon/common/step_key_pair.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "cgl.tideland.biz/identifier" @@ -10,11 +10,11 @@ import ( "log" ) -type stepKeyPair struct { +type StepKeyPair struct { keyName string } -func (s *stepKeyPair) Run(state map[string]interface{}) multistep.StepAction { +func (s *StepKeyPair) Run(state map[string]interface{}) multistep.StepAction { ec2conn := state["ec2"].(*ec2.EC2) ui := state["ui"].(packer.Ui) @@ -23,9 +23,7 @@ func (s *stepKeyPair) Run(state map[string]interface{}) multistep.StepAction { log.Printf("temporary keypair name: %s", keyName) keyResp, err := ec2conn.CreateKeyPair(keyName) if err != nil { - err := fmt.Errorf("Error creating temporary keypair: %s", err) - state["error"] = err - ui.Error(err.Error()) + state["error"] = fmt.Errorf("Error creating temporary keypair: %s", err) return multistep.ActionHalt } @@ -39,7 +37,7 @@ func (s *stepKeyPair) Run(state map[string]interface{}) multistep.StepAction { return multistep.ActionContinue } -func (s *stepKeyPair) Cleanup(state map[string]interface{}) { +func (s *StepKeyPair) Cleanup(state map[string]interface{}) { // If no key name is set, then we never created it, so just return if s.keyName == "" { return diff --git a/builder/amazon/ebs/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go similarity index 69% rename from builder/amazon/ebs/step_run_source_instance.go rename to builder/amazon/common/step_run_source_instance.go index ada21de20..2c7254b49 100644 --- a/builder/amazon/ebs/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "fmt" @@ -8,12 +8,16 @@ import ( "log" ) -type stepRunSourceInstance struct { +type StepRunSourceInstance struct { + ExpectedRootDevice string + InstanceType string + SourceAMI string + SubnetId string + instance *ec2.Instance } -func (s *stepRunSourceInstance) Run(state map[string]interface{}) multistep.StepAction { - config := state["config"].(config) +func (s *StepRunSourceInstance) Run(state map[string]interface{}) multistep.StepAction { ec2conn := state["ec2"].(*ec2.EC2) keyName := state["keyPair"].(string) securityGroupId := state["securityGroupId"].(string) @@ -21,25 +25,26 @@ func (s *stepRunSourceInstance) Run(state map[string]interface{}) multistep.Step runOpts := &ec2.RunInstances{ KeyName: keyName, - ImageId: config.SourceAmi, - InstanceType: config.InstanceType, + ImageId: s.SourceAMI, + InstanceType: s.InstanceType, MinCount: 0, MaxCount: 0, SecurityGroups: []ec2.SecurityGroup{ec2.SecurityGroup{Id: securityGroupId}}, - SubnetId: config.SubnetId, + SubnetId: s.SubnetId, } ui.Say("Launching a source AWS instance...") - imageResp, err := ec2conn.Images([]string{config.SourceAmi}, ec2.NewFilter()) + imageResp, err := ec2conn.Images([]string{s.SourceAMI}, ec2.NewFilter()) if err != nil { state["error"] = fmt.Errorf("There was a problem with the source AMI: %s", err) return multistep.ActionHalt } - if imageResp.Images[0].RootDeviceType != "ebs" { + if s.ExpectedRootDevice != "" && imageResp.Images[0].RootDeviceType != s.ExpectedRootDevice { state["error"] = fmt.Errorf( - "The provided source AMI is instance-store based. The\n" + - "amazon-ebs bundler can only work with EBS based AMIs.") + "The provided source AMI has an invalid root device type.\n"+ + "Expected '%s', got '%s'.", + s.ExpectedRootDevice, imageResp.Images[0].RootDeviceType) return multistep.ActionHalt } @@ -55,7 +60,7 @@ func (s *stepRunSourceInstance) Run(state map[string]interface{}) multistep.Step log.Printf("instance id: %s", s.instance.InstanceId) ui.Say(fmt.Sprintf("Waiting for instance (%s) to become ready...", s.instance.InstanceId)) - s.instance, err = waitForState(ec2conn, s.instance, []string{"pending"}, "running") + s.instance, err = WaitForState(ec2conn, s.instance, []string{"pending"}, "running") if err != nil { err := fmt.Errorf("Error waiting for instance (%s) to become ready: %s", s.instance.InstanceId, err) state["error"] = err @@ -68,7 +73,7 @@ func (s *stepRunSourceInstance) Run(state map[string]interface{}) multistep.Step return multistep.ActionContinue } -func (s *stepRunSourceInstance) Cleanup(state map[string]interface{}) { +func (s *StepRunSourceInstance) Cleanup(state map[string]interface{}) { if s.instance == nil { return } @@ -83,5 +88,5 @@ func (s *stepRunSourceInstance) Cleanup(state map[string]interface{}) { } pending := []string{"pending", "running", "shutting-down", "stopped", "stopping"} - waitForState(ec2conn, s.instance, pending, "terminated") + WaitForState(ec2conn, s.instance, pending, "terminated") } diff --git a/builder/amazon/ebs/step_security_group.go b/builder/amazon/common/step_security_group.go similarity index 61% rename from builder/amazon/ebs/step_security_group.go rename to builder/amazon/common/step_security_group.go index 4a056a1b3..5b01a68f1 100644 --- a/builder/amazon/ebs/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -1,4 +1,4 @@ -package ebs +package common import ( "cgl.tideland.biz/identifier" @@ -10,40 +10,52 @@ import ( "log" ) -type stepSecurityGroup struct { - groupId string +type StepSecurityGroup struct { + SecurityGroupId string + SSHPort int + VpcId string + + createdGroupId string } -func (s *stepSecurityGroup) Run(state map[string]interface{}) multistep.StepAction { - config := state["config"].(config) +func (s *StepSecurityGroup) Run(state map[string]interface{}) multistep.StepAction { ec2conn := state["ec2"].(*ec2.EC2) ui := state["ui"].(packer.Ui) - if config.SecurityGroupId != "" { - log.Printf("Using specified security group: %s", config.SecurityGroupId) - state["securityGroupId"] = config.SecurityGroupId + if s.SecurityGroupId != "" { + log.Printf("Using specified security group: %s", s.SecurityGroupId) + state["securityGroupId"] = s.SecurityGroupId return multistep.ActionContinue } + if s.SSHPort == 0 { + panic("SSHPort must be set to a non-zero value.") + } + // Create the group ui.Say("Creating temporary security group for this instance...") groupName := fmt.Sprintf("packer %s", hex.EncodeToString(identifier.NewUUID().Raw())) log.Printf("Temporary group name: %s", groupName) - groupResp, err := ec2conn.CreateSecurityGroup(ec2.SecurityGroup{Name: groupName, Description: "Temporary group for Packer", VpcId: config.VpcId}) + group := ec2.SecurityGroup{ + Name: groupName, + Description: "Temporary group for Packer", + VpcId: s.VpcId, + } + groupResp, err := ec2conn.CreateSecurityGroup(group) if err != nil { ui.Error(err.Error()) return multistep.ActionHalt } // Set the group ID so we can delete it later - s.groupId = groupResp.Id + s.createdGroupId = groupResp.Id // Authorize the SSH access perms := []ec2.IPPerm{ ec2.IPPerm{ Protocol: "tcp", - FromPort: config.SSHPort, - ToPort: config.SSHPort, + FromPort: s.SSHPort, + ToPort: s.SSHPort, SourceIPs: []string{"0.0.0.0/0"}, }, } @@ -57,13 +69,13 @@ func (s *stepSecurityGroup) Run(state map[string]interface{}) multistep.StepActi } // Set some state data for use in future steps - state["securityGroupId"] = s.groupId + state["securityGroupId"] = s.createdGroupId return multistep.ActionContinue } -func (s *stepSecurityGroup) Cleanup(state map[string]interface{}) { - if s.groupId == "" { +func (s *StepSecurityGroup) Cleanup(state map[string]interface{}) { + if s.createdGroupId == "" { return } @@ -71,10 +83,10 @@ func (s *stepSecurityGroup) Cleanup(state map[string]interface{}) { ui := state["ui"].(packer.Ui) ui.Say("Deleting temporary security group...") - _, err := ec2conn.DeleteSecurityGroup(ec2.SecurityGroup{Id: s.groupId}) + _, err := ec2conn.DeleteSecurityGroup(ec2.SecurityGroup{Id: s.createdGroupId}) if err != nil { log.Printf("Error deleting security group: %s", err) ui.Error(fmt.Sprintf( - "Error cleaning up security group. Please delete the group manually: %s", s.groupId)) + "Error cleaning up security group. Please delete the group manually: %s", s.createdGroupId)) } } diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 6063f995a..46f5231fc 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -11,39 +11,23 @@ import ( "github.com/mitchellh/goamz/aws" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" "github.com/mitchellh/packer/builder/common" "github.com/mitchellh/packer/packer" "log" "text/template" - "time" ) // The unique ID for this builder const BuilderId = "mitchellh.amazonebs" type config struct { - // Access information - AccessKey string `mapstructure:"access_key"` - SecretKey string `mapstructure:"secret_key"` - - // Information for the source instance - Region string - SourceAmi string `mapstructure:"source_ami"` - InstanceType string `mapstructure:"instance_type"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort int `mapstructure:"ssh_port"` - SecurityGroupId string `mapstructure:"security_group_id"` - VpcId string `mapstructure:"vpc_id"` - SubnetId string `mapstructure:"subnet_id"` + common.PackerConfig `mapstructure:",squash"` + awscommon.AccessConfig `mapstructure:",squash"` + awscommon.RunConfig `mapstructure:",squash"` // Configuration of the resulting AMI AMIName string `mapstructure:"ami_name"` - - PackerDebug bool `mapstructure:"packer_debug"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - - // Unexported fields that are calculated from others - sshTimeout time.Duration } type Builder struct { @@ -59,45 +43,10 @@ func (b *Builder) Prepare(raws ...interface{}) error { // Accumulate any errors errs := common.CheckUnusedConfig(md) - - if b.config.SSHPort == 0 { - b.config.SSHPort = 22 - } - - if b.config.RawSSHTimeout == "" { - b.config.RawSSHTimeout = "1m" - } + errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) + errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare()...) // Accumulate any errors - if b.config.SourceAmi == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("A source_ami must be specified")) - } - - if b.config.InstanceType == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("An instance_type must be specified")) - } - - if b.config.Region == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("A region must be specified")) - } else if _, ok := aws.Regions[b.config.Region]; !ok { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Unknown region: %s", b.config.Region)) - } - - if b.config.SSHUsername == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("An ssh_username must be specified")) - } - - b.config.sshTimeout, err = time.ParseDuration(b.config.RawSSHTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - if b.config.AMIName == "" { errs = packer.MultiErrorAppend( errs, errors.New("ami_name must be specified")) @@ -123,7 +72,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe panic("region not found") } - auth, err := aws.GetAuth(b.config.AccessKey, b.config.SecretKey) + auth, err := b.config.AccessConfig.Auth() if err != nil { return nil, err } @@ -139,13 +88,22 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ - &stepKeyPair{}, - &stepSecurityGroup{}, - &stepRunSourceInstance{}, + &awscommon.StepKeyPair{}, + &awscommon.StepSecurityGroup{ + SecurityGroupId: b.config.SecurityGroupId, + SSHPort: b.config.SSHPort, + VpcId: b.config.VpcId, + }, + &awscommon.StepRunSourceInstance{ + ExpectedRootDevice: "ebs", + InstanceType: b.config.InstanceType, + SourceAMI: b.config.SourceAmi, + SubnetId: b.config.SubnetId, + }, &common.StepConnectSSH{ - SSHAddress: sshAddress, - SSHConfig: sshConfig, - SSHWaitTimeout: b.config.sshTimeout, + SSHAddress: awscommon.SSHAddress(b.config.SSHPort), + SSHConfig: awscommon.SSHConfig(b.config.SSHUsername), + SSHWaitTimeout: b.config.SSHTimeout(), }, &common.StepProvision{}, &stepStopInstance{}, @@ -175,9 +133,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } // Build the artifact and return it - artifact := &artifact{ - amis: state["amis"].(map[string]string), - conn: ec2conn, + artifact := &awscommon.Artifact{ + Amis: state["amis"].(map[string]string), + BuilderIdValue: BuilderId, + Conn: ec2conn, } return artifact, nil diff --git a/builder/amazon/ebs/builder_test.go b/builder/amazon/ebs/builder_test.go index ea0d326b5..3ad6d17b6 100644 --- a/builder/amazon/ebs/builder_test.go +++ b/builder/amazon/ebs/builder_test.go @@ -2,19 +2,9 @@ package ebs import ( "github.com/mitchellh/packer/packer" - "os" "testing" ) -func init() { - // Clear out the AWS access key env vars so they don't - // affect our tests. - os.Setenv("AWS_ACCESS_KEY_ID", "") - os.Setenv("AWS_ACCESS_KEY", "") - os.Setenv("AWS_SECRET_ACCESS_KEY", "") - os.Setenv("AWS_SECRET_KEY", "") -} - func testConfig() map[string]interface{} { return map[string]interface{}{ "access_key": "foo", @@ -75,30 +65,6 @@ func TestBuilderPrepare_AMIName(t *testing.T) { } } -func TestBuilderPrepare_InstanceType(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["instance_type"] = "foo" - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.InstanceType != "foo" { - t.Errorf("invalid: %s", b.config.InstanceType) - } - - // Test bad - delete(config, "instance_type") - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } -} - func TestBuilderPrepare_InvalidKey(t *testing.T) { var b Builder config := testConfig() @@ -110,129 +76,3 @@ func TestBuilderPrepare_InvalidKey(t *testing.T) { t.Fatal("should have error") } } - -func TestBuilderPrepare_Region(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["region"] = "us-east-1" - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.Region != "us-east-1" { - t.Errorf("invalid: %s", b.config.Region) - } - - // Test bad - delete(config, "region") - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } - - // Test invalid - config["region"] = "i-am-not-real" - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } -} - -func TestBuilderPrepare_SourceAmi(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["source_ami"] = "foo" - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.SourceAmi != "foo" { - t.Errorf("invalid: %s", b.config.SourceAmi) - } - - // Test bad - delete(config, "source_ami") - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } -} - -func TestBuilderPrepare_SSHPort(t *testing.T) { - var b Builder - config := testConfig() - - // Test default - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.SSHPort != 22 { - t.Errorf("invalid: %d", b.config.SSHPort) - } - - // Test set - config["ssh_port"] = 35 - b = Builder{} - err = b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.SSHPort != 35 { - t.Errorf("invalid: %d", b.config.SSHPort) - } -} - -func TestBuilderPrepare_SSHTimeout(t *testing.T) { - var b Builder - config := testConfig() - - // Test with a bad value - config["ssh_timeout"] = "this is not good" - err := b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } - - // Test with a good one - config["ssh_timeout"] = "5s" - err = b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } -} - -func TestBuilderPrepare_SSHUsername(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["ssh_username"] = "foo" - err := b.Prepare(config) - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.SSHUsername != "foo" { - t.Errorf("invalid: %s", b.config.SSHUsername) - } - - // Test bad - delete(config, "ssh_username") - b = Builder{} - err = b.Prepare(config) - if err == nil { - t.Fatal("should have error") - } -} diff --git a/builder/amazon/ebs/ssh.go b/builder/amazon/ebs/ssh.go deleted file mode 100644 index 599bf5ad9..000000000 --- a/builder/amazon/ebs/ssh.go +++ /dev/null @@ -1,35 +0,0 @@ -package ebs - -import ( - gossh "code.google.com/p/go.crypto/ssh" - "fmt" - "github.com/mitchellh/goamz/ec2" - "github.com/mitchellh/packer/communicator/ssh" -) - -func sshAddress(state map[string]interface{}) (string, error) { - config := state["config"].(config) - instance := state["instance"].(*ec2.Instance) - if config.VpcId == "" { - return fmt.Sprintf("%s:%d", instance.DNSName, config.SSHPort), nil - } else { - return fmt.Sprintf("%s:%d", instance.PrivateIpAddress, config.SSHPort), nil - } -} - -func sshConfig(state map[string]interface{}) (*gossh.ClientConfig, error) { - config := state["config"].(config) - privateKey := state["privateKey"].(string) - - keyring := new(ssh.SimpleKeychain) - if err := keyring.AddPEMKey(privateKey); err != nil { - return nil, fmt.Errorf("Error setting up SSH config: %s", err) - } - - return &gossh.ClientConfig{ - User: config.SSHUsername, - Auth: []gossh.ClientAuth{ - gossh.ClientAuthKeyring(keyring), - }, - }, nil -} diff --git a/builder/amazon/ebs/step_create_ami.go b/builder/amazon/ebs/step_create_ami.go index a33021b49..0b08bc7c9 100644 --- a/builder/amazon/ebs/step_create_ami.go +++ b/builder/amazon/ebs/step_create_ami.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" "github.com/mitchellh/packer/packer" - "log" "strconv" "text/template" "time" @@ -57,23 +57,11 @@ func (s *stepCreateAMI) Run(state map[string]interface{}) multistep.StepAction { // Wait for the image to become ready ui.Say("Waiting for AMI to become ready...") - for { - imageResp, err := ec2conn.Images([]string{createResp.ImageId}, ec2.NewFilter()) - if err != nil { - err := fmt.Errorf("Error querying images: %s", err) - state["error"] = err - ui.Error(err.Error()) - return multistep.ActionHalt - } - - if imageResp.Images[0].State == "available" { - break - } - - log.Printf("Image in state %s, sleeping 2s before checking again", - imageResp.Images[0].State) - - time.Sleep(2 * time.Second) + if err := awscommon.WaitForAMI(ec2conn, createResp.ImageId); err != nil { + err := fmt.Errorf("Error waiting for AMI: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt } return multistep.ActionContinue diff --git a/builder/amazon/ebs/step_stop_instance.go b/builder/amazon/ebs/step_stop_instance.go index ac81bda6b..5c608632f 100644 --- a/builder/amazon/ebs/step_stop_instance.go +++ b/builder/amazon/ebs/step_stop_instance.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" "github.com/mitchellh/packer/packer" ) @@ -26,7 +27,7 @@ func (s *stepStopInstance) Run(state map[string]interface{}) multistep.StepActio // Wait for the instance to actual stop ui.Say("Waiting for the instance to stop...") - instance, err = waitForState(ec2conn, instance, []string{"running", "stopping"}, "stopped") + instance, err = awscommon.WaitForState(ec2conn, instance, []string{"running", "stopping"}, "stopped") if err != nil { err := fmt.Errorf("Error waiting for instance to stop: %s", err) state["error"] = err diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go new file mode 100644 index 000000000..83e486a49 --- /dev/null +++ b/builder/amazon/instance/builder.go @@ -0,0 +1,219 @@ +// The instance package contains a packer.Builder implementation that builds +// AMIs for Amazon EC2 backed by instance storage, as opposed to EBS storage. +package instance + +import ( + "errors" + "fmt" + "github.com/mitchellh/goamz/aws" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" + "github.com/mitchellh/packer/builder/common" + "github.com/mitchellh/packer/packer" + "log" + "os" + "strings" + "text/template" +) + +// The unique ID for this builder +const BuilderId = "mitchellh.amazon.instance" + +// Config is the configuration that is chained through the steps and +// settable from the template. +type Config struct { + common.PackerConfig `mapstructure:",squash"` + awscommon.AccessConfig `mapstructure:",squash"` + awscommon.RunConfig `mapstructure:",squash"` + + AccountId string `mapstructure:"account_id"` + AMIName string `mapstructure:"ami_name"` + BundleDestination string `mapstructure:"bundle_destination"` + BundlePrefix string `mapstructure:"bundle_prefix"` + BundleUploadCommand string `mapstructure:"bundle_upload_command"` + BundleVolCommand string `mapstructure:"bundle_vol_command"` + S3Bucket string `mapstructure:"s3_bucket"` + X509CertPath string `mapstructure:"x509_cert_path"` + X509KeyPath string `mapstructure:"x509_key_path"` + X509UploadPath string `mapstructure:"x509_upload_path"` +} + +type Builder struct { + config Config + runner multistep.Runner +} + +func (b *Builder) Prepare(raws ...interface{}) error { + md, err := common.DecodeConfig(&b.config, raws...) + if err != nil { + return err + } + + if b.config.BundleDestination == "" { + b.config.BundleDestination = "/tmp" + } + + if b.config.BundlePrefix == "" { + b.config.BundlePrefix = "image-{{.CreateTime}}" + } + + if b.config.BundleUploadCommand == "" { + b.config.BundleUploadCommand = "sudo -n ec2-upload-bundle " + + "-b {{.BucketName}} " + + "-m {{.ManifestPath}} " + + "-a {{.AccessKey}} " + + "-s {{.SecretKey}} " + + "-d {{.BundleDirectory}} " + + "--batch " + + "--retry" + } + + if b.config.BundleVolCommand == "" { + b.config.BundleVolCommand = "sudo -n ec2-bundle-vol " + + "-k {{.KeyPath}} " + + "-u {{.AccountId}} " + + "-c {{.CertPath}} " + + "-r {{.Architecture}} " + + "-e {{.PrivatePath}} " + + "-d {{.Destination}} " + + "-p {{.Prefix}} " + + "--batch" + } + + if b.config.X509UploadPath == "" { + b.config.X509UploadPath = "/tmp" + } + + // Accumulate any errors + errs := common.CheckUnusedConfig(md) + errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) + errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare()...) + + if b.config.AccountId == "" { + errs = packer.MultiErrorAppend(errs, errors.New("account_id is required")) + } else { + b.config.AccountId = strings.Replace(b.config.AccountId, "-", "", -1) + } + + if b.config.AMIName == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("ami_name must be specified")) + } else { + _, err = template.New("ami").Parse(b.config.AMIName) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Failed parsing ami_name: %s", err)) + } + } + + if b.config.S3Bucket == "" { + errs = packer.MultiErrorAppend(errs, errors.New("s3_bucket is required")) + } + + if b.config.X509CertPath == "" { + errs = packer.MultiErrorAppend(errs, errors.New("x509_cert_path is required")) + } else if _, err := os.Stat(b.config.X509CertPath); err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("x509_cert_path points to bad file: %s", err)) + } + + if b.config.X509KeyPath == "" { + errs = packer.MultiErrorAppend(errs, errors.New("x509_key_path is required")) + } else if _, err := os.Stat(b.config.X509KeyPath); err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("x509_key_path points to bad file: %s", err)) + } + + if errs != nil && len(errs.Errors) > 0 { + return errs + } + + log.Printf("Config: %+v", b.config) + return nil +} + +func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { + region, ok := aws.Regions[b.config.Region] + if !ok { + panic("region not found") + } + + auth, err := b.config.AccessConfig.Auth() + if err != nil { + return nil, err + } + + ec2conn := ec2.New(auth, region) + + // Setup the state bag and initial state for the steps + state := make(map[string]interface{}) + state["config"] = &b.config + state["ec2"] = ec2conn + state["hook"] = hook + state["ui"] = ui + + // Build the steps + steps := []multistep.Step{ + &awscommon.StepKeyPair{}, + &awscommon.StepSecurityGroup{ + SecurityGroupId: b.config.SecurityGroupId, + SSHPort: b.config.SSHPort, + VpcId: b.config.VpcId, + }, + &awscommon.StepRunSourceInstance{ + ExpectedRootDevice: "instance-store", + InstanceType: b.config.InstanceType, + SourceAMI: b.config.SourceAmi, + SubnetId: b.config.SubnetId, + }, + &common.StepConnectSSH{ + SSHAddress: awscommon.SSHAddress(b.config.SSHPort), + SSHConfig: awscommon.SSHConfig(b.config.SSHUsername), + SSHWaitTimeout: b.config.SSHTimeout(), + }, + &common.StepProvision{}, + &StepUploadX509Cert{}, + &StepBundleVolume{}, + &StepUploadBundle{}, + &StepRegisterAMI{}, + } + + // Run! + if b.config.PackerDebug { + b.runner = &multistep.DebugRunner{ + Steps: steps, + PauseFn: common.MultistepDebugFn(ui), + } + } else { + b.runner = &multistep.BasicRunner{Steps: steps} + } + + b.runner.Run(state) + + // If there was an error, return that + if rawErr, ok := state["error"]; ok { + return nil, rawErr.(error) + } + + // If there are no AMIs, then just return + if _, ok := state["amis"]; !ok { + return nil, nil + } + + // Build the artifact and return it + artifact := &awscommon.Artifact{ + Amis: state["amis"].(map[string]string), + BuilderIdValue: BuilderId, + Conn: ec2conn, + } + + return artifact, nil +} + +func (b *Builder) Cancel() { + if b.runner != nil { + log.Println("Cancelling the step runner...") + b.runner.Cancel() + } +} diff --git a/builder/amazon/instance/builder_test.go b/builder/amazon/instance/builder_test.go new file mode 100644 index 000000000..f643cd9c3 --- /dev/null +++ b/builder/amazon/instance/builder_test.go @@ -0,0 +1,219 @@ +package instance + +import ( + "github.com/mitchellh/packer/packer" + "io/ioutil" + "os" + "testing" +) + +func testConfig() map[string]interface{} { + tf, err := ioutil.TempFile("", "packer") + if err != nil { + panic(err) + } + + return map[string]interface{}{ + "account_id": "foo", + "ami_name": "foo", + "instance_type": "m1.small", + "region": "us-east-1", + "s3_bucket": "foo", + "source_ami": "foo", + "ssh_username": "bob", + "x509_cert_path": tf.Name(), + "x509_key_path": tf.Name(), + "x509_upload_path": "/foo", + } +} + +func TestBuilder_ImplementsBuilder(t *testing.T) { + var raw interface{} + raw = &Builder{} + if _, ok := raw.(packer.Builder); !ok { + t.Fatalf("Builder should be a builder") + } +} + +func TestBuilderPrepare_AccountId(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["account_id"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["account_id"] = "foo" + err = b.Prepare(config) + if err != nil { + t.Errorf("err: %s", err) + } + + config["account_id"] = "0123-0456-7890" + err = b.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + if b.config.AccountId != "012304567890" { + t.Errorf("should strip hyphens: %s", b.config.AccountId) + } +} + +func TestBuilderPrepare_AMIName(t *testing.T) { + var b Builder + config := testConfig() + + // Test good + config["ami_name"] = "foo" + err := b.Prepare(config) + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + // Test bad + config["ami_name"] = "foo {{" + b = Builder{} + err = b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + // Test bad + delete(config, "ami_name") + b = Builder{} + err = b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } +} + +func TestBuilderPrepare_BundleDestination(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["bundle_destination"] = "" + err := b.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + if b.config.BundleDestination != "/tmp" { + t.Fatalf("bad: %s", b.config.BundleDestination) + } +} + +func TestBuilderPrepare_BundlePrefix(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["bundle_prefix"] = "" + err := b.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + if b.config.BundlePrefix != "image-{{.CreateTime}}" { + t.Fatalf("bad: %s", b.config.BundlePrefix) + } +} + +func TestBuilderPrepare_InvalidKey(t *testing.T) { + var b Builder + config := testConfig() + + // Add a random key + config["i_should_not_be_valid"] = true + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } +} + +func TestBuilderPrepare_S3Bucket(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["s3_bucket"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["s3_bucket"] = "foo" + err = b.Prepare(config) + if err != nil { + t.Errorf("err: %s", err) + } +} + +func TestBuilderPrepare_X509CertPath(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["x509_cert_path"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["x509_cert_path"] = "i/am/a/file/that/doesnt/exist" + err = b.Prepare(config) + if err == nil { + t.Error("should have error") + } + + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("error tempfile: %s", err) + } + defer os.Remove(tf.Name()) + + config["x509_cert_path"] = tf.Name() + err = b.Prepare(config) + if err != nil { + t.Fatalf("should not have error: %s", err) + } +} + +func TestBuilderPrepare_X509KeyPath(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["x509_key_path"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["x509_key_path"] = "i/am/a/file/that/doesnt/exist" + err = b.Prepare(config) + if err == nil { + t.Error("should have error") + } + + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("error tempfile: %s", err) + } + defer os.Remove(tf.Name()) + + config["x509_key_path"] = tf.Name() + err = b.Prepare(config) + if err != nil { + t.Fatalf("should not have error: %s", err) + } +} + +func TestBuilderPrepare_X509UploadPath(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["x509_upload_path"] = "" + err := b.Prepare(config) + if err != nil { + t.Fatalf("should not have error: %s", err) + } +} diff --git a/builder/amazon/instance/step_bundle_volume.go b/builder/amazon/instance/step_bundle_volume.go new file mode 100644 index 000000000..f5da54dde --- /dev/null +++ b/builder/amazon/instance/step_bundle_volume.go @@ -0,0 +1,104 @@ +package instance + +import ( + "bytes" + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "strconv" + "text/template" + "time" +) + +type bundleCmdData struct { + AccountId string + Architecture string + CertPath string + Destination string + KeyPath string + Prefix string + PrivatePath string +} + +type bundlePrefixData struct { + CreateTime string +} + +type StepBundleVolume struct{} + +func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepAction { + comm := state["communicator"].(packer.Communicator) + config := state["config"].(*Config) + instance := state["instance"].(*ec2.Instance) + ui := state["ui"].(packer.Ui) + x509RemoteCertPath := state["x509RemoteCertPath"].(string) + x509RemoteKeyPath := state["x509RemoteKeyPath"].(string) + + // Verify the AMI tools are available + ui.Say("Checking for EC2 AMI tools...") + cmd := &packer.RemoteCmd{Command: "ec2-ami-tools-version"} + if err := comm.Start(cmd); err != nil { + state["error"] = fmt.Errorf("Error checking for AMI tools: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + cmd.Wait() + + if cmd.ExitStatus != 0 { + state["error"] = fmt.Errorf( + "The EC2 AMI tools could not be detected. These must be manually\n" + + "via a provisioner or some other means and are required for Packer\n" + + "to create an instance-store AMI.") + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + // Bundle the volume + var bundlePrefix bytes.Buffer + prefixTData := bundlePrefixData{ + CreateTime: strconv.FormatInt(time.Now().UTC().Unix(), 10), + } + t := template.Must(template.New("bundlePrefix").Parse(config.BundlePrefix)) + t.Execute(&bundlePrefix, prefixTData) + + var bundleCmd bytes.Buffer + tData := bundleCmdData{ + AccountId: config.AccountId, + Architecture: instance.Architecture, + CertPath: x509RemoteCertPath, + Destination: config.BundleDestination, + KeyPath: x509RemoteKeyPath, + Prefix: bundlePrefix.String(), + PrivatePath: config.X509UploadPath, + } + t = template.Must(template.New("bundleCmd").Parse(config.BundleVolCommand)) + t.Execute(&bundleCmd, tData) + + ui.Say("Bundling the volume...") + cmd = new(packer.RemoteCmd) + cmd.Command = bundleCmd.String() + if err := cmd.StartWithUi(comm, ui); err != nil { + state["error"] = fmt.Errorf("Error bundling volume: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + if cmd.ExitStatus != 0 { + state["error"] = fmt.Errorf( + "Volume bundling failed. Please see the output above for more\n" + + "details on what went wrong.") + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + // Store the manifest path + manifestName := bundlePrefix.String() + ".manifest.xml" + state["manifest_name"] = manifestName + state["manifest_path"] = fmt.Sprintf( + "%s/%s", config.BundleDestination, manifestName) + + return multistep.ActionContinue +} + +func (s *StepBundleVolume) Cleanup(map[string]interface{}) {} diff --git a/builder/amazon/instance/step_register_ami.go b/builder/amazon/instance/step_register_ami.go new file mode 100644 index 000000000..d95deb888 --- /dev/null +++ b/builder/amazon/instance/step_register_ami.go @@ -0,0 +1,68 @@ +package instance + +import ( + "bytes" + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" + "github.com/mitchellh/packer/packer" + "strconv" + "text/template" + "time" +) + +type amiNameData struct { + CreateTime string +} + +type StepRegisterAMI struct{} + +func (s *StepRegisterAMI) Run(state map[string]interface{}) multistep.StepAction { + config := state["config"].(*Config) + ec2conn := state["ec2"].(*ec2.EC2) + manifestPath := state["remote_manifest_path"].(string) + ui := state["ui"].(packer.Ui) + + // Parse the name of the AMI + amiNameBuf := new(bytes.Buffer) + tData := amiNameData{ + strconv.FormatInt(time.Now().UTC().Unix(), 10), + } + + t := template.Must(template.New("ami").Parse(config.AMIName)) + t.Execute(amiNameBuf, tData) + amiName := amiNameBuf.String() + + ui.Say("Registering the AMI...") + registerOpts := &ec2.RegisterImage{ + ImageLocation: manifestPath, + Name: amiName, + } + + registerResp, err := ec2conn.RegisterImage(registerOpts) + if err != nil { + state["error"] = fmt.Errorf("Error registering AMI: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + // Set the AMI ID in the state + ui.Say(fmt.Sprintf("AMI: %s", registerResp.ImageId)) + amis := make(map[string]string) + amis[config.Region] = registerResp.ImageId + state["amis"] = amis + + // Wait for the image to become ready + ui.Say("Waiting for AMI to become ready...") + if err := awscommon.WaitForAMI(ec2conn, registerResp.ImageId); err != nil { + err := fmt.Errorf("Error waiting for AMI: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *StepRegisterAMI) Cleanup(map[string]interface{}) {} diff --git a/builder/amazon/instance/step_upload_bundle.go b/builder/amazon/instance/step_upload_bundle.go new file mode 100644 index 000000000..8c8a9fdb5 --- /dev/null +++ b/builder/amazon/instance/step_upload_bundle.go @@ -0,0 +1,61 @@ +package instance + +import ( + "bytes" + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "text/template" +) + +type uploadCmdData struct { + AccessKey string + BucketName string + BundleDirectory string + ManifestPath string + SecretKey string +} + +type StepUploadBundle struct{} + +func (s *StepUploadBundle) Run(state map[string]interface{}) multistep.StepAction { + comm := state["communicator"].(packer.Communicator) + config := state["config"].(*Config) + manifestName := state["manifest_name"].(string) + manifestPath := state["manifest_path"].(string) + ui := state["ui"].(packer.Ui) + + var uploadCmd bytes.Buffer + tData := uploadCmdData{ + AccessKey: config.AccessKey, + BucketName: config.S3Bucket, + BundleDirectory: config.BundleDestination, + ManifestPath: manifestPath, + SecretKey: config.SecretKey, + } + t := template.Must(template.New("uploadCmd").Parse(config.BundleUploadCommand)) + t.Execute(&uploadCmd, tData) + + ui.Say("Uploading the bundle...") + cmd := &packer.RemoteCmd{Command: uploadCmd.String()} + if err := cmd.StartWithUi(comm, ui); err != nil { + state["error"] = fmt.Errorf("Error uploading volume: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + if cmd.ExitStatus != 0 { + state["error"] = fmt.Errorf( + "Bundle upload failed. Please see the output above for more\n" + + "details on what went wrong.") + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + state["remote_manifest_path"] = fmt.Sprintf( + "%s/%s", config.S3Bucket, manifestName) + + return multistep.ActionContinue +} + +func (s *StepUploadBundle) Cleanup(state map[string]interface{}) {} diff --git a/builder/amazon/instance/step_upload_x509_cert.go b/builder/amazon/instance/step_upload_x509_cert.go new file mode 100644 index 000000000..5660bc228 --- /dev/null +++ b/builder/amazon/instance/step_upload_x509_cert.go @@ -0,0 +1,49 @@ +package instance + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "os" +) + +type StepUploadX509Cert struct{} + +func (s *StepUploadX509Cert) Run(state map[string]interface{}) multistep.StepAction { + comm := state["communicator"].(packer.Communicator) + config := state["config"].(*Config) + ui := state["ui"].(packer.Ui) + + x509RemoteCertPath := config.X509UploadPath + "/cert.pem" + x509RemoteKeyPath := config.X509UploadPath + "/key.pem" + + ui.Say("Uploading X509 Certificate...") + if err := s.uploadSingle(comm, x509RemoteCertPath, config.X509CertPath); err != nil { + state["error"] = fmt.Errorf("Error uploading X509 cert: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + if err := s.uploadSingle(comm, x509RemoteKeyPath, config.X509KeyPath); err != nil { + state["error"] = fmt.Errorf("Error uploading X509 cert: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + state["x509RemoteCertPath"] = x509RemoteCertPath + state["x509RemoteKeyPath"] = x509RemoteKeyPath + + return multistep.ActionContinue +} + +func (s *StepUploadX509Cert) Cleanup(map[string]interface{}) {} + +func (s *StepUploadX509Cert) uploadSingle(comm packer.Communicator, dst, src string) error { + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + return comm.Upload(dst, f) +} diff --git a/builder/common/packer_config.go b/builder/common/packer_config.go new file mode 100644 index 000000000..f51b92e0b --- /dev/null +++ b/builder/common/packer_config.go @@ -0,0 +1,10 @@ +package common + +// PackerConfig is a struct that contains the configuration keys that +// are sent by packer, properly tagged already so mapstructure can load +// them. Embed this structure into your configuration class to get it. +type PackerConfig struct { + PackerBuildName string `mapstructure:"packer_build_name"` + PackerDebug bool `mapstructure:"packer_debug"` + PackerForce bool `mapstructure:"packer_force"` +} diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index ae79f7c24..c4ab8c5bc 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -28,6 +28,8 @@ type snapshotNameData struct { // to use while communicating with DO and describes the image // you are creating type config struct { + common.PackerConfig `mapstructure:",squash"` + ClientID string `mapstructure:"client_id"` APIKey string `mapstructure:"api_key"` RegionID uint `mapstructure:"region_id"` @@ -38,8 +40,6 @@ type config struct { SSHUsername string `mapstructure:"ssh_username"` SSHPort uint `mapstructure:"ssh_port"` - PackerDebug bool `mapstructure:"packer_debug"` - RawSnapshotName string `mapstructure:"snapshot_name"` RawSSHTimeout string `mapstructure:"ssh_timeout"` RawEventDelay string `mapstructure:"event_delay"` diff --git a/builder/virtualbox/builder.go b/builder/virtualbox/builder.go index af6ffd35a..25a2855f8 100644 --- a/builder/virtualbox/builder.go +++ b/builder/virtualbox/builder.go @@ -24,6 +24,8 @@ type Builder struct { } type config struct { + common.PackerConfig `mapstructure:",squash"` + BootCommand []string `mapstructure:"boot_command"` DiskSize uint `mapstructure:"disk_size"` FloppyFiles []string `mapstructure:"floppy_files"` @@ -49,10 +51,6 @@ type config struct { VBoxManage [][]string `mapstructure:"vboxmanage"` VMName string `mapstructure:"vm_name"` - PackerBuildName string `mapstructure:"packer_build_name"` - PackerDebug bool `mapstructure:"packer_debug"` - PackerForce bool `mapstructure:"packer_force"` - RawBootWait string `mapstructure:"boot_wait"` RawShutdownTimeout string `mapstructure:"shutdown_timeout"` RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` diff --git a/builder/vmware/builder.go b/builder/vmware/builder.go index d2e2ba3ed..48d19a3cf 100644 --- a/builder/vmware/builder.go +++ b/builder/vmware/builder.go @@ -25,6 +25,8 @@ type Builder struct { } type config struct { + common.PackerConfig `mapstructure:",squash"` + DiskName string `mapstructure:"vmdk_name"` DiskSize uint `mapstructure:"disk_size"` FloppyFiles []string `mapstructure:"floppy_files"` @@ -50,10 +52,6 @@ type config struct { VNCPortMin uint `mapstructure:"vnc_port_min"` VNCPortMax uint `mapstructure:"vnc_port_max"` - PackerBuildName string `mapstructure:"packer_build_name"` - PackerDebug bool `mapstructure:"packer_debug"` - PackerForce bool `mapstructure:"packer_force"` - RawBootWait string `mapstructure:"boot_wait"` RawShutdownTimeout string `mapstructure:"shutdown_timeout"` RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` diff --git a/config.go b/config.go index 12a78d300..4660fad1c 100644 --- a/config.go +++ b/config.go @@ -20,6 +20,7 @@ const defaultConfig = ` "builders": { "amazon-ebs": "packer-builder-amazon-ebs", + "amazon-instance": "packer-builder-amazon-instance", "digitalocean": "packer-builder-digitalocean", "virtualbox": "packer-builder-virtualbox", "vmware": "packer-builder-vmware" diff --git a/plugin/builder-amazon-instance/main.go b/plugin/builder-amazon-instance/main.go new file mode 100644 index 000000000..fc1c66b40 --- /dev/null +++ b/plugin/builder-amazon-instance/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/mitchellh/packer/builder/amazon/instance" + "github.com/mitchellh/packer/packer/plugin" +) + +func main() { + plugin.ServeBuilder(new(instance.Builder)) +} diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index e04e7615b..3d067bf8c 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -1,22 +1,17 @@ --- layout: "docs" +page_title: "Amazon AMI Builder (EBS backed)" --- -# Amazon AMI Builder +# AMI Builder (EBS backed) Type: `amazon-ebs` The `amazon-ebs` builder is able to create Amazon AMIs backed by EBS -volumes for use in [EC2](http://aws.amazon.com/ec2/). The builder takes -an initial source AMI, runs any provisioning necesary on the instance, -and snapshots it into a reusable AMI. - -Amazon supports two types of AMIs: EBS-backed and instance-store. Instance -store AMIs are considerably harder to create, requiring many platform-specific -steps that can often take a very long time. EBS-backed AMIs, on the hand, -only require a source AMI to exist. This builder only builds EBS-backed -instances, because they are easier to create, especially across many -platforms running Packer. +volumes for use in [EC2](http://aws.amazon.com/ec2/). For more information +on the difference betwen EBS-backed instances and instance-store backed +instances, see the +["storage for the root device" section in the EC2 documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ComponentsAMIs.html#storage-for-the-root-device). This builder builds an AMI by launching an EC2 instance from a source AMI, provisioning that running machine, and then creating an AMI from that machine. diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown new file mode 100644 index 000000000..8f685136c --- /dev/null +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -0,0 +1,217 @@ +--- +layout: "docs" +page_title: "Amazon AMI Builder (instance-store)" +--- + +# AMI Builder (instance-store) + +Type: `amazon-instance` + +The `amazon-instance` builder is able to create Amazon AMIs backed by +instance storage as the root device. For more information on the difference +between instance storage and EBS-backed instances, see the +["storage for the root device" section in the EC2 documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ComponentsAMIs.html#storage-for-the-root-device). + +This builder builds an AMI by launching an EC2 instance from an existing +instance-storage backed AMI, provisioning that running machine, and then +bundling and creating a new AMI from that machine. +This is all done in your own AWS account. The builder will create temporary +keypairs, security group rules, etc. that provide it temporary access to +the instance while the image is being created. This simplifies configuration +quite a bit. + +The builder does _not_ manage AMIs. Once it creates an AMI and stores it +in your account, it is up to you to use, delete, etc. the AMI. + +## Configuration Reference + +There are many configuration options available for the builder. They are +segmented below into two categories: required and optional parameters. Within +each category, the available configuration keys are alphabetized. + +Required: + +* `access_key` (string) - The access key used to communicate with AWS. + If not specified, Packer will attempt to read this from environmental + variables `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` (in that order). + +* `account_id` (string) - Your AWS account ID. This is required for bundling + the AMI. This is _not the same_ as the access key. You can find your + account ID in the security credentials page of your AWS account. + +* `ami_name` (string) - The name of the resulting AMI that will appear + when managing AMIs in the AWS console or via APIs. This must be unique. + To help make this unique, certain template parameters are available for + this value, which are documented below. + +* `instance_type` (string) - The EC2 instance type to use while building + the AMI, such as "m1.small". + +* `region` (string) - The name of the region, such as "us-east-1", in which + to launch the EC2 instance to create the AMI. + +* `s3_bucket` (string) - The name of the S3 bucket to upload the AMI. + This bucket will be created if it doesn't exist. + +* `secret_key` (string) - The secret key used to communicate with AWS. + If not specified, Packer will attempt to read this from environmental + variables `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` (in that order). + +* `source_ami` (string) - The initial AMI used as a base for the newly + created machine. + +* `ssh_username` (string) - The username to use in order to communicate + over SSH to the running machine. + +* `x509_cert_path` (string) - The local path to a valid X509 certificate for + your AWS account. This is used for bundling the AMI. This X509 certificate + must be registered with your account from the security credentials page + in the AWS console. + +* `x509_key_path` (string) - The local path to the private key for the X509 + certificate specified by `x509_cert_path`. This is used for bundling the AMI. + +Optional: + +* `bundle_destination` (string) - The directory on the running instance + where the bundled AMI will be saved prior to uploading. By default this is + "/tmp". This directory must exist and be writable. + +* `bundle_prefix` (string) - The prefix for files created from bundling + the root volume. By default this is "image-{{.Createtime}}". The `CreateTime` + variable should be used to make sure this is unique, otherwise it can + collide with other created AMIs by Packer in your account. + +* `bundle_upload_command` (string) - The command to use to upload the + bundled volume. See the "custom bundle commands" section below for more + information. + +* `bundle_vol_command` (string) - The command to use to bundle the volume. + See the "custom bundle commands" section below for more information. + +* `security_group_id` (string) - The ID (_not_ the name) of the security + group to assign to the instance. By default this is not set and Packer + will automatically create a new temporary security group to allow SSH + access. Note that if this is specified, you must be sure the security + group allows access to the `ssh_port` given below. + +* `ssh_port` (int) - The port that SSH will be available on. This defaults + to port 22. + +* `ssh_timeout` (string) - The time to wait for SSH to become available + before timing out. The format of this value is a duration such as "5s" + or "5m". The default SSH timeout is "1m", or one minute. + +* `subnet_id` (string) - If using VPC, the ID of the subnet, such as + "subnet-12345def", where Packer will launch the EC2 instance. + +* `vpc_id` (string) - If launching into a VPC subnet, Packer needs the + VPC ID in order to create a temporary security group within the VPC. + +* `x509_upload_path` (string) - The path on the remote machine where the + X509 certificate will be uploaded. This path must already exist and be + writable. X509 certificates are uploaded after provisioning is run, so + it is perfectly okay to create this directory as part of the provisioning + process. + +## Basic Example + +Here is a basic example. It is completely valid except for the access keys: + +
+{ + "type": "amazon-instance", + "access_key": "YOUR KEY HERE", + "secret_key": "YOUR SECRET KEY HERE", + "region": "us-east-1", + "source_ami": "ami-d9d6a6b0", + "instance_type": "m1.small", + "ssh_username": "ubuntu", + + "account_id": "0123-4567-0890", + "s3_bucket": "packer-images", + "x509_cert_path": "x509.cert", + "x509_key_path": "x509.key", + "x509_upload_path": "/tmp", + + "ami_name": "packer-quick-start {{.CreateTime}}" +} ++ +