From eef3223f6cee2d1642c42402ab238dd486665671 Mon Sep 17 00:00:00 2001 From: Peter Leschev Date: Thu, 27 Mar 2014 17:11:34 +1100 Subject: [PATCH 001/208] Adding the ability to skip nat port forwarding for ssh connectivity --- builder/virtualbox/common/ssh_config.go | 1 + builder/virtualbox/common/step_export.go | 29 ++++---- builder/virtualbox/common/step_forward_ssh.go | 70 ++++++++++--------- builder/virtualbox/iso/builder.go | 14 ++-- builder/virtualbox/ovf/builder.go | 14 ++-- .../builders/virtualbox-iso.html.markdown | 4 ++ .../builders/virtualbox-ovf.html.markdown | 4 ++ 7 files changed, 80 insertions(+), 56 deletions(-) diff --git a/builder/virtualbox/common/ssh_config.go b/builder/virtualbox/common/ssh_config.go index 00c6167c6..5da8ad403 100644 --- a/builder/virtualbox/common/ssh_config.go +++ b/builder/virtualbox/common/ssh_config.go @@ -16,6 +16,7 @@ type SSHConfig struct { SSHPort uint `mapstructure:"ssh_port"` SSHUser string `mapstructure:"ssh_username"` RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` + SSHSkipNatMapping bool `mapstructure:"ssh_skip_nat_mapping"` SSHWaitTimeout time.Duration } diff --git a/builder/virtualbox/common/step_export.go b/builder/virtualbox/common/step_export.go index e4e860155..0a3cd816c 100644 --- a/builder/virtualbox/common/step_export.go +++ b/builder/virtualbox/common/step_export.go @@ -17,9 +17,10 @@ import ( // Produces: // exportPath string - The path to the resulting export. type StepExport struct { - Format string - OutputDir string - ExportOpts []string + Format string + OutputDir string + ExportOpts []string + SkipNatMapping bool } func (s *StepExport) Run(state multistep.StateBag) multistep.StepAction { @@ -33,15 +34,19 @@ func (s *StepExport) Run(state multistep.StateBag) multistep.StepAction { // Clear out the Packer-created forwarding rule ui.Say("Preparing to export machine...") - ui.Message(fmt.Sprintf( - "Deleting forwarded port mapping for SSH (host port %d)", - state.Get("sshHostPort"))) - command := []string{"modifyvm", vmName, "--natpf1", "delete", "packerssh"} - if err := driver.VBoxManage(command...); err != nil { - err := fmt.Errorf("Error deleting port forwarding rule: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt + var command []string + + if s.SkipNatMapping == false { + ui.Message(fmt.Sprintf( + "Deleting forwarded port mapping for SSH (host port %d)", + state.Get("sshHostPort"))) + command := []string{"modifyvm", vmName, "--natpf1", "delete", "packerssh"} + if err := driver.VBoxManage(command...); err != nil { + err := fmt.Errorf("Error deleting port forwarding rule: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } } // Export the VM to an OVF diff --git a/builder/virtualbox/common/step_forward_ssh.go b/builder/virtualbox/common/step_forward_ssh.go index 862432952..d6d604e00 100644 --- a/builder/virtualbox/common/step_forward_ssh.go +++ b/builder/virtualbox/common/step_forward_ssh.go @@ -19,9 +19,10 @@ import ( // // Produces: type StepForwardSSH struct { - GuestPort uint - HostPortMin uint - HostPortMax uint + GuestPort uint + HostPortMin uint + HostPortMax uint + SkipNatMapping bool } func (s *StepForwardSSH) Run(state multistep.StateBag) multistep.StepAction { @@ -29,39 +30,44 @@ func (s *StepForwardSSH) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) vmName := state.Get("vmName").(string) - log.Printf("Looking for available SSH port between %d and %d", - s.HostPortMin, s.HostPortMax) var sshHostPort uint - var offset uint = 0 + if s.SkipNatMapping { + sshHostPort = s.GuestPort + log.Printf("Skipping SSH NAT mapping and using SSH port %d", sshHostPort) + } else { + log.Printf("Looking for available SSH port between %d and %d", + s.HostPortMin, s.HostPortMax) + var offset uint = 0 - portRange := int(s.HostPortMax - s.HostPortMin) - if portRange > 0 { - // Have to check if > 0 to avoid a panic - offset = uint(rand.Intn(portRange)) - } - - for { - sshHostPort = offset + s.HostPortMin - log.Printf("Trying port: %d", sshHostPort) - l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", sshHostPort)) - if err == nil { - defer l.Close() - break + portRange := int(s.HostPortMax - s.HostPortMin) + if portRange > 0 { + // Have to check if > 0 to avoid a panic + offset = uint(rand.Intn(portRange)) } - } - // Create a forwarded port mapping to the VM - ui.Say(fmt.Sprintf("Creating forwarded port mapping for SSH (host port %d)", sshHostPort)) - command := []string{ - "modifyvm", vmName, - "--natpf1", - fmt.Sprintf("packerssh,tcp,127.0.0.1,%d,,%d", sshHostPort, s.GuestPort), - } - if err := driver.VBoxManage(command...); err != nil { - err := fmt.Errorf("Error creating port forwarding rule: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt + for { + sshHostPort = offset + s.HostPortMin + log.Printf("Trying port: %d", sshHostPort) + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", sshHostPort)) + if err == nil { + defer l.Close() + break + } + } + + // Create a forwarded port mapping to the VM + ui.Say(fmt.Sprintf("Creating forwarded port mapping for SSH (host port %d)", sshHostPort)) + command := []string{ + "modifyvm", vmName, + "--natpf1", + fmt.Sprintf("packerssh,tcp,127.0.0.1,%d,,%d", sshHostPort, s.GuestPort), + } + if err := driver.VBoxManage(command...); err != nil { + err := fmt.Errorf("Error creating port forwarding rule: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } } // Save the port we're using so that future steps can use it diff --git a/builder/virtualbox/iso/builder.go b/builder/virtualbox/iso/builder.go index 31660a961..7ec34bfe8 100644 --- a/builder/virtualbox/iso/builder.go +++ b/builder/virtualbox/iso/builder.go @@ -290,9 +290,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe new(stepAttachGuestAdditions), new(vboxcommon.StepAttachFloppy), &vboxcommon.StepForwardSSH{ - GuestPort: b.config.SSHPort, - HostPortMin: b.config.SSHHostPortMin, - HostPortMax: b.config.SSHHostPortMax, + GuestPort: b.config.SSHPort, + HostPortMin: b.config.SSHHostPortMin, + HostPortMax: b.config.SSHHostPortMax, + SkipNatMapping: b.config.SSHSkipNatMapping, }, &vboxcommon.StepVBoxManage{ Commands: b.config.VBoxManage, @@ -319,9 +320,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, new(vboxcommon.StepRemoveDevices), &vboxcommon.StepExport{ - Format: b.config.Format, - OutputDir: b.config.OutputDir, - ExportOpts: b.config.ExportOpts.ExportOpts, + Format: b.config.Format, + OutputDir: b.config.OutputDir, + ExportOpts: b.config.ExportOpts.ExportOpts, + SkipNatMapping: b.config.SSHSkipNatMapping, }, } diff --git a/builder/virtualbox/ovf/builder.go b/builder/virtualbox/ovf/builder.go index f2e6a92f5..65c4fe72a 100644 --- a/builder/virtualbox/ovf/builder.go +++ b/builder/virtualbox/ovf/builder.go @@ -65,9 +65,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe */ new(vboxcommon.StepAttachFloppy), &vboxcommon.StepForwardSSH{ - GuestPort: b.config.SSHPort, - HostPortMin: b.config.SSHHostPortMin, - HostPortMax: b.config.SSHHostPortMax, + GuestPort: b.config.SSHPort, + HostPortMin: b.config.SSHHostPortMin, + HostPortMax: b.config.SSHHostPortMax, + SkipNatMapping: b.config.SSHSkipNatMapping, }, &vboxcommon.StepVBoxManage{ Commands: b.config.VBoxManage, @@ -95,9 +96,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, new(vboxcommon.StepRemoveDevices), &vboxcommon.StepExport{ - Format: b.config.Format, - OutputDir: b.config.OutputDir, - ExportOpts: b.config.ExportOpts.ExportOpts, + Format: b.config.Format, + OutputDir: b.config.OutputDir, + ExportOpts: b.config.ExportOpts.ExportOpts, + SkipNatMapping: b.config.SSHSkipNatMapping, }, } diff --git a/website/source/docs/builders/virtualbox-iso.html.markdown b/website/source/docs/builders/virtualbox-iso.html.markdown index e6e3b2454..3c2e87e7f 100644 --- a/website/source/docs/builders/virtualbox-iso.html.markdown +++ b/website/source/docs/builders/virtualbox-iso.html.markdown @@ -191,6 +191,10 @@ Optional: available. By default this is "20m", or 20 minutes. Note that this should be quite long since the timer begins as soon as the virtual machine is booted. +* `ssh_skip_nat_mapping` (bool) - Defaults to false. When enabled, Packer does + not setup forwarded port mapping for SSH requests and uses `ssh_port` on the + host to communicate to the virtual machine + * `vboxmanage` (array of array of strings) - Custom `VBoxManage` commands to execute in order to further customize the virtual machine being created. The value of this is an array of commands to execute. The commands are executed diff --git a/website/source/docs/builders/virtualbox-ovf.html.markdown b/website/source/docs/builders/virtualbox-ovf.html.markdown index 6bf6e49d9..43ce70339 100644 --- a/website/source/docs/builders/virtualbox-ovf.html.markdown +++ b/website/source/docs/builders/virtualbox-ovf.html.markdown @@ -126,6 +126,10 @@ Optional: available. By default this is "20m", or 20 minutes. Note that this should be quite long since the timer begins as soon as the virtual machine is booted. +* `ssh_skip_nat_mapping` (bool) - Defaults to false. When enabled, Packer does + not setup forwarded port mapping for SSH requests and uses `ssh_port` on the + host to communicate to the virtual machine + * `vboxmanage` (array of array of strings) - Custom `VBoxManage` commands to execute in order to further customize the virtual machine being created. The value of this is an array of commands to execute. The commands are executed From fd3b4ecc25865f6464b583595c32c27edc08a93a Mon Sep 17 00:00:00 2001 From: Andrew Beresford Date: Fri, 7 Nov 2014 10:55:31 +0000 Subject: [PATCH 002/208] Add 1/10th second delay between key events to VNC --- builder/vmware/common/step_type_boot_command.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builder/vmware/common/step_type_boot_command.go b/builder/vmware/common/step_type_boot_command.go index 82e8b3e17..f67940280 100644 --- a/builder/vmware/common/step_type_boot_command.go +++ b/builder/vmware/common/step_type_boot_command.go @@ -200,7 +200,9 @@ func vncSendString(c *vnc.ClientConn, original string) { } c.KeyEvent(keyCode, true) + time.Sleep(time.Second/10) c.KeyEvent(keyCode, false) + time.Sleep(time.Second/10) if keyShift { c.KeyEvent(KeyLeftShift, false) From 42d05368ae45b98548fb327d760ce9447f316c53 Mon Sep 17 00:00:00 2001 From: Colin Hebert Date: Sat, 10 Jan 2015 23:52:45 +1100 Subject: [PATCH 003/208] Save the generated SSH key as a file in debug mode --- builder/digitalocean/builder.go | 5 +++- builder/digitalocean/step_create_ssh_key.go | 31 ++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index cf76f6970..055861984 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -244,7 +244,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ - new(stepCreateSSHKey), + &stepCreateSSHKey{ + Debug: b.config.PackerDebug, + DebugKeyPath: fmt.Sprintf("do_%s.pem", b.config.PackerBuildName), + }, new(stepCreateDroplet), new(stepDropletInfo), &common.StepConnectSSH{ diff --git a/builder/digitalocean/step_create_ssh_key.go b/builder/digitalocean/step_create_ssh_key.go index 78bb474c1..ade19d7b3 100644 --- a/builder/digitalocean/step_create_ssh_key.go +++ b/builder/digitalocean/step_create_ssh_key.go @@ -7,6 +7,8 @@ import ( "encoding/pem" "fmt" "log" + "runtime" + "os" "code.google.com/p/gosshold/ssh" "github.com/mitchellh/multistep" @@ -15,7 +17,9 @@ import ( ) type stepCreateSSHKey struct { - keyId uint + Debug bool + DebugKeyPath string + keyId uint } func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { @@ -62,6 +66,31 @@ func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { // Remember some state for the future state.Put("ssh_key_id", keyId) + // If we're in debug mode, output the private key to the working directory. + if s.Debug { + ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath)) + f, err := os.Create(s.DebugKeyPath) + if err != nil { + state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) + return multistep.ActionHalt + } + defer f.Close() + + // Write the key out + if _, err := f.Write(pem.EncodeToMemory(&priv_blk)); err != nil { + state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) + return multistep.ActionHalt + } + + // Chmod it so that it is SSH ready + if runtime.GOOS != "windows" { + if err := f.Chmod(0600); err != nil { + state.Put("error", fmt.Errorf("Error setting permissions of debug key: %s", err)) + return multistep.ActionHalt + } + } + } + return multistep.ActionContinue } From 729a0c01c7d75a0224370925b34923d61a0aa850 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Sat, 24 Jan 2015 17:25:47 -0600 Subject: [PATCH 004/208] Updates to Into: Build An Image docs - the listed AMI isn't found - t2.micros can only be in a VPC(? so said an error), and the docs say we're using a t1.micro anyway - Updates to the vagrant file to get the website to build --- website/Vagrantfile | 2 ++ .../source/intro/getting-started/build-image.html.markdown | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/website/Vagrantfile b/website/Vagrantfile index b8da4b2db..0cdf78d94 100644 --- a/website/Vagrantfile +++ b/website/Vagrantfile @@ -6,6 +6,8 @@ sudo apt-get -y update # RVM/Ruby sudo apt-get -y install curl +sudo apt-get -y install git +gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 curl -sSL https://get.rvm.io | bash -s stable . ~/.bashrc . ~/.bash_profile diff --git a/website/source/intro/getting-started/build-image.html.markdown b/website/source/intro/getting-started/build-image.html.markdown index 1cda08a7c..b17b5f823 100644 --- a/website/source/intro/getting-started/build-image.html.markdown +++ b/website/source/intro/getting-started/build-image.html.markdown @@ -54,8 +54,8 @@ briefly. Create a file `example.json` and fill it with the following contents: "access_key": "{{user `aws_access_key`}}", "secret_key": "{{user `aws_secret_key`}}", "region": "us-east-1", - "source_ami": "ami-9eaa1cf6", - "instance_type": "t2.micro", + "source_ami": "ami-c65be9ae", + "instance_type": "t1.micro", "ssh_username": "ubuntu", "ami_name": "packer-example {{timestamp}}" }] From 2184892f8a4a4da56c6cfc14dd039f099a7bc258 Mon Sep 17 00:00:00 2001 From: "Billie H. Cleek" Date: Thu, 12 Feb 2015 20:18:54 -0800 Subject: [PATCH 005/208] do not request a pty Change the default behavior from requesting a PTY when executing a command with the ssh communicator to requesting a PTY only when configured to do so. Update the vmware builders to be fully backward compatible with the new behavior. --- builder/vmware/iso/builder.go | 2 +- builder/vmware/iso/driver_esx5.go | 1 - builder/vmware/vmx/builder.go | 2 +- common/step_connect_ssh.go | 6 +++--- communicator/ssh/communicator.go | 6 +++--- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go index 674b308c9..661a7ed00 100644 --- a/builder/vmware/iso/builder.go +++ b/builder/vmware/iso/builder.go @@ -349,7 +349,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SSHAddress: driver.SSHAddress, SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), SSHWaitTimeout: b.config.SSHWaitTimeout, - NoPty: b.config.SSHSkipRequestPty, + Pty: !b.config.SSHSkipRequestPty, }, &vmwcommon.StepUploadTools{ RemoteType: b.config.RemoteType, diff --git a/builder/vmware/iso/driver_esx5.go b/builder/vmware/iso/driver_esx5.go index 1f9bd7a78..59c6dcd29 100644 --- a/builder/vmware/iso/driver_esx5.go +++ b/builder/vmware/iso/driver_esx5.go @@ -334,7 +334,6 @@ func (d *ESX5Driver) connect() error { User: d.Username, Auth: auth, }, - NoPty: true, } comm, err := ssh.New(address, sshConfig) diff --git a/builder/vmware/vmx/builder.go b/builder/vmware/vmx/builder.go index 4597e647b..216f45486 100644 --- a/builder/vmware/vmx/builder.go +++ b/builder/vmware/vmx/builder.go @@ -94,7 +94,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SSHAddress: driver.SSHAddress, SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), SSHWaitTimeout: b.config.SSHWaitTimeout, - NoPty: b.config.SSHSkipRequestPty, + Pty: !b.config.SSHSkipRequestPty, }, &vmwcommon.StepUploadTools{ RemoteType: b.config.RemoteType, diff --git a/common/step_connect_ssh.go b/common/step_connect_ssh.go index b00d5bfc0..352d59ace 100644 --- a/common/step_connect_ssh.go +++ b/common/step_connect_ssh.go @@ -34,8 +34,8 @@ type StepConnectSSH struct { // SSHWaitTimeout is the total timeout to wait for SSH to become available. SSHWaitTimeout time.Duration - // NoPty, if true, will not request a Pty from the remote end. - NoPty bool + // Pty, if true, will request a Pty from the remote end. + Pty bool comm packer.Communicator } @@ -138,7 +138,7 @@ func (s *StepConnectSSH) waitForSSH(state multistep.StateBag, cancel <-chan stru config := &ssh.Config{ Connection: connFunc, SSHConfig: sshConfig, - NoPty: s.NoPty, + Pty: s.Pty, } log.Println("Attempting SSH connection...") diff --git a/communicator/ssh/communicator.go b/communicator/ssh/communicator.go index 07fb1eaa2..dfe178361 100644 --- a/communicator/ssh/communicator.go +++ b/communicator/ssh/communicator.go @@ -33,8 +33,8 @@ type Config struct { // case an error occurs. Connection func() (net.Conn, error) - // NoPty, if true, will not request a pty from the remote end. - NoPty bool + // Pty, if true, will request a pty from the remote end. + Pty bool } // Creates a new packer.Communicator implementation over SSH. This takes @@ -65,7 +65,7 @@ func (c *comm) Start(cmd *packer.RemoteCmd) (err error) { session.Stdout = cmd.Stdout session.Stderr = cmd.Stderr - if !c.config.NoPty { + if c.config.Pty { // Request a PTY termModes := ssh.TerminalModes{ ssh.ECHO: 0, // do not echo From 2a9dc513ab19d3f6cdc4ec0c12ddb82964067d77 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 26 Feb 2015 16:35:31 -0800 Subject: [PATCH 006/208] Fix statement in machine-readable 'ui' type docs. The text previously stated that "ui"-type messages represent messages that would be shown if Packer is *not* running in human-readable mode. This is rather talking about what would happen when Packer *is* using human-readable mode. --- website/source/docs/machine-readable/general.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/machine-readable/general.html.markdown b/website/source/docs/machine-readable/general.html.markdown index 05eb2699a..1f08be4d2 100644 --- a/website/source/docs/machine-readable/general.html.markdown +++ b/website/source/docs/machine-readable/general.html.markdown @@ -15,7 +15,7 @@ machine-readable output and are provided by Packer core itself.

Specifies the output and type of output that would've normally - gone to the console if Packer wasn't running in human-readable + gone to the console if Packer were running in human-readable mode.

From 361e859556ace480bf6dcec922ac0a0662b6388e Mon Sep 17 00:00:00 2001 From: Ash Caire Date: Sun, 1 Mar 2015 10:20:37 +1100 Subject: [PATCH 007/208] Add EBS snapshot tags --- builder/amazon/common/step_create_tags.go | 25 +++++++++++++++++-- test/builder_amazon_ebs.bats | 18 +++++++++++++ .../amazon-ebs/ami_snapshot_tags.json | 20 +++++++++++++++ .../docs/builders/amazon-ebs.html.markdown | 3 ++- 4 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 test/fixtures/amazon-ebs/ami_snapshot_tags.json diff --git a/builder/amazon/common/step_create_tags.go b/builder/amazon/common/step_create_tags.go index a204ca321..1d8b80ca7 100644 --- a/builder/amazon/common/step_create_tags.go +++ b/builder/amazon/common/step_create_tags.go @@ -19,7 +19,28 @@ func (s *StepCreateTags) Run(state multistep.StateBag) multistep.StepAction { if len(s.Tags) > 0 { for region, ami := range amis { - ui.Say(fmt.Sprintf("Adding tags to AMI (%s)...", ami)) + ui.Say(fmt.Sprintf("Preparing tags for AMI (%s) and related snapshots", ami)) + + // Declare list of resources to tag + resourceIds := []string{ami} + + // Retrieve image list for given AMI + imageResp, err := ec2conn.Images([]string{ami}, ec2.NewFilter()) + if err != nil { + err := fmt.Errorf("Error retrieving details for AMI (%s): %s", ami, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + image := &imageResp.Images[0] + + // Add only those with a Snapshot ID, i.e. not Ephemeral + for _, device := range image.BlockDevices { + if device.SnapshotId != "" { + ui.Say(fmt.Sprintf("Tagging snapshot: %s", device.SnapshotId)) + resourceIds = append(resourceIds, device.SnapshotId) + } + } var ec2Tags []ec2.Tag for key, value := range s.Tags { @@ -28,7 +49,7 @@ func (s *StepCreateTags) Run(state multistep.StateBag) multistep.StepAction { } regionconn := ec2.New(ec2conn.Auth, aws.Regions[region]) - _, err := regionconn.CreateTags([]string{ami}, ec2Tags) + _, err = regionconn.CreateTags(resourceIds, ec2Tags) if err != nil { err := fmt.Errorf("Error adding tags to AMI (%s): %s", ami, err) state.Put("error", err) diff --git a/test/builder_amazon_ebs.bats b/test/builder_amazon_ebs.bats index 7dd17acbd..89f32a4a0 100755 --- a/test/builder_amazon_ebs.bats +++ b/test/builder_amazon_ebs.bats @@ -15,6 +15,18 @@ aws_ami_region_copy_count() { | wc -l } +# This verifies AMI tags are correctly applied to relevant snapshots +aws_ami_snapshot_tags_count() { + filter='Name=tag:packer-id,Values=ami_snapshot_tags' + aws ec2 describe-images --region $1 --owners self --output text \ + --filters "$filter" \ + --query "Images[*].BlockDeviceMappings[*].Ebs.SnapshotId" \ + | aws ec2 describe-snapshots --region $1 --owners self --output text \ + --filters "$filter" \ + --snapshot-ids \ + | wc -l +} + teardown() { aws_ami_cleanup 'us-east-1' aws_ami_cleanup 'us-west-1' @@ -34,3 +46,9 @@ teardown() { [ "$(aws_ami_region_copy_count 'us-west-1')" -eq "1" ] [ "$(aws_ami_region_copy_count 'us-west-2')" -eq "1" ] } + +@test "amazon-ebs: AMI snapshot tags" { + run packer build $FIXTURE_ROOT/ami_snapshot_tags.json + [ "$status" -eq 0 ] + [ "$(aws_ami_snapshot_tags)" -eq "2" ] +} diff --git a/test/fixtures/amazon-ebs/ami_snapshot_tags.json b/test/fixtures/amazon-ebs/ami_snapshot_tags.json new file mode 100644 index 000000000..278474a32 --- /dev/null +++ b/test/fixtures/amazon-ebs/ami_snapshot_tags.json @@ -0,0 +1,20 @@ +{ + "builders": [{ + "type": "amazon-ebs", + "ami_name": "packer-test {{timestamp}}", + "instance_type": "m1.small", + "region": "us-east-1", + "ssh_username": "ubuntu", + "source_ami": "ami-0568456c", + "tags": { + "packer-test": "true", + "packer-id": "ami_snapshot_tags" + }, + "ami_block_device_mappings": [ + { + "device_name": "/dev/sde", + "volume_type": "standard" + } + ] + }] +} diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index 5e2c31e90..946b18eb9 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -146,7 +146,8 @@ each category, the available configuration keys are alphabetized. * `subnet_id` (string) - If using VPC, the ID of the subnet, such as "subnet-12345def", where Packer will launch the EC2 instance. -* `tags` (object of key/value strings) - Tags applied to the AMI. +* `tags` (object of key/value strings) - Tags applied to the AMI and + relevant snapshots. * `temporary_key_pair_name` (string) - The name of the temporary keypair to generate. By default, Packer generates a name with a UUID. From f9c14aee90d9df48e9c4e7ce5c221ce4af3747ff Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Mon, 2 Mar 2015 11:59:18 -0800 Subject: [PATCH 008/208] post-processor/atlas: fix index out of range panic when artifacts are present --- post-processor/atlas/post-processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/post-processor/atlas/post-processor.go b/post-processor/atlas/post-processor.go index f66eab9ab..504388d81 100644 --- a/post-processor/atlas/post-processor.go +++ b/post-processor/atlas/post-processor.go @@ -178,7 +178,7 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac // Modify the archive options to only include the files // that are in our file list. - include := make([]string, 0, len(fs)) + include := make([]string, len(fs)) for i, f := range fs { include[i] = strings.Replace(f, path, "", 1) } From a81c8905fb0292d8499a529b0e93b4c168a87fea Mon Sep 17 00:00:00 2001 From: Andrew Beresford Date: Fri, 7 Nov 2014 10:55:31 +0000 Subject: [PATCH 009/208] Add 1/10th second delay between key events to VNC --- builder/vmware/common/step_type_boot_command.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builder/vmware/common/step_type_boot_command.go b/builder/vmware/common/step_type_boot_command.go index 82e8b3e17..f67940280 100644 --- a/builder/vmware/common/step_type_boot_command.go +++ b/builder/vmware/common/step_type_boot_command.go @@ -200,7 +200,9 @@ func vncSendString(c *vnc.ClientConn, original string) { } c.KeyEvent(keyCode, true) + time.Sleep(time.Second/10) c.KeyEvent(keyCode, false) + time.Sleep(time.Second/10) if keyShift { c.KeyEvent(KeyLeftShift, false) From d046e1cc5b8e1cf00cf8f1d59fc09258abfcca7e Mon Sep 17 00:00:00 2001 From: renat-sabitov-sirca Date: Tue, 16 Dec 2014 12:18:47 +1100 Subject: [PATCH 010/208] Fixing transient AWS errors during EBS builds Relates to #1539 AWS is eventually consistent and instance can be not visibile for some time after creation. This fix eliminates describe-instances call before going to the proper wait loop --- builder/amazon/common/state.go | 6 +++--- .../amazon/common/step_run_source_instance.go | 16 ++++------------ builder/amazon/ebs/step_create_ami.go | 2 +- builder/amazon/ebs/step_stop_instance.go | 2 +- 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/builder/amazon/common/state.go b/builder/amazon/common/state.go index 62e861d74..a3abd6ee9 100644 --- a/builder/amazon/common/state.go +++ b/builder/amazon/common/state.go @@ -63,9 +63,9 @@ func AMIStateRefreshFunc(conn *ec2.EC2, imageId string) StateRefreshFunc { // InstanceStateRefreshFunc returns a StateRefreshFunc that is used to watch // an EC2 instance. -func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc { +func InstanceStateRefreshFunc(conn *ec2.EC2, instanceId string) StateRefreshFunc { return func() (interface{}, string, error) { - resp, err := conn.Instances([]string{i.InstanceId}, ec2.NewFilter()) + resp, err := conn.Instances([]string{instanceId}, ec2.NewFilter()) if err != nil { if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidInstanceID.NotFound" { // Set this to nil as if we didn't find anything. @@ -85,7 +85,7 @@ func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc { return nil, "", nil } - i = &resp.Reservations[0].Instances[0] + i := &resp.Reservations[0].Instances[0] return i, i.State.Name, nil } } diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 50cedf6ea..22e9da847 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -195,21 +195,13 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi instanceId = spotResp.SpotRequestResults[0].InstanceId } - instanceResp, err := ec2conn.Instances([]string{instanceId}, nil) - if err != nil { - err := fmt.Errorf("Error finding source instance (%s): %s", instanceId, err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - s.instance = &instanceResp.Reservations[0].Instances[0] - ui.Message(fmt.Sprintf("Instance ID: %s", s.instance.InstanceId)) + ui.Message(fmt.Sprintf("Instance ID: %s", instanceId)) - ui.Say(fmt.Sprintf("Waiting for instance (%s) to become ready...", s.instance.InstanceId)) + ui.Say(fmt.Sprintf("Waiting for instance (%v) to become ready...", instanceId)) stateChange := StateChangeConf{ Pending: []string{"pending"}, Target: "running", - Refresh: InstanceStateRefreshFunc(ec2conn, s.instance), + Refresh: InstanceStateRefreshFunc(ec2conn, instanceId), StepState: state, } latestInstance, err := WaitForState(&stateChange) @@ -285,7 +277,7 @@ func (s *StepRunSourceInstance) Cleanup(state multistep.StateBag) { } stateChange := StateChangeConf{ Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"}, - Refresh: InstanceStateRefreshFunc(ec2conn, s.instance), + Refresh: InstanceStateRefreshFunc(ec2conn, s.instance.InstanceId), Target: "terminated", } diff --git a/builder/amazon/ebs/step_create_ami.go b/builder/amazon/ebs/step_create_ami.go index f380ea0b1..66390b08e 100644 --- a/builder/amazon/ebs/step_create_ami.go +++ b/builder/amazon/ebs/step_create_ami.go @@ -87,7 +87,7 @@ func (s *stepCreateAMI) Cleanup(state multistep.StateBag) { ui.Error(fmt.Sprintf("Error deregistering AMI, may still be around: %s", err)) return } else if resp.Return == false { - ui.Error(fmt.Sprintf("Error deregistering AMI, may still be around: %t", resp.Return)) + ui.Error(fmt.Sprintf("Error deregistering AMI, may still be around")) return } } diff --git a/builder/amazon/ebs/step_stop_instance.go b/builder/amazon/ebs/step_stop_instance.go index 09c19bddb..ebd0a27f3 100644 --- a/builder/amazon/ebs/step_stop_instance.go +++ b/builder/amazon/ebs/step_stop_instance.go @@ -37,7 +37,7 @@ func (s *stepStopInstance) Run(state multistep.StateBag) multistep.StepAction { stateChange := awscommon.StateChangeConf{ Pending: []string{"running", "stopping"}, Target: "stopped", - Refresh: awscommon.InstanceStateRefreshFunc(ec2conn, instance), + Refresh: awscommon.InstanceStateRefreshFunc(ec2conn, instance.InstanceId), StepState: state, } _, err = awscommon.WaitForState(&stateChange) From d174ffe1fabb3e26875c97865bf5490ffff1abfc Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Thu, 12 Mar 2015 19:01:51 -0400 Subject: [PATCH 011/208] Fix digitalocean provider for private images [fixes mitchellh/packer#1792] --- builder/digitalocean/api_v2.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/builder/digitalocean/api_v2.go b/builder/digitalocean/api_v2.go index b52aeaaaf..46454a9f8 100644 --- a/builder/digitalocean/api_v2.go +++ b/builder/digitalocean/api_v2.go @@ -138,8 +138,13 @@ func (d DigitalOceanClientV2) CreateDroplet(name string, size string, image stri return 0, fmt.Errorf("Invalid region or lookup failure: '%s': %s", region, err) } + if found_image.Slug == "" { + req.Image = strconv.Itoa(int(found_image.Id)) + } else { + req.Image = found_image.Slug + } + req.Size = found_size.Slug - req.Image = found_image.Slug req.Region = found_region.Slug req.SSHKeys = []string{fmt.Sprintf("%v", keyId)} req.PrivateNetworking = privateNetworking From 7bd37b1f7a4e765dd40d89f6947c0e88a8c2bd69 Mon Sep 17 00:00:00 2001 From: Travis Truman Date: Tue, 14 Apr 2015 11:14:58 -0400 Subject: [PATCH 012/208] Tests were failing incorrectly when OpenStack environment variables were set in the environment running the tests --- builder/openstack/access_config_test.go | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/builder/openstack/access_config_test.go b/builder/openstack/access_config_test.go index 5c92216e3..cf37448cc 100644 --- a/builder/openstack/access_config_test.go +++ b/builder/openstack/access_config_test.go @@ -1,9 +1,17 @@ package openstack import ( + "os" "testing" ) +func init() { + // Clear out the openstack env vars so they don't + // affect our tests. + os.Setenv("SDK_REGION", "") + os.Setenv("OS_REGION_NAME", "") +} + func testAccessConfig() *AccessConfig { return &AccessConfig{} } @@ -16,6 +24,38 @@ func TestAccessConfigPrepare_NoRegion_Rackspace(t *testing.T) { } } +func TestAccessConfigRegionWithEmptyEnv(t *testing.T) { + c := testAccessConfig() + c.Prepare(nil) + if c.Region() != "" { + t.Fatalf("Region should be empty") + } +} + +func TestAccessConfigRegionWithSdkRegionEnv(t *testing.T) { + c := testAccessConfig() + c.Prepare(nil) + + expectedRegion := "sdk_region" + os.Setenv("SDK_REGION", expectedRegion) + os.Setenv("OS_REGION_NAME", "") + if c.Region() != expectedRegion { + t.Fatalf("Region should be: %s", expectedRegion) + } +} + +func TestAccessConfigRegionWithOsRegionNameEnv(t *testing.T) { + c := testAccessConfig() + c.Prepare(nil) + + expectedRegion := "os_region_name" + os.Setenv("SDK_REGION", "") + os.Setenv("OS_REGION_NAME", expectedRegion) + if c.Region() != expectedRegion { + t.Fatalf("Region should be: %s", expectedRegion) + } +} + func TestAccessConfigPrepare_NoRegion_PrivateCloud(t *testing.T) { c := testAccessConfig() c.Provider = "http://some-keystone-server:5000/v2.0" From a7eeb6a6a796690af4c84d0ee6cda0abc48fbc8f Mon Sep 17 00:00:00 2001 From: Brandon Heller Date: Thu, 4 Jun 2015 02:43:16 -0700 Subject: [PATCH 013/208] vmware/iso: support hierarchical output directories When providing a hierarchical output_directory value like 'transient/jenkins-slave', the VM would fail to build in the CreateDisk step. The properly created output directory would not match the location provided to CreateDisk, since datastorePath() did not properly split such paths. Now this case works; tested hierarchical and singular output_directory values. --- builder/vmware/iso/driver_esx5.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/vmware/iso/driver_esx5.go b/builder/vmware/iso/driver_esx5.go index 2b9332760..0d6fd6531 100644 --- a/builder/vmware/iso/driver_esx5.go +++ b/builder/vmware/iso/driver_esx5.go @@ -311,8 +311,8 @@ func (d *ESX5Driver) String() string { } func (d *ESX5Driver) datastorePath(path string) string { - baseDir := filepath.Base(filepath.Dir(path)) - return filepath.ToSlash(filepath.Join("/vmfs/volumes", d.Datastore, baseDir, filepath.Base(path))) + dirPath := filepath.Dir(path) + return filepath.ToSlash(filepath.Join("/vmfs/volumes", d.Datastore, dirPath, filepath.Base(path))) } func (d *ESX5Driver) cachePath(path string) string { From a5c476c6fef5f17a98ced38834c2df110f45eee0 Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Thu, 4 Jun 2015 06:16:44 -0400 Subject: [PATCH 014/208] check the region before do the ami copy to fix [GH-2123] --- builder/amazon/common/step_ami_region_copy.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/builder/amazon/common/step_ami_region_copy.go b/builder/amazon/common/step_ami_region_copy.go index 591dcbdf0..70af06aa5 100644 --- a/builder/amazon/common/step_ami_region_copy.go +++ b/builder/amazon/common/step_ami_region_copy.go @@ -32,6 +32,12 @@ func (s *StepAMIRegionCopy) Run(state multistep.StateBag) multistep.StepAction { var wg sync.WaitGroup errs := new(packer.MultiError) for _, region := range s.Regions { + + if region == ec2conn.Config.Region { + ui.Message(fmt.Sprintf("Avoid copying AMI (%s) to %s", ec2conn.Config.Region, region)) + continue + } + wg.Add(1) ui.Message(fmt.Sprintf("Copying to: %s", region)) From fab9ca9cdbbefd236c10c351d57cba940fcc66b0 Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Tue, 4 Mar 2014 13:23:07 -0500 Subject: [PATCH 015/208] Initial work to implement additional disk support in the vmware-iso builder * Matches the syntax from mitchellh/packer#703 * Creates disk(s), adds them to the vmx template, and runs compact at the end --- builder/vmware/common/step_compact_disk.go | 12 +++++++++ builder/vmware/iso/builder.go | 1 + builder/vmware/iso/step_create_disk.go | 22 ++++++++++++++++ builder/vmware/iso/step_create_vmx.go | 30 ++++++++++++++++++++++ 4 files changed, 65 insertions(+) diff --git a/builder/vmware/common/step_compact_disk.go b/builder/vmware/common/step_compact_disk.go index 777e8aaea..a76cf4d7f 100644 --- a/builder/vmware/common/step_compact_disk.go +++ b/builder/vmware/common/step_compact_disk.go @@ -36,6 +36,18 @@ func (s StepCompactDisk) Run(state multistep.StateBag) multistep.StepAction { state.Put("error", fmt.Errorf("Error compacting disk: %s", err)) return multistep.ActionHalt } + + moreDisks := state.Get("additional_disk_paths").([]string) + if len(moreDisks) > 0 { + for i, path := range moreDisks { + ui.Say(fmt.Sprintf("Compacting additional disk image %d",i+1)) + if err := driver.CompactDisk(path); err != nil { + state.Put("error", fmt.Errorf("Error compacting additional disk %d: %s", i+1, err)) + return multistep.ActionHalt + } + } + + } return multistep.ActionContinue } diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go index c63bfdc8b..065d20603 100644 --- a/builder/vmware/iso/builder.go +++ b/builder/vmware/iso/builder.go @@ -35,6 +35,7 @@ type Config struct { vmwcommon.ToolsConfig `mapstructure:",squash"` vmwcommon.VMXConfig `mapstructure:",squash"` + AdditionalDiskSize []uint `mapstructure:"additionaldisk_size"` DiskName string `mapstructure:"vmdk_name"` DiskSize uint `mapstructure:"disk_size"` DiskTypeId string `mapstructure:"disk_type_id"` diff --git a/builder/vmware/iso/step_create_disk.go b/builder/vmware/iso/step_create_disk.go index b357cad01..b5a1bfd87 100644 --- a/builder/vmware/iso/step_create_disk.go +++ b/builder/vmware/iso/step_create_disk.go @@ -34,6 +34,28 @@ func (stepCreateDisk) Run(state multistep.StateBag) multistep.StepAction { } state.Put("full_disk_path", full_disk_path) + + if len(config.AdditionalDiskSize) > 0 { + // stash the disk paths we create + additional_paths := make([]string,len(config.AdditionalDiskSize)) + + ui.Say("Creating additional hard drives...") + for i, additionalsize := range config.AdditionalDiskSize { + additionalpath := filepath.Join(config.OutputDir, fmt.Sprintf("%s-%d.vmdk",config.DiskName,i+1)) + size := fmt.Sprintf("%dM", uint64(additionalsize)) + + if err := driver.CreateDisk(additionalpath, size, config.DiskTypeId); err != nil { + err := fmt.Errorf("Error creating additional disk: %s", err) + state.Put("error",err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + additional_paths[i] = additionalpath + } + + state.Put("additional_disk_paths", additional_paths) + } return multistep.ActionContinue } diff --git a/builder/vmware/iso/step_create_vmx.go b/builder/vmware/iso/step_create_vmx.go index 5a1a829b5..dde3859a0 100644 --- a/builder/vmware/iso/step_create_vmx.go +++ b/builder/vmware/iso/step_create_vmx.go @@ -20,6 +20,11 @@ type vmxTemplateData struct { Version string } +type additionalDiskTemplateData struct { + DiskNumber int + DiskName string +} + // This step creates the VMX file for the VM. // // Uses: @@ -70,6 +75,25 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { vmxTemplate = string(rawBytes) } + + if len(config.AdditionalDiskSize) > 0 { + for i, _ := range config.AdditionalDiskSize { + data := &additionalDiskTemplateData{ + DiskNumber: i+1, + DiskName: config.DiskName, + } + + diskTemplate, err := config.tpl.Process(DefaultAdditionalDiskTemplate,data) + if err != nil { + err := fmt.Errorf("Error preparing VMX template for additional disk: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + vmxTemplate += diskTemplate + } + } vmxContents, err := interpolate.Render(vmxTemplate, &ctx) if err != nil { @@ -191,3 +215,9 @@ vmci0.pciSlotNumber = "35" vmci0.present = "TRUE" vmotion.checkpointFBSize = "65536000" ` + +const DefaultAdditionalDiskTemplate = ` +scsi0:{{ .DiskNumber }}.fileName = "{{ .DiskName}}-{{ .DiskNumber }}.vmdk" +scsi0:{{ .DiskNumber }}.present = "TRUE" +scsi0:{{ .DiskNumber }}.redo = "" +` From 7dfb837ddb91fa0cd8562b5d16bd84a29c5b2ad7 Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Tue, 4 Mar 2014 15:00:24 -0500 Subject: [PATCH 016/208] Formatting cleanups from `go fmt` --- builder/vmware/iso/step_create_disk.go | 16 ++++++++-------- builder/vmware/iso/step_create_vmx.go | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/builder/vmware/iso/step_create_disk.go b/builder/vmware/iso/step_create_disk.go index b5a1bfd87..ded3a86da 100644 --- a/builder/vmware/iso/step_create_disk.go +++ b/builder/vmware/iso/step_create_disk.go @@ -34,26 +34,26 @@ func (stepCreateDisk) Run(state multistep.StateBag) multistep.StepAction { } state.Put("full_disk_path", full_disk_path) - + if len(config.AdditionalDiskSize) > 0 { // stash the disk paths we create - additional_paths := make([]string,len(config.AdditionalDiskSize)) - + additional_paths := make([]string, len(config.AdditionalDiskSize)) + ui.Say("Creating additional hard drives...") for i, additionalsize := range config.AdditionalDiskSize { - additionalpath := filepath.Join(config.OutputDir, fmt.Sprintf("%s-%d.vmdk",config.DiskName,i+1)) + additionalpath := filepath.Join(config.OutputDir, fmt.Sprintf("%s-%d.vmdk", config.DiskName, i+1)) size := fmt.Sprintf("%dM", uint64(additionalsize)) - + if err := driver.CreateDisk(additionalpath, size, config.DiskTypeId); err != nil { err := fmt.Errorf("Error creating additional disk: %s", err) - state.Put("error",err) + state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - + additional_paths[i] = additionalpath } - + state.Put("additional_disk_paths", additional_paths) } diff --git a/builder/vmware/iso/step_create_vmx.go b/builder/vmware/iso/step_create_vmx.go index dde3859a0..97cd1262c 100644 --- a/builder/vmware/iso/step_create_vmx.go +++ b/builder/vmware/iso/step_create_vmx.go @@ -21,8 +21,8 @@ type vmxTemplateData struct { } type additionalDiskTemplateData struct { - DiskNumber int - DiskName string + DiskNumber int + DiskName string } // This step creates the VMX file for the VM. @@ -75,22 +75,22 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { vmxTemplate = string(rawBytes) } - + if len(config.AdditionalDiskSize) > 0 { for i, _ := range config.AdditionalDiskSize { data := &additionalDiskTemplateData{ - DiskNumber: i+1, - DiskName: config.DiskName, + DiskNumber: i + 1, + DiskName: config.DiskName, } - - diskTemplate, err := config.tpl.Process(DefaultAdditionalDiskTemplate,data) + + diskTemplate, err := config.tpl.Process(DefaultAdditionalDiskTemplate, data) if err != nil { err := fmt.Errorf("Error preparing VMX template for additional disk: %s", err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - + vmxTemplate += diskTemplate } } From 5f183026b337d8b64f5a50a20931de457a8ee023 Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Tue, 4 Mar 2014 15:48:15 -0500 Subject: [PATCH 017/208] Fix vmware compact_disk step when there are no additional disks --- builder/vmware/common/step_compact_disk.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/builder/vmware/common/step_compact_disk.go b/builder/vmware/common/step_compact_disk.go index a76cf4d7f..4319b7fe9 100644 --- a/builder/vmware/common/step_compact_disk.go +++ b/builder/vmware/common/step_compact_disk.go @@ -37,16 +37,16 @@ func (s StepCompactDisk) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - moreDisks := state.Get("additional_disk_paths").([]string) - if len(moreDisks) > 0 { - for i, path := range moreDisks { - ui.Say(fmt.Sprintf("Compacting additional disk image %d",i+1)) - if err := driver.CompactDisk(path); err != nil { - state.Put("error", fmt.Errorf("Error compacting additional disk %d: %s", i+1, err)) - return multistep.ActionHalt + if state.Get("additional_disk_paths") != nil { + if moreDisks := state.Get("additional_disk_paths").([]string); len(moreDisks) > 0 { + for i, path := range moreDisks { + ui.Say(fmt.Sprintf("Compacting additional disk image %d",i+1)) + if err := driver.CompactDisk(path); err != nil { + state.Put("error", fmt.Errorf("Error compacting additional disk %d: %s", i+1, err)) + return multistep.ActionHalt + } } } - } return multistep.ActionContinue From b665339b39f007a2e3121a99eae959487b4208b1 Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Mon, 15 Sep 2014 09:47:26 -0700 Subject: [PATCH 018/208] Added website docs for VMWare ISO additionaldisk_size --- website/source/docs/builders/vmware-iso.html.markdown | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/website/source/docs/builders/vmware-iso.html.markdown b/website/source/docs/builders/vmware-iso.html.markdown index b1d026e44..b021ca532 100644 --- a/website/source/docs/builders/vmware-iso.html.markdown +++ b/website/source/docs/builders/vmware-iso.html.markdown @@ -72,6 +72,12 @@ each category, the available options are alphabetized and described. ### Optional: +* `additionaldisk_size` (array of integers) - The size(s) of any additional + hard disks for the VM in megabytes. If this is not specified then the VM will + only contain a primary hard disk. The builder uses expandable, not fixed-size + virtual hard disks, so the actual file representing the disk will not use the + full size unless it is full. + * `boot_command` (array of strings) - This is an array of commands to type when the virtual machine is first booted. The goal of these commands should be to type just enough to initialize the operating system installer. Special From e9a491ae45871c42bf9b86c302b6ed26657b8acf Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Thu, 4 Jun 2015 11:44:07 -0700 Subject: [PATCH 019/208] New interpolation for additional vmware disks --- builder/vmware/iso/step_create_vmx.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/builder/vmware/iso/step_create_vmx.go b/builder/vmware/iso/step_create_vmx.go index 97cd1262c..69cb3f261 100644 --- a/builder/vmware/iso/step_create_vmx.go +++ b/builder/vmware/iso/step_create_vmx.go @@ -45,15 +45,6 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { ui.Say("Building and writing VMX file") - ctx := config.ctx - ctx.Data = &vmxTemplateData{ - Name: config.VMName, - GuestOS: config.GuestOSType, - DiskName: config.DiskName, - Version: config.Version, - ISOPath: isoPath, - } - vmxTemplate := DefaultVMXTemplate if config.VMXTemplatePath != "" { f, err := os.Open(config.VMXTemplatePath) @@ -76,14 +67,16 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { vmxTemplate = string(rawBytes) } + ctx := config.ctx + if len(config.AdditionalDiskSize) > 0 { for i, _ := range config.AdditionalDiskSize { - data := &additionalDiskTemplateData{ + ctx.Data = &additionalDiskTemplateData{ DiskNumber: i + 1, DiskName: config.DiskName, } - diskTemplate, err := config.tpl.Process(DefaultAdditionalDiskTemplate, data) + diskTemplate, err := interpolate.Render(DefaultAdditionalDiskTemplate, &ctx) if err != nil { err := fmt.Errorf("Error preparing VMX template for additional disk: %s", err) state.Put("error", err) @@ -95,6 +88,14 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { } } + ctx.Data = &vmxTemplateData{ + Name: config.VMName, + GuestOS: config.GuestOSType, + DiskName: config.DiskName, + Version: config.Version, + ISOPath: isoPath, + } + vmxContents, err := interpolate.Render(vmxTemplate, &ctx) if err != nil { err := fmt.Errorf("Error procesing VMX template: %s", err) From a5818b158f9ca52ff06f500e4bdc2e4c2379bd72 Mon Sep 17 00:00:00 2001 From: "James G. Kim" Date: Tue, 2 Jun 2015 18:38:27 +0900 Subject: [PATCH 020/208] Fixes 'unknown configuration key' errors for 'only' and 'except' --- template/parse.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/template/parse.go b/template/parse.go index a7b187f37..dbb29569d 100644 --- a/template/parse.go +++ b/template/parse.go @@ -134,6 +134,8 @@ func (r *rawTemplate) Template() (*Template, error) { } // Set the configuration + delete(c, "except") + delete(c, "only") delete(c, "keep_input_artifact") delete(c, "type") if len(c) > 0 { From d7c77895dcf0b415bb7018b0e5850603cf8879f7 Mon Sep 17 00:00:00 2001 From: "James G. Kim" Date: Fri, 5 Jun 2015 12:26:33 +0900 Subject: [PATCH 021/208] Add tests for only and except of post-processors --- template/parse_test.go | 34 +++++++++++++++++++++ template/test-fixtures/parse-pp-except.json | 8 +++++ template/test-fixtures/parse-pp-only.json | 8 +++++ 3 files changed, 50 insertions(+) create mode 100644 template/test-fixtures/parse-pp-except.json create mode 100644 template/test-fixtures/parse-pp-only.json diff --git a/template/parse_test.go b/template/parse_test.go index 46bb75ad8..9abca2f77 100644 --- a/template/parse_test.go +++ b/template/parse_test.go @@ -174,6 +174,40 @@ func TestParse(t *testing.T) { false, }, + { + "parse-pp-only.json", + &Template{ + PostProcessors: [][]*PostProcessor{ + []*PostProcessor{ + &PostProcessor{ + Type: "foo", + OnlyExcept: OnlyExcept{ + Only: []string{"bar"}, + }, + }, + }, + }, + }, + false, + }, + + { + "parse-pp-except.json", + &Template{ + PostProcessors: [][]*PostProcessor{ + []*PostProcessor{ + &PostProcessor{ + Type: "foo", + OnlyExcept: OnlyExcept{ + Except: []string{"bar"}, + }, + }, + }, + }, + }, + false, + }, + { "parse-pp-string.json", &Template{ diff --git a/template/test-fixtures/parse-pp-except.json b/template/test-fixtures/parse-pp-except.json new file mode 100644 index 000000000..dea70d3d3 --- /dev/null +++ b/template/test-fixtures/parse-pp-except.json @@ -0,0 +1,8 @@ +{ + "post-processors": [ + { + "type": "foo", + "except": ["bar"] + } + ] +} diff --git a/template/test-fixtures/parse-pp-only.json b/template/test-fixtures/parse-pp-only.json new file mode 100644 index 000000000..d2dbe07c6 --- /dev/null +++ b/template/test-fixtures/parse-pp-only.json @@ -0,0 +1,8 @@ +{ + "post-processors": [ + { + "type": "foo", + "only": ["bar"] + } + ] +} From 052b5e66996003f55e4513befafd3c7934fc7802 Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Fri, 5 Jun 2015 11:15:48 +0000 Subject: [PATCH 022/208] "Name" parameter required for copying ami across regions [GH-2172] --- builder/amazon/chroot/builder.go | 4 +++- builder/amazon/common/step_ami_region_copy.go | 6 ++++-- builder/amazon/ebs/builder.go | 1 + builder/amazon/instance/builder.go | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/builder/amazon/chroot/builder.go b/builder/amazon/chroot/builder.go index 66650b01e..9e7452182 100644 --- a/builder/amazon/chroot/builder.go +++ b/builder/amazon/chroot/builder.go @@ -166,7 +166,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &StepSnapshot{}, &StepRegisterAMI{}, &awscommon.StepAMIRegionCopy{ - Regions: b.config.AMIRegions, + AccessConfig: &b.config.AccessConfig, + Regions: b.config.AMIRegions, + Name: b.config.AMIName, }, &awscommon.StepModifyAMIAttributes{ Description: b.config.AMIDescription, diff --git a/builder/amazon/common/step_ami_region_copy.go b/builder/amazon/common/step_ami_region_copy.go index 591dcbdf0..c27b33560 100644 --- a/builder/amazon/common/step_ami_region_copy.go +++ b/builder/amazon/common/step_ami_region_copy.go @@ -14,6 +14,7 @@ import ( type StepAMIRegionCopy struct { AccessConfig *AccessConfig Regions []string + Name string } func (s *StepAMIRegionCopy) Run(state multistep.StateBag) multistep.StepAction { @@ -37,7 +38,7 @@ func (s *StepAMIRegionCopy) Run(state multistep.StateBag) multistep.StepAction { go func(region string) { defer wg.Done() - id, err := amiRegionCopy(state, s.AccessConfig, ami, region, ec2conn.Config.Region) + id, err := amiRegionCopy(state, s.AccessConfig, s.Name, ami, region, ec2conn.Config.Region) lock.Lock() defer lock.Unlock() @@ -69,7 +70,7 @@ func (s *StepAMIRegionCopy) Cleanup(state multistep.StateBag) { // amiRegionCopy does a copy for the given AMI to the target region and // returns the resulting ID or error. -func amiRegionCopy(state multistep.StateBag, config *AccessConfig, imageId string, +func amiRegionCopy(state multistep.StateBag, config *AccessConfig, name string, imageId string, target string, source string) (string, error) { // Connect to the region where the AMI will be copied to @@ -83,6 +84,7 @@ func amiRegionCopy(state multistep.StateBag, config *AccessConfig, imageId strin resp, err := regionconn.CopyImage(&ec2.CopyImageInput{ SourceRegion: &source, SourceImageID: &imageId, + Name: &name, }) if err != nil { diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 274e3cd6d..5df69f658 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -123,6 +123,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepAMIRegionCopy{ AccessConfig: &b.config.AccessConfig, Regions: b.config.AMIRegions, + Name: b.config.AMIName, }, &awscommon.StepModifyAMIAttributes{ Description: b.config.AMIDescription, diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 4808a4031..8534cf8dc 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -205,6 +205,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepAMIRegionCopy{ AccessConfig: &b.config.AccessConfig, Regions: b.config.AMIRegions, + Name: b.config.AMIName, }, &awscommon.StepModifyAMIAttributes{ Description: b.config.AMIDescription, From 5bd2d4e6e06db97377c4ef75005a0eed6698dba4 Mon Sep 17 00:00:00 2001 From: "James G. Kim" Date: Fri, 5 Jun 2015 22:01:29 +0900 Subject: [PATCH 023/208] Fix a problem preventing `tag` or `save` from docker-tag artifacts --- post-processor/docker-save/post-processor.go | 4 +++- post-processor/docker-tag/post-processor.go | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/post-processor/docker-save/post-processor.go b/post-processor/docker-save/post-processor.go index f35b0053e..ab6170802 100644 --- a/post-processor/docker-save/post-processor.go +++ b/post-processor/docker-save/post-processor.go @@ -9,6 +9,7 @@ import ( "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/post-processor/docker-import" + "github.com/mitchellh/packer/post-processor/docker-tag" "github.com/mitchellh/packer/template/interpolate" ) @@ -44,7 +45,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - if artifact.BuilderId() != dockerimport.BuilderId { + if artifact.BuilderId() != dockerimport.BuilderId && + artifact.BuilderId() != dockertag.BuilderId { err := fmt.Errorf( "Unknown artifact type: %s\nCan only save Docker builder artifacts.", artifact.BuilderId()) diff --git a/post-processor/docker-tag/post-processor.go b/post-processor/docker-tag/post-processor.go index 7fc47901f..a531f2820 100644 --- a/post-processor/docker-tag/post-processor.go +++ b/post-processor/docker-tag/post-processor.go @@ -45,7 +45,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - if artifact.BuilderId() != dockerimport.BuilderId { + if artifact.BuilderId() != BuilderId && + artifact.BuilderId() != dockerimport.BuilderId { err := fmt.Errorf( "Unknown artifact type: %s\nCan only tag from Docker builder artifacts.", artifact.BuilderId()) From 3dabf60b52eba1286f1ecb0f1b2f093fc4c8abaa Mon Sep 17 00:00:00 2001 From: Basil Peace Date: Sat, 6 Jun 2015 23:43:06 +0300 Subject: [PATCH 024/208] Fix error in isotime demo in the documentation of Configuration Templates --- .../source/docs/templates/configuration-templates.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/templates/configuration-templates.html.markdown b/website/source/docs/templates/configuration-templates.html.markdown index f940ebcc1..514bf7820 100644 --- a/website/source/docs/templates/configuration-templates.html.markdown +++ b/website/source/docs/templates/configuration-templates.html.markdown @@ -118,7 +118,7 @@ isotime = June 7, 7:22:43pm 2014 {{isotime "2006-01-02"}} = 2014-06-07 {{isotime "Mon 1504"}} = Sat 1922 -{{isotime "01-Jan-06 03\_04\_05"}} = 07-Jun-2014 07\_22\_43 +{{isotime "02-Jan-06 03\_04\_05"}} = 07-Jun-2014 07\_22\_43 {{isotime "Hour15Year200603"}} = Hour19Year201407 ``` From 1539b9459f69adc2fcb86cbb5c9dac347d0faa4a Mon Sep 17 00:00:00 2001 From: Christoph Hartmann Date: Sun, 7 Jun 2015 15:12:05 +0200 Subject: [PATCH 025/208] replace opscode with chef --- .../source/docs/provisioners/chef-solo.html.markdown | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/source/docs/provisioners/chef-solo.html.markdown b/website/source/docs/provisioners/chef-solo.html.markdown index e4f27f627..178ba0c62 100644 --- a/website/source/docs/provisioners/chef-solo.html.markdown +++ b/website/source/docs/provisioners/chef-solo.html.markdown @@ -10,12 +10,12 @@ description: |- Type: `chef-solo` The Chef solo Packer provisioner installs and configures software on machines built -by Packer using [chef-solo](http://docs.opscode.com/chef_solo.html). Cookbooks +by Packer using [chef-solo](https://docs.chef.io/chef_solo.html). Cookbooks can be uploaded from your local machine to the remote machine or remote paths can be used. The provisioner will even install Chef onto your machine if it isn't already -installed, using the official Chef installers provided by Opscode. +installed, using the official Chef installers provided by Chef Inc. ## Basic Example @@ -82,11 +82,11 @@ configuration is actually required, but at least `run_list` is recommended. These will be uploaded to the remote machine in the directory specified by the `staging_directory`. By default, this is empty. -* `run_list` (array of strings) - The [run list](http://docs.opscode.com/essentials_node_object_run_lists.html) +* `run_list` (array of strings) - The [run list](https://docs.chef.io/run_lists.html) for Chef. By default this is empty. * `skip_install` (boolean) - If true, Chef will not automatically be installed - on the machine using the Opscode omnibus installers. + on the machine using the Chef omnibus installers. * `staging_directory` (string) - This is the directory where all the configuration of Chef by Packer will be placed. By default this is "/tmp/packer-chef-solo". @@ -149,7 +149,7 @@ for readability) to install Chef. This command can be customized if you want to install Chef in another way. ```text -curl -L https://www.opscode.com/chef/install.sh | \ +curl -L https://www.chef.io/chef/install.sh | \ {{if .Sudo}}sudo{{end}} bash ``` From 3427dc9ed47ad193b2e75449358699eed1587113 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 09:02:00 -0700 Subject: [PATCH 026/208] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d42ea0bdb..c279ebac3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,9 @@ IMPROVEMENTS: have prohibitive firewalls * command/push: Add `-name` flag for specifying name from CLI [GH-2042] * command/push: Push configuration in templates supports variables [GH-1861] + * post-processor/docker-save: Can be chained [GH-2179] * post-processor/docker-tag: Support `force` option [GH-2055] + * post-processor/docker-tag: Can be chained [GH-2179] BUG FIXES: From 769b7d20b9271f03a4d5122657925e7000fdc4de Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 09:16:01 -0700 Subject: [PATCH 027/208] amazon/ebs: acceptance test for region copy --- builder/amazon/ebs/builder_acc_test.go | 70 ++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/builder/amazon/ebs/builder_acc_test.go b/builder/amazon/ebs/builder_acc_test.go index 19af43512..b70f7f7b9 100644 --- a/builder/amazon/ebs/builder_acc_test.go +++ b/builder/amazon/ebs/builder_acc_test.go @@ -1,10 +1,14 @@ package ebs import ( + "fmt" "os" "testing" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/packer/builder/amazon/common" builderT "github.com/mitchellh/packer/helper/builder/testing" + "github.com/mitchellh/packer/packer" ) func TestBuilderAcc_basic(t *testing.T) { @@ -15,6 +19,48 @@ func TestBuilderAcc_basic(t *testing.T) { }) } +func TestBuilderAcc_regionCopy(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Builder: &Builder{}, + Template: testBuilderAccRegionCopy, + Check: checkRegionCopy([]string{"us-east-1", "us-west-2"}), + }) +} + +func checkRegionCopy(regions []string) builderT.TestCheckFunc { + return func(artifacts []packer.Artifact) error { + if len(artifacts) > 1 { + return fmt.Errorf("more than 1 artifact") + } + + // Get the actual *Artifact pointer so we can access the AMIs directly + artifactRaw := artifacts[0] + artifact, ok := artifactRaw.(*common.Artifact) + if !ok { + return fmt.Errorf("unknown artifact: %#v", artifactRaw) + } + + // Verify that we copied to only the regions given + regionSet := make(map[string]struct{}) + for _, r := range regions { + regionSet[r] = struct{}{} + } + for r, _ := range artifact.Amis { + if _, ok := regionSet[r]; !ok { + return fmt.Errorf("unknown region: %s", r) + } + + delete(regionSet, r) + } + if len(regionSet) > 0 { + return fmt.Errorf("didn't copy to: %#v", regionSet) + } + + return nil + } +} + func testAccPreCheck(t *testing.T) { if v := os.Getenv("AWS_ACCESS_KEY_ID"); v == "" { t.Fatal("AWS_ACCESS_KEY_ID must be set for acceptance tests") @@ -25,6 +71,16 @@ func testAccPreCheck(t *testing.T) { } } +func testEC2Conn() (*ec2.EC2, error) { + access := &common.AccessConfig{RawRegion: "us-east-1"} + config, err := access.Config() + if err != nil { + return nil, err + } + + return ec2.New(config), nil +} + const testBuilderAccBasic = ` { "builders": [{ @@ -37,3 +93,17 @@ const testBuilderAccBasic = ` }] } ` + +const testBuilderAccRegionCopy = ` +{ + "builders": [{ + "type": "test", + "region": "us-east-1", + "instance_type": "m3.medium", + "source_ami": "ami-76b2a71e", + "ssh_username": "ubuntu", + "ami_name": "packer-test {{timestamp}}", + "ami_regions": ["us-east-1", "us-west-2"] + }] +} +` From 8aec42e3634e23a40a4dcd5aa9fd1e03ff745754 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 09:25:56 -0700 Subject: [PATCH 028/208] update CHANGELOG --- CHANGELOG.md | 1 + builder/amazon/common/step_ami_region_copy.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c279ebac3..ae5d76330 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ BUG FIXES: AWS for spot instance. [GH-2017] * builder/amazon: Private key file (only available in debug mode) is deleted on cleanup. [GH-1801] + * builder/amazon: AMI copy won't copy to the source region [GH-2123] * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] * builder/digitalocean: Ignore invalid fields from the ever-changing v2 API diff --git a/builder/amazon/common/step_ami_region_copy.go b/builder/amazon/common/step_ami_region_copy.go index de641c902..0f0c49fdb 100644 --- a/builder/amazon/common/step_ami_region_copy.go +++ b/builder/amazon/common/step_ami_region_copy.go @@ -33,9 +33,9 @@ func (s *StepAMIRegionCopy) Run(state multistep.StateBag) multistep.StepAction { var wg sync.WaitGroup errs := new(packer.MultiError) for _, region := range s.Regions { - if region == ec2conn.Config.Region { - ui.Message(fmt.Sprintf("Avoid copying AMI (%s) to %s", ec2conn.Config.Region, region)) + ui.Message(fmt.Sprintf( + "Avoid copying AMI to duplicate region %s", region)) continue } From 984bbc3a1ee8e377e48a90424af140b39a26e180 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 09:34:53 -0700 Subject: [PATCH 029/208] amazon/common: wording nit --- builder/amazon/common/step_ami_region_copy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/common/step_ami_region_copy.go b/builder/amazon/common/step_ami_region_copy.go index 0f0c49fdb..3f545284f 100644 --- a/builder/amazon/common/step_ami_region_copy.go +++ b/builder/amazon/common/step_ami_region_copy.go @@ -35,7 +35,7 @@ func (s *StepAMIRegionCopy) Run(state multistep.StateBag) multistep.StepAction { for _, region := range s.Regions { if region == ec2conn.Config.Region { ui.Message(fmt.Sprintf( - "Avoid copying AMI to duplicate region %s", region)) + "Avoiding copying AMI to duplicate region %s", region)) continue } From 231f01cd352c7fc93fa5091b7840d12ebe1b3b89 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Mon, 8 Jun 2015 17:08:39 -0500 Subject: [PATCH 030/208] builder/aws: Add pre validate step, to validate things before building. --- builder/amazon/common/step_pre_validate.go | 47 ++++++++++++++++++++++ builder/amazon/ebs/builder.go | 3 ++ 2 files changed, 50 insertions(+) create mode 100644 builder/amazon/common/step_pre_validate.go diff --git a/builder/amazon/common/step_pre_validate.go b/builder/amazon/common/step_pre_validate.go new file mode 100644 index 000000000..1fc3dbade --- /dev/null +++ b/builder/amazon/common/step_pre_validate.go @@ -0,0 +1,47 @@ +package common + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" +) + +// StepPreValidate provides an opportunity to pre-validate any configuration for +// the build before actually doing any time consuming work +// +type StepPreValidate struct { + DestAmiName string +} + +func (s *StepPreValidate) Run(state multistep.StateBag) multistep.StepAction { + ec2conn := state.Get("ec2").(*ec2.EC2) + ui := state.Get("ui").(packer.Ui) + + ui.Say("Prevalidating AMI Name...") + resp, err := ec2conn.DescribeImages(&ec2.DescribeImagesInput{ + Filters: []*ec2.Filter{&ec2.Filter{ + Name: aws.String("name"), + Values: []*string{aws.String(s.DestAmiName)}, + }}}) + + if err != nil { + err := fmt.Errorf("Error querying AMI: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if len(resp.Images) > 0 { + err := fmt.Errorf("Error: an AMI with that name already exists") + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *StepPreValidate) Cleanup(multistep.StateBag) {} diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 274e3cd6d..93bb23e54 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -78,6 +78,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ + &awscommon.StepPreValidate{ + DestAmiName: b.config.AMIName, + }, &awscommon.StepSourceAMIInfo{ SourceAmi: b.config.SourceAmi, EnhancedNetworking: b.config.AMIEnhancedNetworking, From 68040f786c314ef9fc8a839f681f3c2220d56c55 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Mon, 8 Jun 2015 22:00:59 -0500 Subject: [PATCH 031/208] show AMI id in error message --- builder/amazon/common/step_pre_validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/common/step_pre_validate.go b/builder/amazon/common/step_pre_validate.go index 1fc3dbade..5eb263eca 100644 --- a/builder/amazon/common/step_pre_validate.go +++ b/builder/amazon/common/step_pre_validate.go @@ -35,7 +35,7 @@ func (s *StepPreValidate) Run(state multistep.StateBag) multistep.StepAction { } if len(resp.Images) > 0 { - err := fmt.Errorf("Error: an AMI with that name already exists") + err := fmt.Errorf("Error: name conflicts with an existing AMI: %s", *resp.Images[0].ImageID) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt From 0885e03bbf48e5bff2378d2ccb18c99db5a99df0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 20:24:49 -0700 Subject: [PATCH 032/208] virtualbox/iso: acceptance test --- builder/virtualbox/iso/builder_acc_test.go | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 builder/virtualbox/iso/builder_acc_test.go diff --git a/builder/virtualbox/iso/builder_acc_test.go b/builder/virtualbox/iso/builder_acc_test.go new file mode 100644 index 000000000..d756f452e --- /dev/null +++ b/builder/virtualbox/iso/builder_acc_test.go @@ -0,0 +1,30 @@ +package iso + +import ( + "testing" + + builderT "github.com/mitchellh/packer/helper/builder/testing" +) + +func TestBuilderAcc_basic(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + Builder: &Builder{}, + Template: testBuilderAccBasic, + }) +} + +const testBuilderAccBasic = ` +{ + "builders": [{ + "type": "test", + "guest_os_type": "Ubuntu_64", + "iso_url": "http://releases.ubuntu.com/12.04/ubuntu-12.04.5-server-amd64.iso", + "iso_checksum": "769474248a3897f4865817446f9a4a53", + "iso_checksum_type": "md5", + "ssh_username": "packer", + "ssh_password": "packer", + "ssh_wait_timeout": "30s", + "shutdown_command": "echo 'packer' | sudo -S shutdown -P now" + }] +} +` From b441348ba4ad50364be33ce73c163d140730d63a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 20:25:21 -0700 Subject: [PATCH 033/208] virtualbox/common: remove devices should delete floppy controller GH-1879 --- builder/virtualbox/common/step_remove_devices.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/builder/virtualbox/common/step_remove_devices.go b/builder/virtualbox/common/step_remove_devices.go index 8f06b1a61..c184c7c4f 100644 --- a/builder/virtualbox/common/step_remove_devices.go +++ b/builder/virtualbox/common/step_remove_devices.go @@ -38,6 +38,19 @@ func (s *StepRemoveDevices) Run(state multistep.StateBag) multistep.StepAction { ui.Error(err.Error()) return multistep.ActionHalt } + + // Don't forget to remove the floppy controller as well + command = []string{ + "storagectl", vmName, + "--name", "Floppy Controller", + "--remove", + } + if err := driver.VBoxManage(command...); err != nil { + err := fmt.Errorf("Error removing floppy controller: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } } if _, ok := state.GetOk("attachedIso"); ok { From 35246ba98632751f6a60217d508684cf41e8b332 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 20:31:53 -0700 Subject: [PATCH 034/208] virtualbox/common: fix test --- builder/virtualbox/common/step_remove_devices_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/builder/virtualbox/common/step_remove_devices_test.go b/builder/virtualbox/common/step_remove_devices_test.go index e3df5e6e7..6e6e28f79 100644 --- a/builder/virtualbox/common/step_remove_devices_test.go +++ b/builder/virtualbox/common/step_remove_devices_test.go @@ -102,10 +102,13 @@ func TestStepRemoveDevices_floppyPath(t *testing.T) { } // Test that both were removed - if len(driver.VBoxManageCalls) != 1 { + if len(driver.VBoxManageCalls) != 2 { t.Fatalf("bad: %#v", driver.VBoxManageCalls) } if driver.VBoxManageCalls[0][3] != "Floppy Controller" { t.Fatalf("bad: %#v", driver.VBoxManageCalls) } + if driver.VBoxManageCalls[1][3] != "Floppy Controller" { + t.Fatalf("bad: %#v", driver.VBoxManageCalls) + } } From e65e2d104afa8ed5e1846579c9bede7676d5518b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 20:41:39 -0700 Subject: [PATCH 035/208] common: StepDownload can force an extension --- common/step_download.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/common/step_download.go b/common/step_download.go index 8d6378adc..b8bd60b5e 100644 --- a/common/step_download.go +++ b/common/step_download.go @@ -1,6 +1,7 @@ package common import ( + "crypto/sha1" "encoding/hex" "fmt" "log" @@ -36,6 +37,12 @@ type StepDownload struct { // A list of URLs to attempt to download this thing. Url []string + + // Extension is the extension to force for the file that is downloaded. + // Some systems require a certain extension. If this isn't set, the + // extension on the URL is used. Otherwise, this will be forced + // on the downloaded file for every URL. + Extension string } func (s *StepDownload) Run(state multistep.StateBag) multistep.StepAction { @@ -60,9 +67,19 @@ func (s *StepDownload) Run(state multistep.StateBag) multistep.StepAction { targetPath := s.TargetPath if targetPath == "" { + // Determine a cache key. This is normally just the URL but + // if we force a certain extension we hash the URL and add + // the extension to force it. + cacheKey := url + if s.Extension != "" { + hash := sha1.Sum([]byte(url)) + cacheKey = fmt.Sprintf( + "%s.%s", hex.EncodeToString(hash[:]), s.Extension) + } + log.Printf("Acquiring lock to download: %s", url) - targetPath = cache.Lock(url) - defer cache.Unlock(url) + targetPath = cache.Lock(cacheKey) + defer cache.Unlock(cacheKey) } config := &DownloadConfig{ From 9ea34d4ea85d469272a12dc0ab6fe6837128e382 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 20:42:16 -0700 Subject: [PATCH 036/208] virtualbox/iso: force iso extension for downloads --- builder/virtualbox/iso/builder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/virtualbox/iso/builder.go b/builder/virtualbox/iso/builder.go index d095eae2d..b6ca982fd 100644 --- a/builder/virtualbox/iso/builder.go +++ b/builder/virtualbox/iso/builder.go @@ -230,6 +230,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Description: "ISO", ResultKey: "iso_path", Url: b.config.ISOUrls, + Extension: "iso", }, &vboxcommon.StepOutputDir{ Force: b.config.PackerForce, From 9f0b8b71dbb14dd4b53ec567ffa59abc470cee73 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 20:47:47 -0700 Subject: [PATCH 037/208] virtualbox,vmware: http server should listen on IPv4 --- builder/virtualbox/common/step_http_server.go | 2 +- builder/vmware/common/step_http_server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/virtualbox/common/step_http_server.go b/builder/virtualbox/common/step_http_server.go index 440d9adbc..55874992e 100644 --- a/builder/virtualbox/common/step_http_server.go +++ b/builder/virtualbox/common/step_http_server.go @@ -49,7 +49,7 @@ func (s *StepHTTPServer) Run(state multistep.StateBag) multistep.StepAction { } httpPort = offset + s.HTTPPortMin - httpAddr = fmt.Sprintf(":%d", httpPort) + httpAddr = fmt.Sprintf("0.0.0.0:%d", httpPort) log.Printf("Trying port: %d", httpPort) s.l, err = net.Listen("tcp", httpAddr) if err == nil { diff --git a/builder/vmware/common/step_http_server.go b/builder/vmware/common/step_http_server.go index 440d9adbc..55874992e 100644 --- a/builder/vmware/common/step_http_server.go +++ b/builder/vmware/common/step_http_server.go @@ -49,7 +49,7 @@ func (s *StepHTTPServer) Run(state multistep.StateBag) multistep.StepAction { } httpPort = offset + s.HTTPPortMin - httpAddr = fmt.Sprintf(":%d", httpPort) + httpAddr = fmt.Sprintf("0.0.0.0:%d", httpPort) log.Printf("Trying port: %d", httpPort) s.l, err = net.Listen("tcp", httpAddr) if err == nil { From 646edf5ae238ee82f1af35a882b9fea8c411f786 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 20:56:55 -0700 Subject: [PATCH 038/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae5d76330..e02523426 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ BUG FIXES: * builder/amazon: Private key file (only available in debug mode) is deleted on cleanup. [GH-1801] * builder/amazon: AMI copy won't copy to the source region [GH-2123] + * builder/amazon: Validate AMI doesn't exist with name prior to build [GH-1774] * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] * builder/digitalocean: Ignore invalid fields from the ever-changing v2 API From 9dff0adfb16459842d6fe2dd30198465be2981f8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:13:25 -0700 Subject: [PATCH 039/208] builder/google: don't hardcode SSH timeout [GH-1781] --- builder/googlecompute/builder.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/googlecompute/builder.go b/builder/googlecompute/builder.go index cfb9c6e56..8e35598d2 100644 --- a/builder/googlecompute/builder.go +++ b/builder/googlecompute/builder.go @@ -4,11 +4,11 @@ package googlecompute import ( "fmt" + "log" + "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/packer" - "log" - "time" ) // The unique ID for this builder. @@ -63,7 +63,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &common.StepConnectSSH{ SSHAddress: sshAddress, SSHConfig: sshConfig, - SSHWaitTimeout: 5 * time.Minute, + SSHWaitTimeout: b.config.sshTimeout, }, new(common.StepProvision), new(StepTeardownInstance), From 325a44008f4ddb74bdbb186c9d22898369f67c13 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:16:35 -0700 Subject: [PATCH 040/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e02523426..b2a6d5443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ BUG FIXES: running scripts [GH-1993] * builder/docker: Fix crash that could occur at certain timed ctrl-c [GH-1838] * builder/docker: validate that `export_path` is not a directory [GH-2105] + * builder/google: `ssh_timeout` is respected [GH-1781] * builder/qemu: Add `disk_discard` option [GH-2120] * builder/virtualbox: Added SCSI support * builder/vmware: Case-insensitive match of MAC address to find IP [GH-1989] From 673c726b3df6b2c55c8e6721595cbbd315a50ed9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:17:18 -0700 Subject: [PATCH 041/208] update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2a6d5443..bb7f5adf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ IMPROVEMENTS: RackConnect data to appear * buidler/openstakc: Add `ssh_interface` option for rackconnect for users that have prohibitive firewalls + * builder/virtualbox: Added SCSI support * command/push: Add `-name` flag for specifying name from CLI [GH-2042] * command/push: Push configuration in templates supports variables [GH-1861] * post-processor/docker-save: Can be chained [GH-2179] @@ -39,7 +40,8 @@ BUG FIXES: * builder/docker: validate that `export_path` is not a directory [GH-2105] * builder/google: `ssh_timeout` is respected [GH-1781] * builder/qemu: Add `disk_discard` option [GH-2120] - * builder/virtualbox: Added SCSI support + * builder/virtualbox: Remove the floppy controller in addition to the + floppy disk. [GH-1879] * builder/vmware: Case-insensitive match of MAC address to find IP [GH-1989] * builder/vmware: More robust IP parsing from ifconfig output [GH-1999] * command/validate: don't crash for invalid builds [GH-2139] From b8420df62252bc7e27f85e39232e13e0401fd087 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:18:51 -0700 Subject: [PATCH 042/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb7f5adf2..24195a84f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ BUG FIXES: * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] * builder/digitalocean: Ignore invalid fields from the ever-changing v2 API + * builder/digitalocean: Private images can be used as a source [GH-1792] * builder/docker: Fixed hang on prompt while copying script * builder/docker: Use `docker exec` for newer versions of Docker for running scripts [GH-1993] From 5701d3f6f2ada02c2a2afadca9e73b321235e5c2 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:20:51 -0700 Subject: [PATCH 043/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24195a84f..4ebc6712c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ BUG FIXES: * builder/vmware: More robust IP parsing from ifconfig output [GH-1999] * command/validate: don't crash for invalid builds [GH-2139] * post-processor/atlas: Find common archive prefix for Windows [GH-1874] + * post-processor/atlas: Fix index out of range panic [GH-1959] * post-processor/vagrant-cloud: Fixed failing on response * post-processor/vagrant-cloud: Don't delete version on error [GH-2014] * provisioner/puppet-masterless: Allow manifest_file to be a directory From 888621d0291ff31ebbf2be23ae596192ce4a7078 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:24:21 -0700 Subject: [PATCH 044/208] update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ebc6712c..7c30477e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,8 +41,12 @@ BUG FIXES: * builder/docker: validate that `export_path` is not a directory [GH-2105] * builder/google: `ssh_timeout` is respected [GH-1781] * builder/qemu: Add `disk_discard` option [GH-2120] + * builder/virtualbox: Bind HTTP server to IPv4, which is more compatible with + OS installers. [GH-1709] * builder/virtualbox: Remove the floppy controller in addition to the floppy disk. [GH-1879] + * builder/vmware: Bind HTTP server to IPv4, which is more compatible with + OS installers. [GH-1709] * builder/vmware: Case-insensitive match of MAC address to find IP [GH-1989] * builder/vmware: More robust IP parsing from ifconfig output [GH-1999] * command/validate: don't crash for invalid builds [GH-2139] From d18300e3d6e9d62aa932f25e09c5c1d9d68ad918 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:26:28 -0700 Subject: [PATCH 045/208] import style --- signal.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/signal.go b/signal.go index e63dd2fe5..99c9d4c73 100644 --- a/signal.go +++ b/signal.go @@ -1,11 +1,12 @@ package main import ( - "github.com/mitchellh/packer/packer" - "github.com/mitchellh/packer/packer/plugin" "log" "os" "os/signal" + + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/packer/plugin" ) // Prepares the signal handlers so that we handle interrupts properly. From 50fef50e4be908929daf5b917e95e48119678792 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:28:36 -0700 Subject: [PATCH 046/208] add interrupt handling for SIGTERM [GH-1858] --- CHANGELOG.md | 1 + signal.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c30477e5..fb3d26c5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ FEATURES: IMPROVEMENTS: + * core: Interrupt handling for SIGTERM signal as well. [GH-1858] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear * buidler/openstakc: Add `ssh_interface` option for rackconnect for users that diff --git a/signal.go b/signal.go index 99c9d4c73..8846a3451 100644 --- a/signal.go +++ b/signal.go @@ -4,6 +4,7 @@ import ( "log" "os" "os/signal" + "syscall" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/packer/plugin" @@ -14,6 +15,7 @@ import ( func setupSignalHandlers(ui packer.Ui) { ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt) + signal.Notify(ch, syscall.SIGTERM) go func() { // First interrupt. We mostly ignore this because it allows the From fa2bcb8bc53b81ed6160cef571b103a1cab3fe38 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:33:53 -0700 Subject: [PATCH 047/208] update CHANGELOG --- CHANGELOG.md | 2 ++ builder/vmware/common/step_type_boot_command.go | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb3d26c5c..027fc6fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,8 @@ BUG FIXES: OS installers. [GH-1709] * builder/virtualbox: Remove the floppy controller in addition to the floppy disk. [GH-1879] + * builder/vmware: Add 100ms delay between keystrokes to avoid subtle + timing issues in most cases. [GH-1663] * builder/vmware: Bind HTTP server to IPv4, which is more compatible with OS installers. [GH-1709] * builder/vmware: Case-insensitive match of MAC address to find IP [GH-1989] diff --git a/builder/vmware/common/step_type_boot_command.go b/builder/vmware/common/step_type_boot_command.go index 49c579889..b23ede1da 100644 --- a/builder/vmware/common/step_type_boot_command.go +++ b/builder/vmware/common/step_type_boot_command.go @@ -200,10 +200,13 @@ func vncSendString(c *vnc.ClientConn, original string) { c.KeyEvent(KeyLeftShift, true) } + // Send the key events. We add a 100ms sleep after each key event + // to deal with network latency and the OS responding to the keystroke. + // It is kind of arbitrary but it is better than nothing. c.KeyEvent(keyCode, true) - time.Sleep(time.Second/10) + time.Sleep(100 * time.Millisecond) c.KeyEvent(keyCode, false) - time.Sleep(time.Second/10) + time.Sleep(100 * time.Millisecond) if keyShift { c.KeyEvent(KeyLeftShift, false) From 23a48d6619fd09fd0a8311111f37f0cd3163d560 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 21:34:20 -0700 Subject: [PATCH 048/208] go fmt --- builder/digitalocean/ssh.go | 2 +- builder/googlecompute/ssh.go | 2 +- builder/null/ssh.go | 2 +- builder/openstack/ssh.go | 2 +- builder/parallels/common/ssh.go | 2 +- builder/qemu/ssh.go | 2 +- builder/virtualbox/common/ssh.go | 2 +- builder/vmware/common/ssh.go | 2 +- builder/vmware/iso/step_upload_vmx.go | 9 ++++----- common/step_connect_ssh.go | 2 +- communicator/ssh/communicator.go | 2 +- communicator/ssh/communicator_test.go | 2 +- post-processor/atlas/util_test.go | 2 +- 13 files changed, 16 insertions(+), 17 deletions(-) diff --git a/builder/digitalocean/ssh.go b/builder/digitalocean/ssh.go index bd0afc3fe..12046b04b 100644 --- a/builder/digitalocean/ssh.go +++ b/builder/digitalocean/ssh.go @@ -1,9 +1,9 @@ package digitalocean import ( - "golang.org/x/crypto/ssh" "fmt" "github.com/mitchellh/multistep" + "golang.org/x/crypto/ssh" ) func sshAddress(state multistep.StateBag) (string, error) { diff --git a/builder/googlecompute/ssh.go b/builder/googlecompute/ssh.go index a4e0151f4..e04029e44 100644 --- a/builder/googlecompute/ssh.go +++ b/builder/googlecompute/ssh.go @@ -1,9 +1,9 @@ package googlecompute import ( - "golang.org/x/crypto/ssh" "fmt" "github.com/mitchellh/multistep" + "golang.org/x/crypto/ssh" ) // sshAddress returns the ssh address. diff --git a/builder/null/ssh.go b/builder/null/ssh.go index a9c2af330..e6ac9ab16 100644 --- a/builder/null/ssh.go +++ b/builder/null/ssh.go @@ -1,10 +1,10 @@ package null import ( - gossh "golang.org/x/crypto/ssh" "fmt" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/communicator/ssh" + gossh "golang.org/x/crypto/ssh" "io/ioutil" ) diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go index 16afda64d..d20f24170 100644 --- a/builder/openstack/ssh.go +++ b/builder/openstack/ssh.go @@ -1,10 +1,10 @@ package openstack import ( - "golang.org/x/crypto/ssh" "errors" "fmt" "github.com/mitchellh/multistep" + "golang.org/x/crypto/ssh" "time" "github.com/mitchellh/gophercloud-fork-40444fb" diff --git a/builder/parallels/common/ssh.go b/builder/parallels/common/ssh.go index 142b6c99d..becf68e42 100644 --- a/builder/parallels/common/ssh.go +++ b/builder/parallels/common/ssh.go @@ -3,10 +3,10 @@ package common import ( "fmt" - "golang.org/x/crypto/ssh" "github.com/mitchellh/multistep" commonssh "github.com/mitchellh/packer/common/ssh" packerssh "github.com/mitchellh/packer/communicator/ssh" + "golang.org/x/crypto/ssh" ) func SSHAddress(state multistep.StateBag) (string, error) { diff --git a/builder/qemu/ssh.go b/builder/qemu/ssh.go index deb7ba405..9724d7483 100644 --- a/builder/qemu/ssh.go +++ b/builder/qemu/ssh.go @@ -3,10 +3,10 @@ package qemu import ( "fmt" - gossh "golang.org/x/crypto/ssh" "github.com/mitchellh/multistep" commonssh "github.com/mitchellh/packer/common/ssh" "github.com/mitchellh/packer/communicator/ssh" + gossh "golang.org/x/crypto/ssh" ) func sshAddress(state multistep.StateBag) (string, error) { diff --git a/builder/virtualbox/common/ssh.go b/builder/virtualbox/common/ssh.go index c07c2ce9c..9ca2529b8 100644 --- a/builder/virtualbox/common/ssh.go +++ b/builder/virtualbox/common/ssh.go @@ -3,10 +3,10 @@ package common import ( "fmt" - gossh "golang.org/x/crypto/ssh" "github.com/mitchellh/multistep" commonssh "github.com/mitchellh/packer/common/ssh" "github.com/mitchellh/packer/communicator/ssh" + gossh "golang.org/x/crypto/ssh" ) func SSHAddress(state multistep.StateBag) (string, error) { diff --git a/builder/vmware/common/ssh.go b/builder/vmware/common/ssh.go index bfd0b8bc7..167bd6792 100644 --- a/builder/vmware/common/ssh.go +++ b/builder/vmware/common/ssh.go @@ -7,10 +7,10 @@ import ( "log" "os" - gossh "golang.org/x/crypto/ssh" "github.com/mitchellh/multistep" commonssh "github.com/mitchellh/packer/common/ssh" "github.com/mitchellh/packer/communicator/ssh" + gossh "golang.org/x/crypto/ssh" ) func SSHAddressFunc(config *SSHConfig) func(multistep.StateBag) (string, error) { diff --git a/builder/vmware/iso/step_upload_vmx.go b/builder/vmware/iso/step_upload_vmx.go index 39e14af16..96dde2cfc 100644 --- a/builder/vmware/iso/step_upload_vmx.go +++ b/builder/vmware/iso/step_upload_vmx.go @@ -3,12 +3,11 @@ package iso import ( "fmt" "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" vmwcommon "github.com/mitchellh/packer/builder/vmware/common" + "github.com/mitchellh/packer/packer" "path/filepath" ) - // This step upload the VMX to the remote host // // Uses: @@ -18,8 +17,8 @@ import ( // // Produces: // -type StepUploadVMX struct{ - RemoteType string +type StepUploadVMX struct { + RemoteType string } func (c *StepUploadVMX) Run(state multistep.StateBag) multistep.StepAction { @@ -31,7 +30,7 @@ func (c *StepUploadVMX) Run(state multistep.StateBag) multistep.StepAction { if c.RemoteType == "esx5" { remoteDriver, ok := driver.(RemoteDriver) if ok { - remoteVmxPath := filepath.ToSlash(filepath.Join(fmt.Sprintf("%s",remoteDriver), filepath.Base(vmxPath))) + remoteVmxPath := filepath.ToSlash(filepath.Join(fmt.Sprintf("%s", remoteDriver), filepath.Base(vmxPath))) if err := remoteDriver.upload(remoteVmxPath, vmxPath); err != nil { state.Put("error", fmt.Errorf("Error writing VMX: %s", err)) return multistep.ActionHalt diff --git a/common/step_connect_ssh.go b/common/step_connect_ssh.go index 30064c7f2..c3d8aac2d 100644 --- a/common/step_connect_ssh.go +++ b/common/step_connect_ssh.go @@ -1,12 +1,12 @@ package common import ( - gossh "golang.org/x/crypto/ssh" "errors" "fmt" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/communicator/ssh" "github.com/mitchellh/packer/packer" + gossh "golang.org/x/crypto/ssh" "log" "strings" "time" diff --git a/communicator/ssh/communicator.go b/communicator/ssh/communicator.go index 611622750..3f0e6191b 100644 --- a/communicator/ssh/communicator.go +++ b/communicator/ssh/communicator.go @@ -3,10 +3,10 @@ package ssh import ( "bufio" "bytes" - "golang.org/x/crypto/ssh" "errors" "fmt" "github.com/mitchellh/packer/packer" + "golang.org/x/crypto/ssh" "io" "io/ioutil" "log" diff --git a/communicator/ssh/communicator_test.go b/communicator/ssh/communicator_test.go index 26cf76757..e9f73d2dc 100644 --- a/communicator/ssh/communicator_test.go +++ b/communicator/ssh/communicator_test.go @@ -4,9 +4,9 @@ package ssh import ( "bytes" - "golang.org/x/crypto/ssh" "fmt" "github.com/mitchellh/packer/packer" + "golang.org/x/crypto/ssh" "net" "testing" ) diff --git a/post-processor/atlas/util_test.go b/post-processor/atlas/util_test.go index 9f2535ccb..b6b9da3d9 100644 --- a/post-processor/atlas/util_test.go +++ b/post-processor/atlas/util_test.go @@ -6,7 +6,7 @@ import ( func TestLongestCommonPrefix(t *testing.T) { cases := []struct { - Input []string + Input []string Output string }{ { From 8c321138f4ecf78c22a19338656d00028740850a Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Tue, 9 Jun 2015 10:41:39 -0500 Subject: [PATCH 049/208] Revert "Changing --region to --location" This reverts commit f40fd36c3150440de0ad939e2e8c46782d709a59. According to the documentation below, `--region` is the current flag. If you're using an older version of the tools that use `--location`, you can customize the commands in your Packer config with `bundle_vol_command` and `bundle_upload_command` - http://docs.aws.amazon.com/AWSEC2/latest/CommandLineReference/CLTRG-ami-upload-bundle.html - https://www.packer.io/docs/builders/amazon-instance.html --- builder/amazon/instance/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 8534cf8dc..d164f91ee 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -80,7 +80,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { "-s {{.SecretKey}} " + "-d {{.BundleDirectory}} " + "--batch " + - "--location {{.Region}} " + + "--region {{.Region}} " + "--retry" } From 999b0874cce88279a3d39dce441a342e7c5f5017 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Tue, 9 Jun 2015 11:38:53 -0500 Subject: [PATCH 050/208] Update AWS EBS builder to fix invalid params --- builder/amazon/common/block_device.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/block_device.go b/builder/amazon/common/block_device.go index 14add3276..c44d03561 100644 --- a/builder/amazon/common/block_device.go +++ b/builder/amazon/common/block_device.go @@ -29,13 +29,23 @@ func buildBlockDevices(b []BlockDevice) []*ec2.BlockDeviceMapping { for _, blockDevice := range b { ebsBlockDevice := &ec2.EBSBlockDevice{ - SnapshotID: &blockDevice.SnapshotId, - Encrypted: &blockDevice.Encrypted, - IOPS: &blockDevice.IOPS, VolumeType: &blockDevice.VolumeType, VolumeSize: &blockDevice.VolumeSize, DeleteOnTermination: &blockDevice.DeleteOnTermination, } + + // IOPS is only valid for SSD Volumes + if blockDevice.VolumeType != "standard" && blockDevice.VolumeType != "gp2" { + ebsBlockDevice.IOPS = &blockDevice.IOPS + } + + // You cannot specify Encrypted if you specify a Snapshot ID + if blockDevice.SnapshotId != "" { + ebsBlockDevice.SnapshotID = &blockDevice.SnapshotId + } else { + ebsBlockDevice.Encrypted = &blockDevice.Encrypted + } + mapping := &ec2.BlockDeviceMapping{ EBS: ebsBlockDevice, DeviceName: &blockDevice.DeviceName, From 4da118c64f22b950cb596c5c370751be5f59121b Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Tue, 9 Jun 2015 11:56:40 -0500 Subject: [PATCH 051/208] fix up tests --- builder/amazon/common/block_device_test.go | 27 +++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/block_device_test.go b/builder/amazon/common/block_device_test.go index c4f644f67..aacd54cb4 100644 --- a/builder/amazon/common/block_device_test.go +++ b/builder/amazon/common/block_device_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awsutil" "github.com/aws/aws-sdk-go/service/ec2" ) @@ -28,11 +29,31 @@ func TestBlockDevice(t *testing.T) { DeviceName: aws.String("/dev/sdb"), VirtualName: aws.String("ephemeral0"), EBS: &ec2.EBSBlockDevice{ - Encrypted: aws.Boolean(false), SnapshotID: aws.String("snap-1234"), VolumeType: aws.String("standard"), VolumeSize: aws.Long(8), DeleteOnTermination: aws.Boolean(true), + }, + }, + }, + { + Config: &BlockDevice{ + DeviceName: "/dev/sdb", + VirtualName: "ephemeral0", + VolumeType: "io1", + VolumeSize: 8, + DeleteOnTermination: true, + IOPS: 1000, + }, + + Result: &ec2.BlockDeviceMapping{ + DeviceName: aws.String("/dev/sdb"), + VirtualName: aws.String("ephemeral0"), + EBS: &ec2.EBSBlockDevice{ + Encrypted: aws.Boolean(false), + VolumeType: aws.String("io1"), + VolumeSize: aws.Long(8), + DeleteOnTermination: aws.Boolean(true), IOPS: aws.Long(1000), }, }, @@ -48,11 +69,11 @@ func TestBlockDevice(t *testing.T) { expected := []*ec2.BlockDeviceMapping{tc.Result} got := blockDevices.BuildAMIDevices() if !reflect.DeepEqual(expected, got) { - t.Fatalf("bad: %#v", expected) + t.Fatalf("Bad block device, \nexpected: %s\n\ngot: %s", awsutil.StringValue(expected), awsutil.StringValue(got)) } if !reflect.DeepEqual(expected, blockDevices.BuildLaunchDevices()) { - t.Fatalf("bad: %#v", expected) + t.Fatalf("Bad block device, \nexpected: %s\n\ngot: %s", awsutil.StringValue(expected), awsutil.StringValue(blockDevices.BuildLaunchDevices())) } } } From 91cfe5edf45404dc44b7decb0854baa3f2cc7236 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Jun 2015 20:57:32 -0700 Subject: [PATCH 052/208] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 027fc6fed..26f0274a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,8 @@ BUG FIXES: OS installers. [GH-1709] * builder/virtualbox: Remove the floppy controller in addition to the floppy disk. [GH-1879] + * builder/virtualbox: Fixed regression where downloading ISO without a + ".iso" extension didn't work. [GH-1839] * builder/vmware: Add 100ms delay between keystrokes to avoid subtle timing issues in most cases. [GH-1663] * builder/vmware: Bind HTTP server to IPv4, which is more compatible with From 6e9fb6a9d1c019da4d9b442b73b46312c7c91d87 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Jun 2015 21:09:09 -0700 Subject: [PATCH 053/208] command/push: the -name parameter actually works --- command/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/push.go b/command/push.go index 6c04917dd..b4007db80 100644 --- a/command/push.go +++ b/command/push.go @@ -157,7 +157,7 @@ func (c *PushCommand) Run(args []string) int { // Build the upload options var uploadOpts uploadOpts - uploadOpts.Slug = push.Name + uploadOpts.Slug = name uploadOpts.Builds = make(map[string]*uploadBuildInfo) for _, b := range tpl.Builders { info := &uploadBuildInfo{Type: b.Type} From 2a6c4e0d2c2ee20b3715691ca0ed1934c71e3bec Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Jun 2015 21:09:56 -0700 Subject: [PATCH 054/208] command/push: output fix --- command/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/push.go b/command/push.go index b4007db80..f888186de 100644 --- a/command/push.go +++ b/command/push.go @@ -236,7 +236,7 @@ func (c *PushCommand) Run(args []string) int { return 1 } - c.Ui.Say(fmt.Sprintf("Push successful to '%s'", push.Name)) + c.Ui.Say(fmt.Sprintf("Push successful to '%s'", name)) return 0 } From 8393b85ed579dd2b6a04325c8e5e7c14ed1610d6 Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Wed, 10 Jun 2015 07:41:12 +0200 Subject: [PATCH 055/208] Support Parallels Desktop 11 --- builder/parallels/common/driver.go | 6 ++++++ builder/parallels/common/driver_10.go | 1 + 2 files changed, 7 insertions(+) diff --git a/builder/parallels/common/driver.go b/builder/parallels/common/driver.go index bdb73719c..5c3d93c09 100644 --- a/builder/parallels/common/driver.go +++ b/builder/parallels/common/driver.go @@ -73,6 +73,12 @@ func NewDriver() (Driver, error) { log.Printf("prlctl path: %s", prlctlPath) drivers = map[string]Driver{ + "11": &Parallels10Driver{ + Parallels9Driver: Parallels9Driver{ + PrlctlPath: prlctlPath, + dhcp_lease_file: dhcp_lease_file, + }, + }, "10": &Parallels10Driver{ Parallels9Driver: Parallels9Driver{ PrlctlPath: prlctlPath, diff --git a/builder/parallels/common/driver_10.go b/builder/parallels/common/driver_10.go index ae39240f4..9ab0754de 100644 --- a/builder/parallels/common/driver_10.go +++ b/builder/parallels/common/driver_10.go @@ -1,6 +1,7 @@ package common // Parallels10Driver are inherited from Parallels9Driver. +// Used for Parallels v 10 & 11 type Parallels10Driver struct { Parallels9Driver } From 5d600c70f2356c7b5d8a06eb363e9c05ea824a77 Mon Sep 17 00:00:00 2001 From: Jan Schumann Date: Wed, 10 Jun 2015 17:20:41 +0200 Subject: [PATCH 056/208] added test to illustrate wrong behavior --- builder/amazon/common/block_device_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/builder/amazon/common/block_device_test.go b/builder/amazon/common/block_device_test.go index aacd54cb4..12d1530bf 100644 --- a/builder/amazon/common/block_device_test.go +++ b/builder/amazon/common/block_device_test.go @@ -36,6 +36,23 @@ func TestBlockDevice(t *testing.T) { }, }, }, + { + Config: &BlockDevice{ + DeviceName: "/dev/sdb", + VolumeSize: 8, + }, + + Result: &ec2.BlockDeviceMapping{ + DeviceName: aws.String("/dev/sdb"), + VirtualName: aws.String(""), + EBS: &ec2.EBSBlockDevice{ + Encrypted: aws.Boolean(false), + VolumeType: aws.String(""), + VolumeSize: aws.Long(8), + DeleteOnTermination: aws.Boolean(false), + }, + }, + }, { Config: &BlockDevice{ DeviceName: "/dev/sdb", From 802cfa38715bd12851f2e8719c5165338c78f6ed Mon Sep 17 00:00:00 2001 From: Jan Schumann Date: Wed, 10 Jun 2015 17:21:25 +0200 Subject: [PATCH 057/208] make sure IOPS is not set for empty VolumeType --- builder/amazon/common/block_device.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/common/block_device.go b/builder/amazon/common/block_device.go index c44d03561..e97cd4107 100644 --- a/builder/amazon/common/block_device.go +++ b/builder/amazon/common/block_device.go @@ -35,7 +35,7 @@ func buildBlockDevices(b []BlockDevice) []*ec2.BlockDeviceMapping { } // IOPS is only valid for SSD Volumes - if blockDevice.VolumeType != "standard" && blockDevice.VolumeType != "gp2" { + if blockDevice.VolumeType != "" && blockDevice.VolumeType != "standard" && blockDevice.VolumeType != "gp2" { ebsBlockDevice.IOPS = &blockDevice.IOPS } From 7410289cbad45e6b8f15aee27aef0ab8443d9869 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:21:32 -0700 Subject: [PATCH 058/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26f0274a2..39f222fae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] + * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear * buidler/openstakc: Add `ssh_interface` option for rackconnect for users that From be4a82dfae69f990092db29a24749a43cf3c3dfb Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:33:01 -0700 Subject: [PATCH 059/208] amazon/*: fix some merge conflicts --- builder/amazon/common/state.go | 2 +- builder/amazon/common/step_run_source_instance.go | 2 +- builder/amazon/ebs/step_stop_instance.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/state.go b/builder/amazon/common/state.go index 9acedd020..075ce8ef7 100644 --- a/builder/amazon/common/state.go +++ b/builder/amazon/common/state.go @@ -91,7 +91,7 @@ func InstanceStateRefreshFunc(conn *ec2.EC2, instanceId string) StateRefreshFunc return nil, "", nil } - i = resp.Reservations[0].Instances[0] + i := resp.Reservations[0].Instances[0] return i, *i.State.Name, nil } } diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 87ac077b9..92dafa564 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -310,7 +310,7 @@ func (s *StepRunSourceInstance) Cleanup(state multistep.StateBag) { } stateChange := StateChangeConf{ Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"}, - Refresh: InstanceStateRefreshFunc(ec2conn, s.instance.InstanceId), + Refresh: InstanceStateRefreshFunc(ec2conn, *s.instance.InstanceID), Target: "terminated", } diff --git a/builder/amazon/ebs/step_stop_instance.go b/builder/amazon/ebs/step_stop_instance.go index 7312fd200..967e5bbf4 100644 --- a/builder/amazon/ebs/step_stop_instance.go +++ b/builder/amazon/ebs/step_stop_instance.go @@ -40,7 +40,7 @@ func (s *stepStopInstance) Run(state multistep.StateBag) multistep.StepAction { stateChange := awscommon.StateChangeConf{ Pending: []string{"running", "stopping"}, Target: "stopped", - Refresh: awscommon.InstanceStateRefreshFunc(ec2conn, instance.InstanceId), + Refresh: awscommon.InstanceStateRefreshFunc(ec2conn, *instance.InstanceID), StepState: state, } _, err = awscommon.WaitForState(&stateChange) From 77c8df1e7b3b8910e8fb60d6f8ad37d57d4e417f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:33:30 -0700 Subject: [PATCH 060/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39f222fae..7542c7b97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ BUG FIXES: is deleted on cleanup. [GH-1801] * builder/amazon: AMI copy won't copy to the source region [GH-2123] * builder/amazon: Validate AMI doesn't exist with name prior to build [GH-1774] + * builder/amazon: Improved retry logic around waiting for instances. [GH-1764] * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] * builder/digitalocean: Ignore invalid fields from the ever-changing v2 API From bd04b52b323472922d0fa1dbec9a870296179836 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:50:08 -0700 Subject: [PATCH 061/208] virtualbox/common: style --- builder/virtualbox/common/step_export.go | 9 +++------ builder/virtualbox/common/step_forward_ssh.go | 7 ++----- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/builder/virtualbox/common/step_export.go b/builder/virtualbox/common/step_export.go index 0a3cd816c..56b013d9f 100644 --- a/builder/virtualbox/common/step_export.go +++ b/builder/virtualbox/common/step_export.go @@ -31,12 +31,10 @@ func (s *StepExport) Run(state multistep.StateBag) multistep.StepAction { // Wait a second to ensure VM is really shutdown. log.Println("1 second timeout to ensure VM is really shutdown") time.Sleep(1 * time.Second) + ui.Say("Preparing to export machine...") // Clear out the Packer-created forwarding rule - ui.Say("Preparing to export machine...") - var command []string - - if s.SkipNatMapping == false { + if !s.SkipNatMapping { ui.Message(fmt.Sprintf( "Deleting forwarded port mapping for SSH (host port %d)", state.Get("sshHostPort"))) @@ -52,13 +50,12 @@ func (s *StepExport) Run(state multistep.StateBag) multistep.StepAction { // Export the VM to an OVF outputPath := filepath.Join(s.OutputDir, vmName+"."+s.Format) - command = []string{ + command := []string{ "export", vmName, "--output", outputPath, } - command = append(command, s.ExportOpts...) ui.Say("Exporting virtual machine...") diff --git a/builder/virtualbox/common/step_forward_ssh.go b/builder/virtualbox/common/step_forward_ssh.go index d6d604e00..fe6004281 100644 --- a/builder/virtualbox/common/step_forward_ssh.go +++ b/builder/virtualbox/common/step_forward_ssh.go @@ -30,11 +30,8 @@ func (s *StepForwardSSH) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) vmName := state.Get("vmName").(string) - var sshHostPort uint - if s.SkipNatMapping { - sshHostPort = s.GuestPort - log.Printf("Skipping SSH NAT mapping and using SSH port %d", sshHostPort) - } else { + sshHostPort := s.GuestPort + if !s.SkipNatMapping { log.Printf("Looking for available SSH port between %d and %d", s.HostPortMin, s.HostPortMax) var offset uint = 0 From 9b83b7f2e05fd3a83ee65b53eb05aab1f4786527 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:50:40 -0700 Subject: [PATCH 062/208] update CHANGELOg --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7542c7b97..a98e0bedc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ IMPROVEMENTS: RackConnect data to appear * buidler/openstakc: Add `ssh_interface` option for rackconnect for users that have prohibitive firewalls + * builder/virtualbox: Added option: `ssh_skip_nat_mapping` to skip the + automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support * command/push: Add `-name` flag for specifying name from CLI [GH-2042] * command/push: Push configuration in templates supports variables [GH-1861] From 626243baed02592f76088c3e5ae7140e73d82b5b Mon Sep 17 00:00:00 2001 From: Clint Date: Wed, 10 Jun 2015 13:18:17 -0500 Subject: [PATCH 063/208] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a98e0bedc..a6f179461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] + * builder/amazon-ebs: Add pre-build step, validate unique AMI name [GH-2187] * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear From 0ff508c80d57447c740830a968eaff32cf42257b Mon Sep 17 00:00:00 2001 From: Clint Date: Wed, 10 Jun 2015 13:20:39 -0500 Subject: [PATCH 064/208] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6f179461..b0abd0cb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] - * builder/amazon-ebs: Add pre-build step, validate unique AMI name [GH-2187] * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear @@ -38,6 +37,7 @@ BUG FIXES: * builder/amazon: Improved retry logic around waiting for instances. [GH-1764] * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] + * builder/amazon/instance: Use `--region` flag for bundle upload command. [GH-1931] * builder/digitalocean: Ignore invalid fields from the ever-changing v2 API * builder/digitalocean: Private images can be used as a source [GH-1792] * builder/docker: Fixed hang on prompt while copying script From 83fc70fa00d84cee46399bef2ff8cdaeddfa49bf Mon Sep 17 00:00:00 2001 From: Clint Date: Wed, 10 Jun 2015 13:22:36 -0500 Subject: [PATCH 065/208] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0abd0cb9..8c9a2c151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ BUG FIXES: * builder/amazon: AMI copy won't copy to the source region [GH-2123] * builder/amazon: Validate AMI doesn't exist with name prior to build [GH-1774] * builder/amazon: Improved retry logic around waiting for instances. [GH-1764] + * builder/amazon: Fix issues with creating Block Devices. [GH-2195] * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] * builder/amazon/instance: Use `--region` flag for bundle upload command. [GH-1931] From fafdfc962f7272afe390961e804de4fe67879f85 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 11:31:20 -0700 Subject: [PATCH 066/208] vmware/common: detect Vmware 'unknown error' and show better message --- builder/vmware/common/driver.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/builder/vmware/common/driver.go b/builder/vmware/common/driver.go index c645f40b7..ee8dbc30e 100644 --- a/builder/vmware/common/driver.go +++ b/builder/vmware/common/driver.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os/exec" + "regexp" "runtime" "strconv" "strings" @@ -135,6 +136,18 @@ func runAndLog(cmd *exec.Cmd) (string, string, error) { } err = fmt.Errorf("VMware error: %s", message) + + // If "unknown error" is in there, add some additional notes + re := regexp.MustCompile(`(?i)unknown error`) + if re.MatchString(message) { + err = fmt.Errorf( + "%s\n\n%s", err, + "Packer detected a VMware 'Unknown Error'. Unfortunately VMware\n"+ + "often has extremely vague error messages such as this and Packer\n"+ + "itself can't do much about that. Please check the vmware.log files\n"+ + "created by VMware when a VM is started (in the directory of the\n"+ + "vmx file), which often contains more detailed error information.") + } } log.Printf("stdout: %s", stdoutString) From 802a76685ee14cbd1ece648f8721d7564287c831 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 11:34:37 -0700 Subject: [PATCH 067/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c9a2c151..86e1cbdb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ BUG FIXES: OS installers. [GH-1709] * builder/vmware: Case-insensitive match of MAC address to find IP [GH-1989] * builder/vmware: More robust IP parsing from ifconfig output [GH-1999] + * builder/vmware: Nested output directories for ESXi work [GH-2174] * command/validate: don't crash for invalid builds [GH-2139] * post-processor/atlas: Find common archive prefix for Windows [GH-1874] * post-processor/atlas: Fix index out of range panic [GH-1959] From ffdd5e6772f2785e4d8ee3ea97d7438a76c1892e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 12:40:20 -0700 Subject: [PATCH 068/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86e1cbdb1..3a33dbc92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ IMPROVEMENTS: * builder/virtualbox: Added option: `ssh_skip_nat_mapping` to skip the automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support + * builder/vmware: Support for additional disks [GH-1382] * command/push: Add `-name` flag for specifying name from CLI [GH-2042] * command/push: Push configuration in templates supports variables [GH-1861] * post-processor/docker-save: Can be chained [GH-2179] From 9da9ce6046b3c40a9fe5ac09a38aea6661325d9d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 12:41:17 -0700 Subject: [PATCH 069/208] vmware/iso: disk_additional_size --- builder/vmware/iso/builder.go | 28 +++++++++---------- .../docs/builders/vmware-iso.html.markdown | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go index 065d20603..352d02960 100644 --- a/builder/vmware/iso/builder.go +++ b/builder/vmware/iso/builder.go @@ -35,20 +35,20 @@ type Config struct { vmwcommon.ToolsConfig `mapstructure:",squash"` vmwcommon.VMXConfig `mapstructure:",squash"` - AdditionalDiskSize []uint `mapstructure:"additionaldisk_size"` - DiskName string `mapstructure:"vmdk_name"` - DiskSize uint `mapstructure:"disk_size"` - DiskTypeId string `mapstructure:"disk_type_id"` - FloppyFiles []string `mapstructure:"floppy_files"` - GuestOSType string `mapstructure:"guest_os_type"` - ISOChecksum string `mapstructure:"iso_checksum"` - ISOChecksumType string `mapstructure:"iso_checksum_type"` - ISOUrls []string `mapstructure:"iso_urls"` - Version string `mapstructure:"version"` - VMName string `mapstructure:"vm_name"` - BootCommand []string `mapstructure:"boot_command"` - SkipCompaction bool `mapstructure:"skip_compaction"` - VMXTemplatePath string `mapstructure:"vmx_template_path"` + AdditionalDiskSize []uint `mapstructure:"disk_additional_size"` + DiskName string `mapstructure:"vmdk_name"` + DiskSize uint `mapstructure:"disk_size"` + DiskTypeId string `mapstructure:"disk_type_id"` + FloppyFiles []string `mapstructure:"floppy_files"` + GuestOSType string `mapstructure:"guest_os_type"` + ISOChecksum string `mapstructure:"iso_checksum"` + ISOChecksumType string `mapstructure:"iso_checksum_type"` + ISOUrls []string `mapstructure:"iso_urls"` + Version string `mapstructure:"version"` + VMName string `mapstructure:"vm_name"` + BootCommand []string `mapstructure:"boot_command"` + SkipCompaction bool `mapstructure:"skip_compaction"` + VMXTemplatePath string `mapstructure:"vmx_template_path"` RemoteType string `mapstructure:"remote_type"` RemoteDatastore string `mapstructure:"remote_datastore"` diff --git a/website/source/docs/builders/vmware-iso.html.markdown b/website/source/docs/builders/vmware-iso.html.markdown index b021ca532..177bfc608 100644 --- a/website/source/docs/builders/vmware-iso.html.markdown +++ b/website/source/docs/builders/vmware-iso.html.markdown @@ -72,7 +72,7 @@ each category, the available options are alphabetized and described. ### Optional: -* `additionaldisk_size` (array of integers) - The size(s) of any additional +* `disk_additional_size` (array of integers) - The size(s) of any additional hard disks for the VM in megabytes. If this is not specified then the VM will only contain a primary hard disk. The builder uses expandable, not fixed-size virtual hard disks, so the actual file representing the disk will not use the From c9b413e3d1214f94fae52f91666e277a6c5fa85e Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Wed, 10 Jun 2015 15:18:05 -0500 Subject: [PATCH 070/208] builder/amazon-instance: Omit access, secrety key if using IAM Instance Profile --- builder/amazon/instance/builder.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index d164f91ee..91355b913 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -73,15 +73,25 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { } if b.config.BundleUploadCommand == "" { - b.config.BundleUploadCommand = "sudo -i -n ec2-upload-bundle " + - "-b {{.BucketName}} " + - "-m {{.ManifestPath}} " + - "-a {{.AccessKey}} " + - "-s {{.SecretKey}} " + - "-d {{.BundleDirectory}} " + - "--batch " + - "--region {{.Region}} " + - "--retry" + if b.config.IamInstanceProfile != "" { + b.config.BundleUploadCommand = "sudo -i -n ec2-upload-bundle " + + "-b {{.BucketName}} " + + "-m {{.ManifestPath}} " + + "-d {{.BundleDirectory}} " + + "--batch " + + "--region {{.Region}} " + + "--retry" + } else { + b.config.BundleUploadCommand = "sudo -i -n ec2-upload-bundle " + + "-b {{.BucketName}} " + + "-m {{.ManifestPath}} " + + "-a {{.AccessKey}} " + + "-s {{.SecretKey}} " + + "-d {{.BundleDirectory}} " + + "--batch " + + "--region {{.Region}} " + + "--retry" + } } if b.config.BundleVolCommand == "" { From d9c48e82fbc06d42d2b840eded67d344874ee53b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:02:06 -0700 Subject: [PATCH 071/208] builder/digitalocean: switch to new lib --- builder/digitalocean/api.go | 76 ---- builder/digitalocean/api_v1.go | 382 ---------------- builder/digitalocean/api_v2.go | 462 -------------------- builder/digitalocean/artifact.go | 9 +- builder/digitalocean/builder.go | 196 +-------- builder/digitalocean/builder_acc_test.go | 30 ++ builder/digitalocean/builder_test.go | 84 ---- builder/digitalocean/config.go | 140 ++++++ builder/digitalocean/step_create_droplet.go | 39 +- builder/digitalocean/step_create_ssh_key.go | 26 +- builder/digitalocean/step_droplet_info.go | 20 +- builder/digitalocean/step_power_off.go | 11 +- builder/digitalocean/step_shutdown.go | 9 +- builder/digitalocean/step_snapshot.go | 14 +- builder/digitalocean/token_source.go | 15 + builder/digitalocean/wait.go | 10 +- 16 files changed, 277 insertions(+), 1246 deletions(-) delete mode 100644 builder/digitalocean/api.go delete mode 100644 builder/digitalocean/api_v1.go delete mode 100644 builder/digitalocean/api_v2.go create mode 100644 builder/digitalocean/builder_acc_test.go create mode 100644 builder/digitalocean/config.go create mode 100644 builder/digitalocean/token_source.go diff --git a/builder/digitalocean/api.go b/builder/digitalocean/api.go deleted file mode 100644 index 87339ffc9..000000000 --- a/builder/digitalocean/api.go +++ /dev/null @@ -1,76 +0,0 @@ -// All of the methods used to communicate with the digital_ocean API -// are here. Their API is on a path to V2, so just plain JSON is used -// in place of a proper client library for now. - -package digitalocean - -type Region struct { - Slug string `json:"slug"` - Name string `json:"name"` - - // v1 only - Id uint `json:"id,omitempty"` - - // v2 only - Sizes []string `json:"sizes,omitempty"` - Available bool `json:"available,omitempty"` - Features []string `json:"features,omitempty"` -} - -type RegionsResp struct { - Regions []Region -} - -type Size struct { - Slug string `json:"slug"` - - // v1 only - Id uint `json:"id,omitempty"` - Name string `json:"name,omitempty"` - - // v2 only - Memory uint `json:"memory,omitempty"` - VCPUS uint `json:"vcpus,omitempty"` - Disk uint `json:"disk,omitempty"` - Transfer float64 `json:"transfer,omitempty"` - PriceMonthly float64 `json:"price_monthly,omitempty"` - PriceHourly float64 `json:"price_hourly,omitempty"` -} - -type SizesResp struct { - Sizes []Size -} - -type Image struct { - Id uint `json:"id"` - Name string `json:"name"` - Slug string `json:"slug"` - Distribution string `json:"distribution"` - - // v2 only - Public bool `json:"public,omitempty"` - ActionIds []string `json:"action_ids,omitempty"` - CreatedAt string `json:"created_at,omitempty"` -} - -type ImagesResp struct { - Images []Image -} - -type DigitalOceanClient interface { - CreateKey(string, string) (uint, error) - DestroyKey(uint) error - CreateDroplet(string, string, string, string, uint, bool) (uint, error) - DestroyDroplet(uint) error - PowerOffDroplet(uint) error - ShutdownDroplet(uint) error - CreateSnapshot(uint, string) error - Images() ([]Image, error) - DestroyImage(uint) error - DropletStatus(uint) (string, string, error) - Image(string) (Image, error) - Regions() ([]Region, error) - Region(string) (Region, error) - Sizes() ([]Size, error) - Size(string) (Size, error) -} diff --git a/builder/digitalocean/api_v1.go b/builder/digitalocean/api_v1.go deleted file mode 100644 index 23746d11f..000000000 --- a/builder/digitalocean/api_v1.go +++ /dev/null @@ -1,382 +0,0 @@ -// All of the methods used to communicate with the digital_ocean API -// are here. Their API is on a path to V2, so just plain JSON is used -// in place of a proper client library for now. - -package digitalocean - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "log" - "net/http" - "net/url" - "strconv" - "strings" - "time" - - "github.com/mitchellh/mapstructure" -) - -type DigitalOceanClientV1 struct { - // The http client for communicating - client *http.Client - - // Credentials - ClientID string - APIKey string - // The base URL of the API - APIURL string -} - -// Creates a new client for communicating with DO -func DigitalOceanClientNewV1(client string, key string, url string) *DigitalOceanClientV1 { - c := &DigitalOceanClientV1{ - client: &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - }, - }, - APIURL: url, - ClientID: client, - APIKey: key, - } - return c -} - -// Creates an SSH Key and returns it's id -func (d DigitalOceanClientV1) CreateKey(name string, pub string) (uint, error) { - params := url.Values{} - params.Set("name", name) - params.Set("ssh_pub_key", pub) - - body, err := NewRequestV1(d, "ssh_keys/new", params) - if err != nil { - return 0, err - } - - // Read the SSH key's ID we just created - key := body["ssh_key"].(map[string]interface{}) - keyId := key["id"].(float64) - return uint(keyId), nil -} - -// Destroys an SSH key -func (d DigitalOceanClientV1) DestroyKey(id uint) error { - path := fmt.Sprintf("ssh_keys/%v/destroy", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Creates a droplet and returns it's id -func (d DigitalOceanClientV1) CreateDroplet(name string, size string, image string, region string, keyId uint, privateNetworking bool) (uint, error) { - params := url.Values{} - params.Set("name", name) - - found_size, err := d.Size(size) - if err != nil { - return 0, fmt.Errorf("Invalid size or lookup failure: '%s': %s", size, err) - } - - found_image, err := d.Image(image) - if err != nil { - return 0, fmt.Errorf("Invalid image or lookup failure: '%s': %s", image, err) - } - - found_region, err := d.Region(region) - if err != nil { - return 0, fmt.Errorf("Invalid region or lookup failure: '%s': %s", region, err) - } - - params.Set("size_slug", found_size.Slug) - params.Set("image_slug", found_image.Slug) - params.Set("region_slug", found_region.Slug) - params.Set("ssh_key_ids", fmt.Sprintf("%v", keyId)) - params.Set("private_networking", fmt.Sprintf("%v", privateNetworking)) - - body, err := NewRequestV1(d, "droplets/new", params) - if err != nil { - return 0, err - } - - // Read the Droplets ID - droplet := body["droplet"].(map[string]interface{}) - dropletId := droplet["id"].(float64) - return uint(dropletId), err -} - -// Destroys a droplet -func (d DigitalOceanClientV1) DestroyDroplet(id uint) error { - path := fmt.Sprintf("droplets/%v/destroy", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Powers off a droplet -func (d DigitalOceanClientV1) PowerOffDroplet(id uint) error { - path := fmt.Sprintf("droplets/%v/power_off", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Shutsdown a droplet. This is a "soft" shutdown. -func (d DigitalOceanClientV1) ShutdownDroplet(id uint) error { - path := fmt.Sprintf("droplets/%v/shutdown", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Creates a snaphot of a droplet by it's ID -func (d DigitalOceanClientV1) CreateSnapshot(id uint, name string) error { - path := fmt.Sprintf("droplets/%v/snapshot", id) - - params := url.Values{} - params.Set("name", name) - - _, err := NewRequestV1(d, path, params) - - return err -} - -// Returns all available images. -func (d DigitalOceanClientV1) Images() ([]Image, error) { - resp, err := NewRequestV1(d, "images", url.Values{}) - if err != nil { - return nil, err - } - - var result ImagesResp - if err := mapstructure.Decode(resp, &result); err != nil { - return nil, err - } - - return result.Images, nil -} - -// Destroys an image by its ID. -func (d DigitalOceanClientV1) DestroyImage(id uint) error { - path := fmt.Sprintf("images/%d/destroy", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Returns DO's string representation of status "off" "new" "active" etc. -func (d DigitalOceanClientV1) DropletStatus(id uint) (string, string, error) { - path := fmt.Sprintf("droplets/%v", id) - - body, err := NewRequestV1(d, path, url.Values{}) - if err != nil { - return "", "", err - } - - var ip string - - // Read the droplet's "status" - droplet := body["droplet"].(map[string]interface{}) - status := droplet["status"].(string) - - if droplet["ip_address"] != nil { - ip = droplet["ip_address"].(string) - } - - return ip, status, err -} - -// Sends an api request and returns a generic map[string]interface of -// the response. -func NewRequestV1(d DigitalOceanClientV1, path string, params url.Values) (map[string]interface{}, error) { - client := d.client - - // Add the authentication parameters - params.Set("client_id", d.ClientID) - params.Set("api_key", d.APIKey) - - url := fmt.Sprintf("%s/%s?%s", d.APIURL, path, params.Encode()) - - // Do some basic scrubbing so sensitive information doesn't appear in logs - scrubbedUrl := strings.Replace(url, d.ClientID, "CLIENT_ID", -1) - scrubbedUrl = strings.Replace(scrubbedUrl, d.APIKey, "API_KEY", -1) - log.Printf("sending new request to digitalocean: %s", scrubbedUrl) - - var lastErr error - for attempts := 1; attempts < 10; attempts++ { - resp, err := client.Get(url) - if err != nil { - return nil, err - } - - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return nil, err - } - - log.Printf("response from digitalocean: %s", body) - - var decodedResponse map[string]interface{} - err = json.Unmarshal(body, &decodedResponse) - if err != nil { - err = errors.New(fmt.Sprintf("Failed to decode JSON response (HTTP %v) from DigitalOcean: %s", - resp.StatusCode, body)) - return decodedResponse, err - } - - // Check for errors sent by digitalocean - status := decodedResponse["status"].(string) - if status == "OK" { - return decodedResponse, nil - } - - if status == "ERROR" { - statusRaw, ok := decodedResponse["error_message"] - if ok { - status = statusRaw.(string) - } else { - status = fmt.Sprintf( - "Unknown error. Full response body: %s", body) - } - } - - lastErr = errors.New(fmt.Sprintf("Received error from DigitalOcean (%d): %s", - resp.StatusCode, status)) - log.Println(lastErr) - if strings.Contains(status, "a pending event") { - // Retry, DigitalOcean sends these dumb "pending event" - // errors all the time. - time.Sleep(5 * time.Second) - continue - } - - // Some other kind of error. Just return. - return decodedResponse, lastErr - } - - return nil, lastErr -} - -func (d DigitalOceanClientV1) Image(slug_or_name_or_id string) (Image, error) { - images, err := d.Images() - if err != nil { - return Image{}, err - } - - for _, image := range images { - if strings.EqualFold(image.Slug, slug_or_name_or_id) { - return image, nil - } - } - - for _, image := range images { - if strings.EqualFold(image.Name, slug_or_name_or_id) { - return image, nil - } - } - - for _, image := range images { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if image.Id == uint(id) { - return image, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown image '%v'", slug_or_name_or_id)) - - return Image{}, err -} - -// Returns all available regions. -func (d DigitalOceanClientV1) Regions() ([]Region, error) { - resp, err := NewRequestV1(d, "regions", url.Values{}) - if err != nil { - return nil, err - } - - var result RegionsResp - if err := mapstructure.Decode(resp, &result); err != nil { - return nil, err - } - - return result.Regions, nil -} - -func (d DigitalOceanClientV1) Region(slug_or_name_or_id string) (Region, error) { - regions, err := d.Regions() - if err != nil { - return Region{}, err - } - - for _, region := range regions { - if strings.EqualFold(region.Slug, slug_or_name_or_id) { - return region, nil - } - } - - for _, region := range regions { - if strings.EqualFold(region.Name, slug_or_name_or_id) { - return region, nil - } - } - - for _, region := range regions { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if region.Id == uint(id) { - return region, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown region '%v'", slug_or_name_or_id)) - - return Region{}, err -} - -// Returns all available sizes. -func (d DigitalOceanClientV1) Sizes() ([]Size, error) { - resp, err := NewRequestV1(d, "sizes", url.Values{}) - if err != nil { - return nil, err - } - - var result SizesResp - if err := mapstructure.Decode(resp, &result); err != nil { - return nil, err - } - - return result.Sizes, nil -} - -func (d DigitalOceanClientV1) Size(slug_or_name_or_id string) (Size, error) { - sizes, err := d.Sizes() - if err != nil { - return Size{}, err - } - - for _, size := range sizes { - if strings.EqualFold(size.Slug, slug_or_name_or_id) { - return size, nil - } - } - - for _, size := range sizes { - if strings.EqualFold(size.Name, slug_or_name_or_id) { - return size, nil - } - } - - for _, size := range sizes { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if size.Id == uint(id) { - return size, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown size '%v'", slug_or_name_or_id)) - - return Size{}, err -} diff --git a/builder/digitalocean/api_v2.go b/builder/digitalocean/api_v2.go deleted file mode 100644 index 46454a9f8..000000000 --- a/builder/digitalocean/api_v2.go +++ /dev/null @@ -1,462 +0,0 @@ -// are here. Their API is on a path to V2, so just plain JSON is used -// in place of a proper client library for now. - -package digitalocean - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "log" - "net/http" - "strconv" - "strings" -) - -type DigitalOceanClientV2 struct { - // The http client for communicating - client *http.Client - - // Credentials - APIToken string - - // The base URL of the API - APIURL string -} - -// Creates a new client for communicating with DO -func DigitalOceanClientNewV2(token string, url string) *DigitalOceanClientV2 { - c := &DigitalOceanClientV2{ - client: &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - }, - }, - APIURL: url, - APIToken: token, - } - return c -} - -// Creates an SSH Key and returns it's id -func (d DigitalOceanClientV2) CreateKey(name string, pub string) (uint, error) { - type KeyReq struct { - Name string `json:"name"` - PublicKey string `json:"public_key"` - } - type KeyRes struct { - SSHKey struct { - Id uint - Name string - Fingerprint string - PublicKey string `json:"public_key"` - } `json:"ssh_key"` - } - req := &KeyReq{Name: name, PublicKey: pub} - res := KeyRes{} - err := NewRequestV2(d, "v2/account/keys", "POST", req, &res) - if err != nil { - return 0, err - } - - return res.SSHKey.Id, err -} - -// Destroys an SSH key -func (d DigitalOceanClientV2) DestroyKey(id uint) error { - path := fmt.Sprintf("v2/account/keys/%v", id) - return NewRequestV2(d, path, "DELETE", nil, nil) -} - -// Creates a droplet and returns it's id -func (d DigitalOceanClientV2) CreateDroplet(name string, size string, image string, region string, keyId uint, privateNetworking bool) (uint, error) { - type DropletReq struct { - Name string `json:"name"` - Region string `json:"region"` - Size string `json:"size"` - Image string `json:"image"` - SSHKeys []string `json:"ssh_keys,omitempty"` - Backups bool `json:"backups,omitempty"` - IPv6 bool `json:"ipv6,omitempty"` - PrivateNetworking bool `json:"private_networking,omitempty"` - } - type DropletRes struct { - Droplet struct { - Id uint - Name string - Memory uint - VCPUS uint `json:"vcpus"` - Disk uint - Region Region - Image Image - Size Size - Locked bool - CreateAt string `json:"created_at"` - Status string - Networks struct { - V4 []struct { - IPAddr string `json:"ip_address"` - Netmask string - Gateway string - Type string - } `json:"v4,omitempty"` - V6 []struct { - IPAddr string `json:"ip_address"` - CIDR uint `json:"cidr"` - Gateway string - Type string - } `json:"v6,omitempty"` - } - Kernel struct { - Id uint - Name string - Version string - } - BackupIds []uint - SnapshotIds []uint - ActionIds []uint - Features []string `json:"features,omitempty"` - } - } - req := &DropletReq{Name: name} - res := DropletRes{} - - found_size, err := d.Size(size) - if err != nil { - return 0, fmt.Errorf("Invalid size or lookup failure: '%s': %s", size, err) - } - - found_image, err := d.Image(image) - if err != nil { - return 0, fmt.Errorf("Invalid image or lookup failure: '%s': %s", image, err) - } - - found_region, err := d.Region(region) - if err != nil { - return 0, fmt.Errorf("Invalid region or lookup failure: '%s': %s", region, err) - } - - if found_image.Slug == "" { - req.Image = strconv.Itoa(int(found_image.Id)) - } else { - req.Image = found_image.Slug - } - - req.Size = found_size.Slug - req.Region = found_region.Slug - req.SSHKeys = []string{fmt.Sprintf("%v", keyId)} - req.PrivateNetworking = privateNetworking - - err = NewRequestV2(d, "v2/droplets", "POST", req, &res) - if err != nil { - return 0, err - } - - return res.Droplet.Id, err -} - -// Destroys a droplet -func (d DigitalOceanClientV2) DestroyDroplet(id uint) error { - path := fmt.Sprintf("v2/droplets/%v", id) - return NewRequestV2(d, path, "DELETE", nil, nil) -} - -// Powers off a droplet -func (d DigitalOceanClientV2) PowerOffDroplet(id uint) error { - type ActionReq struct { - Type string `json:"type"` - } - type ActionRes struct { - } - req := &ActionReq{Type: "power_off"} - path := fmt.Sprintf("v2/droplets/%v/actions", id) - return NewRequestV2(d, path, "POST", req, nil) -} - -// Shutsdown a droplet. This is a "soft" shutdown. -func (d DigitalOceanClientV2) ShutdownDroplet(id uint) error { - type ActionReq struct { - Type string `json:"type"` - } - type ActionRes struct { - } - req := &ActionReq{Type: "shutdown"} - - path := fmt.Sprintf("v2/droplets/%v/actions", id) - return NewRequestV2(d, path, "POST", req, nil) -} - -// Creates a snaphot of a droplet by it's ID -func (d DigitalOceanClientV2) CreateSnapshot(id uint, name string) error { - type ActionReq struct { - Type string `json:"type"` - Name string `json:"name"` - } - type ActionRes struct { - } - req := &ActionReq{Type: "snapshot", Name: name} - path := fmt.Sprintf("v2/droplets/%v/actions", id) - return NewRequestV2(d, path, "POST", req, nil) -} - -// Returns all available images. -func (d DigitalOceanClientV2) Images() ([]Image, error) { - res := ImagesResp{} - - err := NewRequestV2(d, "v2/images?per_page=200", "GET", nil, &res) - if err != nil { - return nil, err - } - - return res.Images, nil -} - -// Destroys an image by its ID. -func (d DigitalOceanClientV2) DestroyImage(id uint) error { - path := fmt.Sprintf("v2/images/%d", id) - return NewRequestV2(d, path, "DELETE", nil, nil) -} - -// Returns DO's string representation of status "off" "new" "active" etc. -func (d DigitalOceanClientV2) DropletStatus(id uint) (string, string, error) { - path := fmt.Sprintf("v2/droplets/%v", id) - type DropletRes struct { - Droplet struct { - Id uint - Name string - Memory uint - VCPUS uint `json:"vcpus"` - Disk uint - Region Region - Image Image - Size Size - Locked bool - CreateAt string `json:"created_at"` - Status string - Networks struct { - V4 []struct { - IPAddr string `json:"ip_address"` - Netmask string - Gateway string - Type string - } `json:"v4,omitempty"` - V6 []struct { - IPAddr string `json:"ip_address"` - CIDR uint `json:"cidr"` - Gateway string - Type string - } `json:"v6,omitempty"` - } - Kernel struct { - Id uint - Name string - Version string - } - BackupIds []uint - SnapshotIds []uint - ActionIds []uint - Features []string `json:"features,omitempty"` - } - } - res := DropletRes{} - err := NewRequestV2(d, path, "GET", nil, &res) - if err != nil { - return "", "", err - } - var ip string - - for _, n := range res.Droplet.Networks.V4 { - if n.Type == "public" { - ip = n.IPAddr - } - } - - return ip, res.Droplet.Status, err -} - -// Sends an api request and returns a generic map[string]interface of -// the response. -func NewRequestV2(d DigitalOceanClientV2, path string, method string, req interface{}, res interface{}) error { - var err error - var request *http.Request - - client := d.client - - buf := new(bytes.Buffer) - // Add the authentication parameters - url := fmt.Sprintf("%s/%s", d.APIURL, path) - if req != nil { - enc := json.NewEncoder(buf) - enc.Encode(req) - defer buf.Reset() - request, err = http.NewRequest(method, url, buf) - request.Header.Add("Content-Type", "application/json") - } else { - request, err = http.NewRequest(method, url, nil) - } - if err != nil { - return err - } - - // Add the authentication parameters - request.Header.Add("Authorization", "Bearer "+d.APIToken) - if buf != nil { - log.Printf("sending new request to digitalocean: %s buffer: %s", url, buf) - } else { - log.Printf("sending new request to digitalocean: %s", url) - } - resp, err := client.Do(request) - if err != nil { - return err - } - - if method == "DELETE" && resp.StatusCode == 204 { - if resp.Body != nil { - resp.Body.Close() - } - return nil - } - - if resp.Body == nil { - return errors.New("Request returned empty body") - } - - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return err - } - - log.Printf("response from digitalocean: %s", body) - - err = json.Unmarshal(body, &res) - if err != nil { - return errors.New(fmt.Sprintf("Failed to decode JSON response %s (HTTP %v) from DigitalOcean: %s", err.Error(), - resp.StatusCode, body)) - } - switch resp.StatusCode { - case 403, 401, 429, 422, 404, 503, 500: - return errors.New(fmt.Sprintf("digitalocean request error: %+v", res)) - } - return nil -} - -func (d DigitalOceanClientV2) Image(slug_or_name_or_id string) (Image, error) { - images, err := d.Images() - if err != nil { - return Image{}, err - } - - for _, image := range images { - if strings.EqualFold(image.Slug, slug_or_name_or_id) { - return image, nil - } - } - - for _, image := range images { - if strings.EqualFold(image.Name, slug_or_name_or_id) { - return image, nil - } - } - - for _, image := range images { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if image.Id == uint(id) { - return image, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown image '%v'", slug_or_name_or_id)) - - return Image{}, err -} - -// Returns all available regions. -func (d DigitalOceanClientV2) Regions() ([]Region, error) { - res := RegionsResp{} - err := NewRequestV2(d, "v2/regions?per_page=200", "GET", nil, &res) - if err != nil { - return nil, err - } - - return res.Regions, nil -} - -func (d DigitalOceanClientV2) Region(slug_or_name_or_id string) (Region, error) { - regions, err := d.Regions() - if err != nil { - return Region{}, err - } - - for _, region := range regions { - if strings.EqualFold(region.Slug, slug_or_name_or_id) { - return region, nil - } - } - - for _, region := range regions { - if strings.EqualFold(region.Name, slug_or_name_or_id) { - return region, nil - } - } - - for _, region := range regions { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if region.Id == uint(id) { - return region, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown region '%v'", slug_or_name_or_id)) - - return Region{}, err -} - -// Returns all available sizes. -func (d DigitalOceanClientV2) Sizes() ([]Size, error) { - res := SizesResp{} - err := NewRequestV2(d, "v2/sizes?per_page=200", "GET", nil, &res) - if err != nil { - return nil, err - } - - return res.Sizes, nil -} - -func (d DigitalOceanClientV2) Size(slug_or_name_or_id string) (Size, error) { - sizes, err := d.Sizes() - if err != nil { - return Size{}, err - } - - for _, size := range sizes { - if strings.EqualFold(size.Slug, slug_or_name_or_id) { - return size, nil - } - } - - for _, size := range sizes { - if strings.EqualFold(size.Name, slug_or_name_or_id) { - return size, nil - } - } - - for _, size := range sizes { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if size.Id == uint(id) { - return size, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown size '%v'", slug_or_name_or_id)) - - return Size{}, err -} diff --git a/builder/digitalocean/artifact.go b/builder/digitalocean/artifact.go index d1d878193..3b6a05e53 100644 --- a/builder/digitalocean/artifact.go +++ b/builder/digitalocean/artifact.go @@ -4,6 +4,8 @@ import ( "fmt" "log" "strconv" + + "github.com/digitalocean/godo" ) type Artifact struct { @@ -11,13 +13,13 @@ type Artifact struct { snapshotName string // The ID of the image - snapshotId uint + snapshotId int // The name of the region regionName string // The client for making API calls - client DigitalOceanClient + client *godo.Client } func (*Artifact) BuilderId() string { @@ -43,5 +45,6 @@ func (a *Artifact) State(name string) interface{} { func (a *Artifact) Destroy() error { log.Printf("Destroying image: %d (%s)", a.snapshotId, a.snapshotName) - return a.client.DestroyImage(a.snapshotId) + _, err := a.client.Images.Delete(a.snapshotId) + return err } diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index 3292bea10..3ba7074a2 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -4,18 +4,14 @@ package digitalocean import ( - "errors" - "fmt" "log" - "os" "time" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common" - "github.com/mitchellh/packer/common/uuid" - "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" - "github.com/mitchellh/packer/template/interpolate" + "golang.org/x/oauth2" ) // see https://api.digitalocean.com/images/?client_id=[client_id]&api_key=[api_key] @@ -33,179 +29,25 @@ const DefaultSize = "512mb" // The unique id for the builder const BuilderId = "pearkes.digitalocean" -// Configuration tells the builder the credentials -// to use while communicating with DO and describes the image -// you are creating -type Config struct { - common.PackerConfig `mapstructure:",squash"` - - ClientID string `mapstructure:"client_id"` - APIKey string `mapstructure:"api_key"` - APIURL string `mapstructure:"api_url"` - APIToken string `mapstructure:"api_token"` - RegionID uint `mapstructure:"region_id"` - SizeID uint `mapstructure:"size_id"` - ImageID uint `mapstructure:"image_id"` - - Region string `mapstructure:"region"` - Size string `mapstructure:"size"` - Image string `mapstructure:"image"` - - PrivateNetworking bool `mapstructure:"private_networking"` - SnapshotName string `mapstructure:"snapshot_name"` - DropletName string `mapstructure:"droplet_name"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort uint `mapstructure:"ssh_port"` - - RawSSHTimeout string `mapstructure:"ssh_timeout"` - RawStateTimeout string `mapstructure:"state_timeout"` - - // These are unexported since they're set by other fields - // being set. - sshTimeout time.Duration - stateTimeout time.Duration - - ctx *interpolate.Context -} - type Builder struct { config Config runner multistep.Runner } func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { - err := config.Decode(&b.config, &config.DecodeOpts{ - Interpolate: true, - }, raws...) - if err != nil { - return nil, err + c, warnings, errs := NewConfig(raws...) + if errs != nil { + return warnings, errs } + b.config = *c - // Optional configuration with defaults - if b.config.APIKey == "" { - // Default to environment variable for api_key, if it exists - b.config.APIKey = os.Getenv("DIGITALOCEAN_API_KEY") - } - - if b.config.ClientID == "" { - // Default to environment variable for client_id, if it exists - b.config.ClientID = os.Getenv("DIGITALOCEAN_CLIENT_ID") - } - - if b.config.APIURL == "" { - // Default to environment variable for api_url, if it exists - b.config.APIURL = os.Getenv("DIGITALOCEAN_API_URL") - } - - if b.config.APIToken == "" { - // Default to environment variable for api_token, if it exists - b.config.APIToken = os.Getenv("DIGITALOCEAN_API_TOKEN") - } - - if b.config.Region == "" { - if b.config.RegionID != 0 { - b.config.Region = fmt.Sprintf("%v", b.config.RegionID) - } else { - b.config.Region = DefaultRegion - } - } - - if b.config.Size == "" { - if b.config.SizeID != 0 { - b.config.Size = fmt.Sprintf("%v", b.config.SizeID) - } else { - b.config.Size = DefaultSize - } - } - - if b.config.Image == "" { - if b.config.ImageID != 0 { - b.config.Image = fmt.Sprintf("%v", b.config.ImageID) - } else { - b.config.Image = DefaultImage - } - } - - if b.config.SnapshotName == "" { - // Default to packer-{{ unix timestamp (utc) }} - b.config.SnapshotName = "packer-{{timestamp}}" - } - - if b.config.DropletName == "" { - // Default to packer-[time-ordered-uuid] - b.config.DropletName = fmt.Sprintf("packer-%s", uuid.TimeOrderedUUID()) - } - - if b.config.SSHUsername == "" { - // Default to "root". You can override this if your - // SourceImage has a different user account then the DO default - b.config.SSHUsername = "root" - } - - if b.config.SSHPort == 0 { - // Default to port 22 per DO default - b.config.SSHPort = 22 - } - - if b.config.RawSSHTimeout == "" { - // Default to 1 minute timeouts - b.config.RawSSHTimeout = "1m" - } - - if b.config.RawStateTimeout == "" { - // Default to 6 minute timeouts waiting for - // desired state. i.e waiting for droplet to become active - b.config.RawStateTimeout = "6m" - } - - var errs *packer.MultiError - if b.config.APIToken == "" { - // Required configurations that will display errors if not set - if b.config.ClientID == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("a client_id for v1 auth or api_token for v2 auth must be specified")) - } - - if b.config.APIKey == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("a api_key for v1 auth or api_token for v2 auth must be specified")) - } - } - - if b.config.APIURL == "" { - b.config.APIURL = "https://api.digitalocean.com" - } - - sshTimeout, err := time.ParseDuration(b.config.RawSSHTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - b.config.sshTimeout = sshTimeout - - stateTimeout, err := time.ParseDuration(b.config.RawStateTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing state_timeout: %s", err)) - } - b.config.stateTimeout = stateTimeout - - if errs != nil && len(errs.Errors) > 0 { - return nil, errs - } - - common.ScrubConfig(b.config, b.config.ClientID, b.config.APIKey) return nil, nil } func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { - var client DigitalOceanClient - // Initialize the DO API client - if b.config.APIToken == "" { - client = DigitalOceanClientNewV1(b.config.ClientID, b.config.APIKey, b.config.APIURL) - } else { - client = DigitalOceanClientNewV2(b.config.APIToken, b.config.APIURL) - } + client := godo.NewClient(oauth2.NewClient(oauth2.NoContext, &apiTokenSource{ + AccessToken: b.config.APIToken, + })) // Set up the state state := new(multistep.BasicStateBag) @@ -252,26 +94,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe return nil, nil } - sregion := state.Get("region") - - var region string - - if sregion != nil { - region = sregion.(string) - } else { - region = fmt.Sprintf("%v", state.Get("region_id").(uint)) - } - - found_region, err := client.Region(region) - - if err != nil { - return nil, err - } - artifact := &Artifact{ snapshotName: state.Get("snapshot_name").(string), - snapshotId: state.Get("snapshot_image_id").(uint), - regionName: found_region.Name, + snapshotId: state.Get("snapshot_image_id").(int), + regionName: state.Get("region").(string), client: client, } diff --git a/builder/digitalocean/builder_acc_test.go b/builder/digitalocean/builder_acc_test.go new file mode 100644 index 000000000..20e56b924 --- /dev/null +++ b/builder/digitalocean/builder_acc_test.go @@ -0,0 +1,30 @@ +package digitalocean + +import ( + "os" + "testing" + + builderT "github.com/mitchellh/packer/helper/builder/testing" +) + +func TestBuilderAcc_basic(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Builder: &Builder{}, + Template: testBuilderAccBasic, + }) +} + +func testAccPreCheck(t *testing.T) { + if v := os.Getenv("DIGITALOCEAN_API_TOKEN"); v == "" { + t.Fatal("DIGITALOCEAN_API_TOKEN must be set for acceptance tests") + } +} + +const testBuilderAccBasic = ` +{ + "builders": [{ + "type": "test" + }] +} +` diff --git a/builder/digitalocean/builder_test.go b/builder/digitalocean/builder_test.go index bd3bb1d21..8985aae2f 100644 --- a/builder/digitalocean/builder_test.go +++ b/builder/digitalocean/builder_test.go @@ -43,90 +43,6 @@ func TestBuilder_Prepare_BadType(t *testing.T) { } } -func TestBuilderPrepare_APIKey(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["api_key"] = "foo" - warnings, err := b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.APIKey != "foo" { - t.Errorf("access key invalid: %s", b.config.APIKey) - } - - // Test bad - delete(config, "api_key") - b = Builder{} - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err == nil { - t.Fatal("should have error") - } - - // Test env variable - delete(config, "api_key") - os.Setenv("DIGITALOCEAN_API_KEY", "foo") - defer os.Setenv("DIGITALOCEAN_API_KEY", "") - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } -} - -func TestBuilderPrepare_ClientID(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["client_id"] = "foo" - warnings, err := b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.ClientID != "foo" { - t.Errorf("invalid: %s", b.config.ClientID) - } - - // Test bad - delete(config, "client_id") - b = Builder{} - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err == nil { - t.Fatal("should have error") - } - - // Test env variable - delete(config, "client_id") - os.Setenv("DIGITALOCEAN_CLIENT_ID", "foo") - defer os.Setenv("DIGITALOCEAN_CLIENT_ID", "") - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } -} - func TestBuilderPrepare_InvalidKey(t *testing.T) { var b Builder config := testConfig() diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go new file mode 100644 index 000000000..0be49f08e --- /dev/null +++ b/builder/digitalocean/config.go @@ -0,0 +1,140 @@ +package digitalocean + +import ( + "errors" + "fmt" + "os" + "time" + + "github.com/mitchellh/mapstructure" + "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/helper/config" + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/template/interpolate" + //"github.com/digitalocean/godo" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + APIToken string `mapstructure:"api_token"` + + // OLD STUFF + + Region string `mapstructure:"region"` + Size string `mapstructure:"size"` + Image string `mapstructure:"image"` + + PrivateNetworking bool `mapstructure:"private_networking"` + SnapshotName string `mapstructure:"snapshot_name"` + DropletName string `mapstructure:"droplet_name"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort uint `mapstructure:"ssh_port"` + + RawSSHTimeout string `mapstructure:"ssh_timeout"` + RawStateTimeout string `mapstructure:"state_timeout"` + + // These are unexported since they're set by other fields + // being set. + sshTimeout time.Duration + stateTimeout time.Duration + + ctx *interpolate.Context +} + +func NewConfig(raws ...interface{}) (*Config, []string, error) { + var c Config + + var md mapstructure.Metadata + err := config.Decode(&c, &config.DecodeOpts{ + Metadata: &md, + Interpolate: true, + InterpolateFilter: &interpolate.RenderFilter{ + Exclude: []string{ + "run_command", + }, + }, + }, raws...) + if err != nil { + return nil, nil, err + } + + // Defaults + if c.APIToken == "" { + // Default to environment variable for api_token, if it exists + c.APIToken = os.Getenv("DIGITALOCEAN_API_TOKEN") + } + + if c.Region == "" { + c.Region = DefaultRegion + } + + if c.Size == "" { + c.Size = DefaultSize + } + + if c.Image == "" { + c.Image = DefaultImage + } + + if c.SnapshotName == "" { + // Default to packer-{{ unix timestamp (utc) }} + c.SnapshotName = "packer-{{timestamp}}" + } + + if c.DropletName == "" { + // Default to packer-[time-ordered-uuid] + c.DropletName = fmt.Sprintf("packer-%s", uuid.TimeOrderedUUID()) + } + + if c.SSHUsername == "" { + // Default to "root". You can override this if your + // SourceImage has a different user account then the DO default + c.SSHUsername = "root" + } + + if c.SSHPort == 0 { + // Default to port 22 per DO default + c.SSHPort = 22 + } + + if c.RawSSHTimeout == "" { + // Default to 1 minute timeouts + c.RawSSHTimeout = "1m" + } + + if c.RawStateTimeout == "" { + // Default to 6 minute timeouts waiting for + // desired state. i.e waiting for droplet to become active + c.RawStateTimeout = "6m" + } + + var errs *packer.MultiError + if c.APIToken == "" { + // Required configurations that will display errors if not set + errs = packer.MultiErrorAppend( + errs, errors.New("api_token for auth must be specified")) + } + + sshTimeout, err := time.ParseDuration(c.RawSSHTimeout) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) + } + c.sshTimeout = sshTimeout + + stateTimeout, err := time.ParseDuration(c.RawStateTimeout) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Failed parsing state_timeout: %s", err)) + } + c.stateTimeout = stateTimeout + + if errs != nil && len(errs.Errors) > 0 { + return nil, nil, errs + } + + common.ScrubConfig(c, c.APIToken) + return &c, nil, nil +} diff --git a/builder/digitalocean/step_create_droplet.go b/builder/digitalocean/step_create_droplet.go index afb3e5814..40ac8f0e9 100644 --- a/builder/digitalocean/step_create_droplet.go +++ b/builder/digitalocean/step_create_droplet.go @@ -3,25 +3,35 @@ package digitalocean import ( "fmt" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) type stepCreateDroplet struct { - dropletId uint + dropletId int } func (s *stepCreateDroplet) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) c := state.Get("config").(Config) - sshKeyId := state.Get("ssh_key_id").(uint) - - ui.Say("Creating droplet...") + sshKeyId := state.Get("ssh_key_id").(int) // Create the droplet based on configuration - dropletId, err := client.CreateDroplet(c.DropletName, c.Size, c.Image, c.Region, sshKeyId, c.PrivateNetworking) - + ui.Say("Creating droplet...") + droplet, _, err := client.Droplets.Create(&godo.DropletCreateRequest{ + Name: c.DropletName, + Region: c.Region, + Size: c.Size, + Image: godo.DropletCreateImage{ + Slug: c.Image, + }, + SSHKeys: []godo.DropletCreateSSHKey{ + godo.DropletCreateSSHKey{ID: int(sshKeyId)}, + }, + PrivateNetworking: c.PrivateNetworking, + }) if err != nil { err := fmt.Errorf("Error creating droplet: %s", err) state.Put("error", err) @@ -30,10 +40,10 @@ func (s *stepCreateDroplet) Run(state multistep.StateBag) multistep.StepAction { } // We use this in cleanup - s.dropletId = dropletId + s.dropletId = droplet.ID // Store the droplet id for later - state.Put("droplet_id", dropletId) + state.Put("droplet_id", droplet.ID) return multistep.ActionContinue } @@ -44,19 +54,14 @@ func (s *stepCreateDroplet) Cleanup(state multistep.StateBag) { return } - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) - c := state.Get("config").(Config) // Destroy the droplet we just created ui.Say("Destroying droplet...") - - err := client.DestroyDroplet(s.dropletId) + _, err := client.Droplets.Delete(s.dropletId) if err != nil { - curlstr := fmt.Sprintf("curl '%v/droplets/%v/destroy?client_id=%v&api_key=%v'", - c.APIURL, s.dropletId, c.ClientID, c.APIKey) - ui.Error(fmt.Sprintf( - "Error destroying droplet. Please destroy it manually: %v", curlstr)) + "Error destroying droplet. Please destroy it manually: %s", err)) } } diff --git a/builder/digitalocean/step_create_ssh_key.go b/builder/digitalocean/step_create_ssh_key.go index db1ad9c16..fa0940c23 100644 --- a/builder/digitalocean/step_create_ssh_key.go +++ b/builder/digitalocean/step_create_ssh_key.go @@ -9,17 +9,18 @@ import ( "log" "code.google.com/p/gosshold/ssh" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common/uuid" "github.com/mitchellh/packer/packer" ) type stepCreateSSHKey struct { - keyId uint + keyId int } func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) ui.Say("Creating temporary ssh key for droplet...") @@ -46,7 +47,10 @@ func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { name := fmt.Sprintf("packer-%s", uuid.TimeOrderedUUID()) // Create the key! - keyId, err := client.CreateKey(name, pub_sshformat) + key, _, err := client.Keys.Create(&godo.KeyCreateRequest{ + Name: name, + PublicKey: pub_sshformat, + }) if err != nil { err := fmt.Errorf("Error creating temporary SSH key: %s", err) state.Put("error", err) @@ -55,12 +59,12 @@ func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { } // We use this to check cleanup - s.keyId = keyId + s.keyId = key.ID log.Printf("temporary ssh key name: %s", name) // Remember some state for the future - state.Put("ssh_key_id", keyId) + state.Put("ssh_key_id", key.ID) return multistep.ActionContinue } @@ -71,18 +75,14 @@ func (s *stepCreateSSHKey) Cleanup(state multistep.StateBag) { return } - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) - c := state.Get("config").(Config) ui.Say("Deleting temporary ssh key...") - err := client.DestroyKey(s.keyId) - - curlstr := fmt.Sprintf("curl -H 'Authorization: Bearer #TOKEN#' -X DELETE '%v/v2/account/keys/%v'", c.APIURL, s.keyId) - + _, err := client.Keys.DeleteByID(s.keyId) if err != nil { - log.Printf("Error cleaning up ssh key: %v", err.Error()) + log.Printf("Error cleaning up ssh key: %s", err) ui.Error(fmt.Sprintf( - "Error cleaning up ssh key. Please delete the key manually: %v", curlstr)) + "Error cleaning up ssh key. Please delete the key manually: %s", err)) } } diff --git a/builder/digitalocean/step_droplet_info.go b/builder/digitalocean/step_droplet_info.go index 8e9b69927..5fbcb7141 100644 --- a/builder/digitalocean/step_droplet_info.go +++ b/builder/digitalocean/step_droplet_info.go @@ -3,6 +3,7 @@ package digitalocean import ( "fmt" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) @@ -10,10 +11,10 @@ import ( type stepDropletInfo struct{} func (s *stepDropletInfo) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) c := state.Get("config").(Config) - dropletId := state.Get("droplet_id").(uint) + dropletId := state.Get("droplet_id").(int) ui.Say("Waiting for droplet to become active...") @@ -26,16 +27,25 @@ func (s *stepDropletInfo) Run(state multistep.StateBag) multistep.StepAction { } // Set the IP on the state for later - ip, _, err := client.DropletStatus(dropletId) + droplet, _, err := client.Droplets.Get(dropletId) if err != nil { - err := fmt.Errorf("Error retrieving droplet ID: %s", err) + err := fmt.Errorf("Error retrieving droplet: %s", err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - state.Put("droplet_ip", ip) + // Verify we have an IPv4 address + invalid := droplet.Networks == nil || + len(droplet.Networks.V4) == 0 + if invalid { + err := fmt.Errorf("IPv4 address not found for droplet!") + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + state.Put("droplet_ip", droplet.Networks.V4[0].IPAddress) return multistep.ActionContinue } diff --git a/builder/digitalocean/step_power_off.go b/builder/digitalocean/step_power_off.go index d6ef49a22..3d547e8c2 100644 --- a/builder/digitalocean/step_power_off.go +++ b/builder/digitalocean/step_power_off.go @@ -4,6 +4,7 @@ import ( "fmt" "log" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) @@ -11,12 +12,12 @@ import ( type stepPowerOff struct{} func (s *stepPowerOff) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) c := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) - dropletId := state.Get("droplet_id").(uint) + dropletId := state.Get("droplet_id").(int) - _, status, err := client.DropletStatus(dropletId) + droplet, _, err := client.Droplets.Get(dropletId) if err != nil { err := fmt.Errorf("Error checking droplet state: %s", err) state.Put("error", err) @@ -24,14 +25,14 @@ func (s *stepPowerOff) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - if status == "off" { + if droplet.Status == "off" { // Droplet is already off, don't do anything return multistep.ActionContinue } // Pull the plug on the Droplet ui.Say("Forcefully shutting down Droplet...") - err = client.PowerOffDroplet(dropletId) + _, _, err = client.DropletActions.PowerOff(dropletId) if err != nil { err := fmt.Errorf("Error powering off droplet: %s", err) state.Put("error", err) diff --git a/builder/digitalocean/step_shutdown.go b/builder/digitalocean/step_shutdown.go index 06a2ae9f5..602f3e690 100644 --- a/builder/digitalocean/step_shutdown.go +++ b/builder/digitalocean/step_shutdown.go @@ -5,6 +5,7 @@ import ( "log" "time" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) @@ -12,16 +13,16 @@ import ( type stepShutdown struct{} func (s *stepShutdown) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) - dropletId := state.Get("droplet_id").(uint) + dropletId := state.Get("droplet_id").(int) // Gracefully power off the droplet. We have to retry this a number // of times because sometimes it says it completed when it actually // did absolutely nothing (*ALAKAZAM!* magic!). We give up after // a pretty arbitrary amount of time. ui.Say("Gracefully shutting down droplet...") - err := client.ShutdownDroplet(dropletId) + _, _, err := client.DropletActions.Shutdown(dropletId) if err != nil { // If we get an error the first time, actually report it err := fmt.Errorf("Error shutting down droplet: %s", err) @@ -48,7 +49,7 @@ func (s *stepShutdown) Run(state multistep.StateBag) multistep.StepAction { for attempts := 2; attempts > 0; attempts++ { log.Printf("ShutdownDroplet attempt #%d...", attempts) - err := client.ShutdownDroplet(dropletId) + _, _, err := client.DropletActions.Shutdown(dropletId) if err != nil { log.Printf("Shutdown retry error: %s", err) } diff --git a/builder/digitalocean/step_snapshot.go b/builder/digitalocean/step_snapshot.go index 1903c1a34..7ff384924 100644 --- a/builder/digitalocean/step_snapshot.go +++ b/builder/digitalocean/step_snapshot.go @@ -5,6 +5,7 @@ import ( "fmt" "log" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) @@ -12,13 +13,13 @@ import ( type stepSnapshot struct{} func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) c := state.Get("config").(Config) - dropletId := state.Get("droplet_id").(uint) + dropletId := state.Get("droplet_id").(int) ui.Say(fmt.Sprintf("Creating snapshot: %v", c.SnapshotName)) - err := client.CreateSnapshot(dropletId, c.SnapshotName) + _, _, err := client.DropletActions.Snapshot(dropletId, c.SnapshotName) if err != nil { err := fmt.Errorf("Error creating snapshot: %s", err) state.Put("error", err) @@ -36,7 +37,7 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { } log.Printf("Looking up snapshot ID for snapshot: %s", c.SnapshotName) - images, err := client.Images() + images, _, err := client.Images.List(nil) if err != nil { err := fmt.Errorf("Error looking up snapshot ID: %s", err) state.Put("error", err) @@ -44,10 +45,10 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - var imageId uint + var imageId int for _, image := range images { if image.Name == c.SnapshotName { - imageId = image.Id + imageId = image.ID break } } @@ -60,7 +61,6 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { } log.Printf("Snapshot image ID: %d", imageId) - state.Put("snapshot_image_id", imageId) state.Put("snapshot_name", c.SnapshotName) state.Put("region", c.Region) diff --git a/builder/digitalocean/token_source.go b/builder/digitalocean/token_source.go new file mode 100644 index 000000000..eab5a084b --- /dev/null +++ b/builder/digitalocean/token_source.go @@ -0,0 +1,15 @@ +package digitalocean + +import ( + "golang.org/x/oauth2" +) + +type apiTokenSource struct { + AccessToken string +} + +func (t *apiTokenSource) Token() (*oauth2.Token, error) { + return &oauth2.Token{ + AccessToken: t.AccessToken, + }, nil +} diff --git a/builder/digitalocean/wait.go b/builder/digitalocean/wait.go index e5b1dee90..3d299d433 100644 --- a/builder/digitalocean/wait.go +++ b/builder/digitalocean/wait.go @@ -4,11 +4,15 @@ import ( "fmt" "log" "time" + + "github.com/digitalocean/godo" ) // waitForState simply blocks until the droplet is in // a state we expect, while eventually timing out. -func waitForDropletState(desiredState string, dropletId uint, client DigitalOceanClient, timeout time.Duration) error { +func waitForDropletState( + desiredState string, dropletId int, + client *godo.Client, timeout time.Duration) error { done := make(chan struct{}) defer close(done) @@ -19,13 +23,13 @@ func waitForDropletState(desiredState string, dropletId uint, client DigitalOcea attempts += 1 log.Printf("Checking droplet status... (attempt: %d)", attempts) - _, status, err := client.DropletStatus(dropletId) + droplet, _, err := client.Droplets.Get(dropletId) if err != nil { result <- err return } - if status == desiredState { + if droplet.Status == desiredState { result <- nil return } From 486c7e4ae61237a82fb0ec1be57d9adde197deb8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:07:24 -0700 Subject: [PATCH 072/208] builder/digitalocean: remove unused things --- builder/digitalocean/config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 0be49f08e..5defe89db 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -12,7 +12,6 @@ import ( "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" - //"github.com/digitalocean/godo" ) type Config struct { @@ -20,8 +19,6 @@ type Config struct { APIToken string `mapstructure:"api_token"` - // OLD STUFF - Region string `mapstructure:"region"` Size string `mapstructure:"size"` Image string `mapstructure:"image"` From a691a1521c5f722d7e3449fd84fcf40fa3183b6a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:16:40 -0700 Subject: [PATCH 073/208] website: update do docs --- .../docs/builders/digitalocean.html.markdown | 51 ++----------------- 1 file changed, 4 insertions(+), 47 deletions(-) diff --git a/website/source/docs/builders/digitalocean.html.markdown b/website/source/docs/builders/digitalocean.html.markdown index 5ffe1c668..28254b19c 100644 --- a/website/source/docs/builders/digitalocean.html.markdown +++ b/website/source/docs/builders/digitalocean.html.markdown @@ -24,31 +24,13 @@ There are many configuration options available for the builder. They are segmented below into two categories: required and optional parameters. Within each category, the available configuration keys are alphabetized. -### Required v1 api: +### Required: -* `api_key` (string) - The API key to use to access your account. You can - retrieve this on the "API" page visible after logging into your account - on DigitalOcean. - If not specified, Packer will use the environment variable - `DIGITALOCEAN_API_KEY`, if set. - -* `client_id` (string) - The client ID to use to access your account. You can - find this on the "API" page visible after logging into your account on - DigitalOcean. - If not specified, Packer will use the environment variable - `DIGITALOCEAN_CLIENT_ID`, if set. - -### Required v2 api: - -* `api_token` (string) - The client TOKEN to use to access your account. If it - specified, then use v2 api (current), if not then used old (v1) deprecated api. - Also it can be specified via environment variable `DIGITALOCEAN_API_TOKEN`, if set. +* `api_token` (string) - The client TOKEN to use to access your account. + It can also be specified via environment variable `DIGITALOCEAN_API_TOKEN`, if set. ### Optional: -* `api_url` (string) - API endpoint, by default use https://api.digitalocean.com - Also it can be specified via environment variable `DIGITALOCEAN_API_URL`, if set. - * `droplet_name` (string) - The name assigned to the droplet. DigitalOcean sets the hostname of the machine to this value. @@ -57,10 +39,6 @@ each category, the available configuration keys are alphabetized. defaults to 'ubuntu-12-04-x64' which is the slug for "Ubuntu 12.04.4 x64". See https://developers.digitalocean.com/documentation/v2/#list-all-images for details on how to get a list of the the accepted image names/slugs. -* `image_id` (integer) - The ID of the base image to use. This is the image that - will be used to launch a new droplet and provision it. - This setting is deprecated. Use `image` instead. - * `private_networking` (boolean) - Set to `true` to enable private networking for the droplet being created. This defaults to `false`, or not enabled. @@ -69,17 +47,10 @@ each category, the available configuration keys are alphabetized. This defaults to "nyc3", which is the slug for "New York 3". See https://developers.digitalocean.com/documentation/v2/#list-all-regions for the accepted region names/slugs. -* `region_id` (integer) - The ID of the region to launch the droplet in. Consequently, - this is the region where the snapshot will be available. - This setting is deprecated. Use `region` instead. - * `size` (string) - The name (or slug) of the droplet size to use. This defaults to "512mb", which is the slug for "512MB". See https://developers.digitalocean.com/documentation/v2/#list-all-sizes for the accepted size names/slugs. -* `size_id` (integer) - The ID of the droplet size to use. - This setting is deprecated. Use `size` instead. - * `snapshot_name` (string) - The name of the resulting snapshot that will appear in your account. This must be unique. To help make this unique, use a function like `timestamp` (see @@ -107,20 +78,6 @@ own access tokens: ```javascript { "type": "digitalocean", - "client_id": "YOUR CLIENT ID", - "api_key": "YOUR API KEY" + "api_token": "YOUR API KEY" } ``` - -## Finding Image, Region, and Size IDs - -Unfortunately, finding a list of available values for `image_id`, `region_id`, -and `size_id` is not easy at the moment. Basically, it has to be done through -the [DigitalOcean API](https://www.digitalocean.com/api_access) using the -`/images`, `/regions`, and `/sizes` endpoints. You can use `curl` for this -or request it in your browser. - -If you're comfortable installing RubyGems, [Tugboat](https://github.com/pearkes/tugboat) -is a fantastic DigitalOcean command-line client that has commands to -find the available images, regions, and sizes. For example, to see all the -global images, you can run `tugboat images --global`. From 9a393a560172e286cf9b9319a6689822b47c2783 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:18:38 -0700 Subject: [PATCH 074/208] builder/digitalocean: only list user images --- builder/digitalocean/step_snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/digitalocean/step_snapshot.go b/builder/digitalocean/step_snapshot.go index 7ff384924..cfda7af20 100644 --- a/builder/digitalocean/step_snapshot.go +++ b/builder/digitalocean/step_snapshot.go @@ -37,7 +37,7 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { } log.Printf("Looking up snapshot ID for snapshot: %s", c.SnapshotName) - images, _, err := client.Images.List(nil) + images, _, err := client.Images.ListUser(&godo.ListOptions{PerPage: 200}) if err != nil { err := fmt.Errorf("Error looking up snapshot ID: %s", err) state.Put("error", err) From 2056fda4d3a91e670f628f554b0fcb0e3725bddd Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Wed, 10 Jun 2015 16:19:36 -0500 Subject: [PATCH 075/208] builder/amazon: Allow spaces in AMI names --- builder/amazon/common/template_funcs.go | 2 +- builder/amazon/common/template_funcs_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/template_funcs.go b/builder/amazon/common/template_funcs.go index 45f68af92..30d49fdb4 100644 --- a/builder/amazon/common/template_funcs.go +++ b/builder/amazon/common/template_funcs.go @@ -20,7 +20,7 @@ func isalphanumeric(b byte) bool { // Clean up AMI name by replacing invalid characters with "-" func templateCleanAMIName(s string) string { - allowed := []byte{'(', ')', ',', '/', '-', '_'} + allowed := []byte{'(', ')', ',', '/', '-', '_', ' '} b := []byte(s) newb := make([]byte, len(b)) for i, c := range b { diff --git a/builder/amazon/common/template_funcs_test.go b/builder/amazon/common/template_funcs_test.go index 0e8c568ef..e4126bf61 100644 --- a/builder/amazon/common/template_funcs_test.go +++ b/builder/amazon/common/template_funcs_test.go @@ -5,8 +5,8 @@ import ( ) func TestAMITemplatePrepare_clean(t *testing.T) { - origName := "AMZamz09(),/-_:&^$%" - expected := "AMZamz09(),/-_-----" + origName := "AMZamz09(),/-_:&^ $%" + expected := "AMZamz09(),/-_--- --" name := templateCleanAMIName(origName) From 311c9eb5c2b594312ba6e69ff47b71419657349c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:28:05 -0700 Subject: [PATCH 076/208] builder/digitalocean: fix unit tests --- builder/digitalocean/builder_test.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/builder/digitalocean/builder_test.go b/builder/digitalocean/builder_test.go index 8985aae2f..878431691 100644 --- a/builder/digitalocean/builder_test.go +++ b/builder/digitalocean/builder_test.go @@ -1,22 +1,15 @@ package digitalocean import ( - "github.com/mitchellh/packer/packer" - "os" "strconv" "testing" -) -func init() { - // Clear out the credential env vars - os.Setenv("DIGITALOCEAN_API_KEY", "") - os.Setenv("DIGITALOCEAN_CLIENT_ID", "") -} + "github.com/mitchellh/packer/packer" +) func testConfig() map[string]interface{} { return map[string]interface{}{ - "client_id": "foo", - "api_key": "bar", + "api_token": "bar", } } @@ -78,7 +71,6 @@ func TestBuilderPrepare_Region(t *testing.T) { expected := "sfo1" // Test set - config["region_id"] = 0 config["region"] = expected b = Builder{} warnings, err = b.Prepare(config) @@ -114,7 +106,6 @@ func TestBuilderPrepare_Size(t *testing.T) { expected := "1024mb" // Test set - config["size_id"] = 0 config["size"] = expected b = Builder{} warnings, err = b.Prepare(config) @@ -150,7 +141,6 @@ func TestBuilderPrepare_Image(t *testing.T) { expected := "ubuntu-14-04-x64" // Test set - config["image_id"] = 0 config["image"] = expected b = Builder{} warnings, err = b.Prepare(config) From be8443d50813919a3d9fc0e39135d0d570a6c968 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:49:27 -0700 Subject: [PATCH 077/208] update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a33dbc92..cc2ea5217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## 0.8.0 (unreleased) +BACKWARDS INCOMPATIBILITIES: + + * The DigitalOcean builder no longer supports the v1 API which has been + deprecated for some time. Most configurations should continue to + work as long as you use the `api_token` field for auth. + FEATURES: * **New config function: `template_dir`**: The directory to the template From 5da56d2aa6622b5096cdb1c5efbeba0408c63aa0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:54:51 -0700 Subject: [PATCH 078/208] builder/digitalocean: image, region, etc. required --- builder/digitalocean/builder.go | 12 ----------- builder/digitalocean/builder_acc_test.go | 5 ++++- builder/digitalocean/builder_test.go | 24 ++++++--------------- builder/digitalocean/config.go | 27 +++++++++++++----------- 4 files changed, 25 insertions(+), 43 deletions(-) diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index 3ba7074a2..996eb2e09 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -14,18 +14,6 @@ import ( "golang.org/x/oauth2" ) -// see https://api.digitalocean.com/images/?client_id=[client_id]&api_key=[api_key] -// name="Ubuntu 12.04.4 x64", id=6374128, -const DefaultImage = "ubuntu-12-04-x64" - -// see https://api.digitalocean.com/regions/?client_id=[client_id]&api_key=[api_key] -// name="New York 3", id=8 -const DefaultRegion = "nyc3" - -// see https://api.digitalocean.com/sizes/?client_id=[client_id]&api_key=[api_key] -// name="512MB", id=66 (the smallest droplet size) -const DefaultSize = "512mb" - // The unique id for the builder const BuilderId = "pearkes.digitalocean" diff --git a/builder/digitalocean/builder_acc_test.go b/builder/digitalocean/builder_acc_test.go index 20e56b924..f9df863cb 100644 --- a/builder/digitalocean/builder_acc_test.go +++ b/builder/digitalocean/builder_acc_test.go @@ -24,7 +24,10 @@ func testAccPreCheck(t *testing.T) { const testBuilderAccBasic = ` { "builders": [{ - "type": "test" + "type": "test", + "region": "nyc2", + "size": "512mb", + "image": "ubuntu-12-04-x64" }] } ` diff --git a/builder/digitalocean/builder_test.go b/builder/digitalocean/builder_test.go index 878431691..3d2378f2d 100644 --- a/builder/digitalocean/builder_test.go +++ b/builder/digitalocean/builder_test.go @@ -60,12 +60,8 @@ func TestBuilderPrepare_Region(t *testing.T) { if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.Region != DefaultRegion { - t.Errorf("found %s, expected %s", b.config.Region, DefaultRegion) + if err == nil { + t.Fatalf("should error") } expected := "sfo1" @@ -95,12 +91,8 @@ func TestBuilderPrepare_Size(t *testing.T) { if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.Size != DefaultSize { - t.Errorf("found %s, expected %s", b.config.Size, DefaultSize) + if err == nil { + t.Fatalf("should error") } expected := "1024mb" @@ -130,12 +122,8 @@ func TestBuilderPrepare_Image(t *testing.T) { if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.Image != DefaultImage { - t.Errorf("found %s, expected %s", b.config.Image, DefaultImage) + if err == nil { + t.Fatal("should error") } expected := "ubuntu-14-04-x64" diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 5defe89db..057138633 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -63,18 +63,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { c.APIToken = os.Getenv("DIGITALOCEAN_API_TOKEN") } - if c.Region == "" { - c.Region = DefaultRegion - } - - if c.Size == "" { - c.Size = DefaultSize - } - - if c.Image == "" { - c.Image = DefaultImage - } - if c.SnapshotName == "" { // Default to packer-{{ unix timestamp (utc) }} c.SnapshotName = "packer-{{timestamp}}" @@ -114,6 +102,21 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { errs, errors.New("api_token for auth must be specified")) } + if c.Region == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("region is required")) + } + + if c.Size == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("size is required")) + } + + if c.Image == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("image is required")) + } + sshTimeout, err := time.ParseDuration(c.RawSSHTimeout) if err != nil { errs = packer.MultiErrorAppend( From 910b16104ed5ed5d654c56f9e0749873fedfdd84 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:56:03 -0700 Subject: [PATCH 079/208] website: update required options for DO --- CHANGELOG.md | 1 + .../docs/builders/digitalocean.html.markdown | 30 +++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc2ea5217..fd57fb8c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ BACKWARDS INCOMPATIBILITIES: * The DigitalOcean builder no longer supports the v1 API which has been deprecated for some time. Most configurations should continue to work as long as you use the `api_token` field for auth. + * builder/digitalocean: `image`, `region`, and `size` are now required. FEATURES: diff --git a/website/source/docs/builders/digitalocean.html.markdown b/website/source/docs/builders/digitalocean.html.markdown index 28254b19c..34d11bedb 100644 --- a/website/source/docs/builders/digitalocean.html.markdown +++ b/website/source/docs/builders/digitalocean.html.markdown @@ -29,28 +29,25 @@ each category, the available configuration keys are alphabetized. * `api_token` (string) - The client TOKEN to use to access your account. It can also be specified via environment variable `DIGITALOCEAN_API_TOKEN`, if set. +* `image` (string) - The name (or slug) of the base image to use. This is the + image that will be used to launch a new droplet and provision it. + See https://developers.digitalocean.com/documentation/v2/#list-all-images for details on how to get a list of the the accepted image names/slugs. + +* `region` (string) - The name (or slug) of the region to launch the droplet in. + Consequently, this is the region where the snapshot will be available. + See https://developers.digitalocean.com/documentation/v2/#list-all-regions for the accepted region names/slugs. + +* `size` (string) - The name (or slug) of the droplet size to use. + See https://developers.digitalocean.com/documentation/v2/#list-all-sizes for the accepted size names/slugs. + ### Optional: * `droplet_name` (string) - The name assigned to the droplet. DigitalOcean sets the hostname of the machine to this value. -* `image` (string) - The name (or slug) of the base image to use. This is the - image that will be used to launch a new droplet and provision it. This - defaults to 'ubuntu-12-04-x64' which is the slug for "Ubuntu 12.04.4 x64". - See https://developers.digitalocean.com/documentation/v2/#list-all-images for details on how to get a list of the the accepted image names/slugs. - * `private_networking` (boolean) - Set to `true` to enable private networking for the droplet being created. This defaults to `false`, or not enabled. -* `region` (string) - The name (or slug) of the region to launch the droplet in. - Consequently, this is the region where the snapshot will be available. - This defaults to "nyc3", which is the slug for "New York 3". - See https://developers.digitalocean.com/documentation/v2/#list-all-regions for the accepted region names/slugs. - -* `size` (string) - The name (or slug) of the droplet size to use. - This defaults to "512mb", which is the slug for "512MB". - See https://developers.digitalocean.com/documentation/v2/#list-all-sizes for the accepted size names/slugs. - * `snapshot_name` (string) - The name of the resulting snapshot that will appear in your account. This must be unique. To help make this unique, use a function like `timestamp` (see @@ -78,6 +75,9 @@ own access tokens: ```javascript { "type": "digitalocean", - "api_token": "YOUR API KEY" + "api_token": "YOUR API KEY", + "image": "ubuntu-12-04-x64", + "region": "nyc2", + "size": "512mb" } ``` From 6b29c2d26a80289c720300cc0381613148c938ad Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:58:27 -0700 Subject: [PATCH 080/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd57fb8c3..357afa298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] + * builder/digitalocean: Save SSH key to pwd if debug mode is on. [GH-1829] * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear From c9c9e2871c2cfc6b2e3b17c5c24eb58cc5a6f45c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:58:50 -0700 Subject: [PATCH 081/208] builder/digitalocean: fix build --- builder/digitalocean/builder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index c269da6e1..97569d0fe 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -4,6 +4,7 @@ package digitalocean import ( + "fmt" "log" "time" @@ -47,7 +48,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ &stepCreateSSHKey{ - Debug: b.config.PackerDebug, + Debug: b.config.PackerDebug, DebugKeyPath: fmt.Sprintf("do_%s.pem", b.config.PackerBuildName), }, new(stepCreateDroplet), From 5cfd26a0d336494f2273076b6f1cf2341bcec551 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 19:29:48 -0700 Subject: [PATCH 082/208] builder/digitalocean: user data support [GH-2113] --- builder/digitalocean/config.go | 1 + builder/digitalocean/step_create_droplet.go | 1 + website/source/docs/builders/digitalocean.html.markdown | 2 ++ 3 files changed, 4 insertions(+) diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 057138633..5621d64d7 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -26,6 +26,7 @@ type Config struct { PrivateNetworking bool `mapstructure:"private_networking"` SnapshotName string `mapstructure:"snapshot_name"` DropletName string `mapstructure:"droplet_name"` + UserData string `mapstructure:"user_data"` SSHUsername string `mapstructure:"ssh_username"` SSHPort uint `mapstructure:"ssh_port"` diff --git a/builder/digitalocean/step_create_droplet.go b/builder/digitalocean/step_create_droplet.go index 40ac8f0e9..aafd53622 100644 --- a/builder/digitalocean/step_create_droplet.go +++ b/builder/digitalocean/step_create_droplet.go @@ -31,6 +31,7 @@ func (s *stepCreateDroplet) Run(state multistep.StateBag) multistep.StepAction { godo.DropletCreateSSHKey{ID: int(sshKeyId)}, }, PrivateNetworking: c.PrivateNetworking, + UserData: c.UserData, }) if err != nil { err := fmt.Errorf("Error creating droplet: %s", err) diff --git a/website/source/docs/builders/digitalocean.html.markdown b/website/source/docs/builders/digitalocean.html.markdown index 34d11bedb..829424e3d 100644 --- a/website/source/docs/builders/digitalocean.html.markdown +++ b/website/source/docs/builders/digitalocean.html.markdown @@ -67,6 +67,8 @@ each category, the available configuration keys are alphabetized. for a droplet to enter a desired state (such as "active") before timing out. The default state timeout is "6m". +* `user_data` (string) - User data to launch with the Droplet. + ## Basic Example Here is a basic example. It is completely valid as soon as you enter your From 0e0cd28071942199858f2e9384368473c3c8d54a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 19:31:48 -0700 Subject: [PATCH 083/208] builder/digitalocean: fix failing unit tests --- builder/digitalocean/builder_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/builder/digitalocean/builder_test.go b/builder/digitalocean/builder_test.go index 3d2378f2d..22c9e3b50 100644 --- a/builder/digitalocean/builder_test.go +++ b/builder/digitalocean/builder_test.go @@ -10,6 +10,9 @@ import ( func testConfig() map[string]interface{} { return map[string]interface{}{ "api_token": "bar", + "region": "nyc2", + "size": "512mb", + "image": "foo", } } @@ -56,6 +59,7 @@ func TestBuilderPrepare_Region(t *testing.T) { config := testConfig() // Test default + delete(config, "region") warnings, err := b.Prepare(config) if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) @@ -87,6 +91,7 @@ func TestBuilderPrepare_Size(t *testing.T) { config := testConfig() // Test default + delete(config, "size") warnings, err := b.Prepare(config) if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) @@ -118,6 +123,7 @@ func TestBuilderPrepare_Image(t *testing.T) { config := testConfig() // Test default + delete(config, "image") warnings, err := b.Prepare(config) if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) From dcf140f99facb21ef1e177866c9cb41eb4349093 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 19:53:07 -0700 Subject: [PATCH 084/208] builder/digitalocean: more robust wait for pending --- builder/digitalocean/config.go | 7 +++- builder/digitalocean/step_power_off.go | 10 ++++++ builder/digitalocean/step_shutdown.go | 14 +++++++- builder/digitalocean/step_snapshot.go | 13 +++++++ builder/digitalocean/wait.go | 49 ++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 2 deletions(-) diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 057138633..dd1460583 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -64,8 +64,13 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { } if c.SnapshotName == "" { + def, err := interpolate.Render("packer-{{timestamp}}", nil) + if err != nil { + panic(err) + } + // Default to packer-{{ unix timestamp (utc) }} - c.SnapshotName = "packer-{{timestamp}}" + c.SnapshotName = def } if c.DropletName == "" { diff --git a/builder/digitalocean/step_power_off.go b/builder/digitalocean/step_power_off.go index 3d547e8c2..94891e227 100644 --- a/builder/digitalocean/step_power_off.go +++ b/builder/digitalocean/step_power_off.go @@ -3,6 +3,7 @@ package digitalocean import ( "fmt" "log" + "time" "github.com/digitalocean/godo" "github.com/mitchellh/multistep" @@ -48,6 +49,15 @@ func (s *stepPowerOff) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } + // Wait for the droplet to become unlocked for future steps + if err := waitForDropletUnlocked(client, dropletId, 2*time.Minute); err != nil { + // If we get an error the first time, actually report it + err := fmt.Errorf("Error powering off droplet: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + return multistep.ActionContinue } diff --git a/builder/digitalocean/step_shutdown.go b/builder/digitalocean/step_shutdown.go index 602f3e690..da04aee33 100644 --- a/builder/digitalocean/step_shutdown.go +++ b/builder/digitalocean/step_shutdown.go @@ -65,7 +65,19 @@ func (s *stepShutdown) Run(state multistep.StateBag) multistep.StepAction { err = waitForDropletState("off", dropletId, client, 2*time.Minute) if err != nil { - log.Printf("Error waiting for graceful off: %s", err) + // If we get an error the first time, actually report it + err := fmt.Errorf("Error shutting down droplet: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if err := waitForDropletUnlocked(client, dropletId, 2*time.Minute); err != nil { + // If we get an error the first time, actually report it + err := fmt.Errorf("Error shutting down droplet: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt } return multistep.ActionContinue diff --git a/builder/digitalocean/step_snapshot.go b/builder/digitalocean/step_snapshot.go index cfda7af20..f6902b8c5 100644 --- a/builder/digitalocean/step_snapshot.go +++ b/builder/digitalocean/step_snapshot.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "log" + "time" "github.com/digitalocean/godo" "github.com/mitchellh/multistep" @@ -27,6 +28,18 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } + // Wait for the droplet to become unlocked first. For snapshots + // this can end up taking quite a long time, so we hardcode this to + // 10 minutes. + if err := waitForDropletUnlocked(client, dropletId, 10*time.Minute); err != nil { + // If we get an error the first time, actually report it + err := fmt.Errorf("Error shutting down droplet: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // With the pending state over, verify that we're in the active state ui.Say("Waiting for snapshot to complete...") err = waitForDropletState("active", dropletId, client, c.stateTimeout) if err != nil { diff --git a/builder/digitalocean/wait.go b/builder/digitalocean/wait.go index 3d299d433..a41bbb3ed 100644 --- a/builder/digitalocean/wait.go +++ b/builder/digitalocean/wait.go @@ -8,6 +8,55 @@ import ( "github.com/digitalocean/godo" ) +// waitForDropletUnlocked waits for the Droplet to be unlocked to +// avoid "pending" errors when making state changes. +func waitForDropletUnlocked( + client *godo.Client, dropletId int, timeout time.Duration) error { + done := make(chan struct{}) + defer close(done) + + result := make(chan error, 1) + go func() { + attempts := 0 + for { + attempts += 1 + + log.Printf("[DEBUG] Checking droplet lock state... (attempt: %d)", attempts) + droplet, _, err := client.Droplets.Get(dropletId) + if err != nil { + result <- err + return + } + + if !droplet.Locked { + result <- nil + return + } + + // Wait 3 seconds in between + time.Sleep(3 * time.Second) + + // Verify we shouldn't exit + select { + case <-done: + // We finished, so just exit the goroutine + return + default: + // Keep going + } + } + }() + + log.Printf("[DEBUG] Waiting for up to %d seconds for droplet to unlock", timeout/time.Second) + select { + case err := <-result: + return err + case <-time.After(timeout): + return fmt.Errorf( + "Timeout while waiting to for droplet to unlock") + } +} + // waitForState simply blocks until the droplet is in // a state we expect, while eventually timing out. func waitForDropletState( From c9d308c5d7f4a26c620fc5be10df045c018b7505 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 19:53:52 -0700 Subject: [PATCH 085/208] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 357afa298..b79d6a4bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,8 @@ BUG FIXES: * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] * builder/amazon/instance: Use `--region` flag for bundle upload command. [GH-1931] + * builder/digitalocean: Wait for droplet to unlock before changing state, + should lower the "pending event" errors. * builder/digitalocean: Ignore invalid fields from the ever-changing v2 API * builder/digitalocean: Private images can be used as a source [GH-1792] * builder/docker: Fixed hang on prompt while copying script From 03032c26fbef7d2a4ba79fc06d437b10f0a4eb18 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 20:23:39 -0700 Subject: [PATCH 086/208] ignore non-exe plugins on Windows [GH-2173] --- config.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config.go b/config.go index 745922b1e..20e8929c3 100644 --- a/config.go +++ b/config.go @@ -6,6 +6,7 @@ import ( "log" "os/exec" "path/filepath" + "runtime" "strings" "github.com/mitchellh/osext" @@ -172,6 +173,15 @@ func (c *config) discoverSingle(glob string, m *map[string]string) error { for _, match := range matches { file := filepath.Base(match) + // One Windows, ignore any plugins that don't end in .exe. + // We could do a full PATHEXT parse, but this is probably good enough. + if runtime.GOOS == "windows" && filepath.Ext(file) != "exe" { + log.Printf( + "[DEBUG] Ignoring plugin match %s, no exe extension", + match) + continue + } + // If the filename has a ".", trim up to there if idx := strings.Index(file, "."); idx >= 0 { file = file[:idx] From 9514be0df0613f4c7c6dd8e0b25f62f026ace9d3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 20:24:34 -0700 Subject: [PATCH 087/208] lowercase the extension of plugins just in case --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 20e8929c3..2ebc66422 100644 --- a/config.go +++ b/config.go @@ -175,7 +175,7 @@ func (c *config) discoverSingle(glob string, m *map[string]string) error { // One Windows, ignore any plugins that don't end in .exe. // We could do a full PATHEXT parse, but this is probably good enough. - if runtime.GOOS == "windows" && filepath.Ext(file) != "exe" { + if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != "exe" { log.Printf( "[DEBUG] Ignoring plugin match %s, no exe extension", match) From 9c1e6bc478c150920a2a6433fddbf6808b0f1a12 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 20:30:00 -0700 Subject: [PATCH 088/208] website: clarify shutdown command [GH-2011] --- website/source/docs/builders/virtualbox-iso.html.markdown | 8 +++++--- website/source/docs/builders/virtualbox-ovf.html.markdown | 8 +++++--- website/source/docs/builders/vmware-vmx.html.markdown | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/website/source/docs/builders/virtualbox-iso.html.markdown b/website/source/docs/builders/virtualbox-iso.html.markdown index 03a3fb23a..07204a33c 100644 --- a/website/source/docs/builders/virtualbox-iso.html.markdown +++ b/website/source/docs/builders/virtualbox-iso.html.markdown @@ -179,9 +179,11 @@ each category, the available options are alphabetized and described. By default this is "output-BUILDNAME" where "BUILDNAME" is the name of the build. -* `shutdown_command` (string) - The command to use to gracefully shut down - the machine once all the provisioning is done. By default this is an empty - string, which tells Packer to just forcefully shut down the machine. +* `shutdown_command` (string) - The command to use to gracefully shut down the machine once all + the provisioning is done. By default this is an empty string, which tells Packer to just + forcefully shut down the machine unless a shutdown command takes place inside script so this may + safely be omitted. If one or more scripts require a reboot it is suggested to leave this blank + since reboots may fail and specify the final shutdown command in your last script. * `shutdown_timeout` (string) - The amount of time to wait after executing the `shutdown_command` for the virtual machine to actually shut down. diff --git a/website/source/docs/builders/virtualbox-ovf.html.markdown b/website/source/docs/builders/virtualbox-ovf.html.markdown index 88a92b674..9635d0e60 100644 --- a/website/source/docs/builders/virtualbox-ovf.html.markdown +++ b/website/source/docs/builders/virtualbox-ovf.html.markdown @@ -154,9 +154,11 @@ each category, the available options are alphabetized and described. By default this is "output-BUILDNAME" where "BUILDNAME" is the name of the build. -* `shutdown_command` (string) - The command to use to gracefully shut down - the machine once all the provisioning is done. By default this is an empty - string, which tells Packer to just forcefully shut down the machine. +* `shutdown_command` (string) - The command to use to gracefully shut down the machine once all + the provisioning is done. By default this is an empty string, which tells Packer to just + forcefully shut down the machine unless a shutdown command takes place inside script so this may + safely be omitted. If one or more scripts require a reboot it is suggested to leave this blank + since reboots may fail and specify the final shutdown command in your last script. * `shutdown_timeout` (string) - The amount of time to wait after executing the `shutdown_command` for the virtual machine to actually shut down. diff --git a/website/source/docs/builders/vmware-vmx.html.markdown b/website/source/docs/builders/vmware-vmx.html.markdown index 399bab8a9..bbdd8925c 100644 --- a/website/source/docs/builders/vmware-vmx.html.markdown +++ b/website/source/docs/builders/vmware-vmx.html.markdown @@ -109,9 +109,11 @@ each category, the available options are alphabetized and described. By default this is "output-BUILDNAME" where "BUILDNAME" is the name of the build. -* `shutdown_command` (string) - The command to use to gracefully shut down - the machine once all the provisioning is done. By default this is an empty - string, which tells Packer to just forcefully shut down the machine. +* `shutdown_command` (string) - The command to use to gracefully shut down the machine once all + the provisioning is done. By default this is an empty string, which tells Packer to just + forcefully shut down the machine unless a shutdown command takes place inside script so this may + safely be omitted. If one or more scripts require a reboot it is suggested to leave this blank + since reboots may fail and specify the final shutdown command in your last script. * `shutdown_timeout` (string) - The amount of time to wait after executing the `shutdown_command` for the virtual machine to actually shut down. From 9c1e461402bb88c32e08c8019e7f9816e36eb2a0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 20:33:00 -0700 Subject: [PATCH 089/208] website: document chef_environment for chef-solo --- website/source/docs/provisioners/chef-solo.html.markdown | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/source/docs/provisioners/chef-solo.html.markdown b/website/source/docs/provisioners/chef-solo.html.markdown index 178ba0c62..3a76c5514 100644 --- a/website/source/docs/provisioners/chef-solo.html.markdown +++ b/website/source/docs/provisioners/chef-solo.html.markdown @@ -34,6 +34,9 @@ The example below is fully functional and expects cookbooks in the The reference of available configuration options is listed below. No configuration is actually required, but at least `run_list` is recommended. +* `chef_environment` (string) - The name of the `chef_environment` sent to the + Chef server. By default this is empty and will not use an environment + * `config_template` (string) - Path to a template that will be used for the Chef configuration file. By default Packer only sets configuration it needs to match the settings set in the provisioner configuration. If From 952077ccb0b1cfe189a43b8001d6dac3ea29b64c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 21:02:22 -0700 Subject: [PATCH 090/208] provisioner/shell: set -e for inline [GH-2069] --- provisioner/shell/provisioner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index 48904710d..8b6ceb705 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -184,6 +184,7 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { // Write our contents to it writer := bufio.NewWriter(tf) writer.WriteString(fmt.Sprintf("#!%s\n", p.config.InlineShebang)) + writer.WriteString("set -e\n") for _, command := range p.config.Inline { if _, err := writer.WriteString(command + "\n"); err != nil { return fmt.Errorf("Error preparing shell script: %s", err) From a995df352e7a6967c7e40d1a165b89af98a4abd3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 21:18:46 -0700 Subject: [PATCH 091/208] provisioner/shell: uploaded script should be 0755 [GH-1708] --- provisioner/shell/provisioner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index 48904710d..f07c91936 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -247,11 +247,11 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { } cmd = &packer.RemoteCmd{ - Command: fmt.Sprintf("chmod 0777 %s", p.config.RemotePath), + Command: fmt.Sprintf("chmod 0755 %s", p.config.RemotePath), } if err := comm.Start(cmd); err != nil { return fmt.Errorf( - "Error chmodding script file to 0777 in remote "+ + "Error chmodding script file to 0755 in remote "+ "machine: %s", err) } cmd.Wait() From e8f846e47e93669a9eee2d1daaa4d649a71667ab Mon Sep 17 00:00:00 2001 From: Clint Date: Thu, 11 Jun 2015 08:50:48 -0500 Subject: [PATCH 092/208] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b79d6a4bb..abb0d46d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ IMPROVEMENTS: BUG FIXES: * core: Fix potential panic for post-processor plugin exits [GH-2098] + * builder/amazon: Allow spaces in AMI names when using `clean_ami_name` [GH-2182] * builder/amazon: Remove deprecated ec2-upload-bundle paramger [GH-1931] * builder/amazon: Retry finding created instance for eventual consistency. [GH-2129] From 1fbf8b7f32beaf98a1a85f396a0e98eecfd12ca4 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Thu, 11 Jun 2015 10:43:27 -0500 Subject: [PATCH 093/208] update create_tags for new sdk --- builder/amazon/common/step_create_tags.go | 51 ++++++++++++++++------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/builder/amazon/common/step_create_tags.go b/builder/amazon/common/step_create_tags.go index 3a31202ff..7c89e5a59 100644 --- a/builder/amazon/common/step_create_tags.go +++ b/builder/amazon/common/step_create_tags.go @@ -3,6 +3,7 @@ package common import ( "fmt" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" @@ -19,41 +20,61 @@ func (s *StepCreateTags) Run(state multistep.StateBag) multistep.StepAction { if len(s.Tags) > 0 { for region, ami := range amis { - ui.Say(fmt.Sprintf("Preparing tags for AMI (%s) and related snapshots", ami)) + ui.Say(fmt.Sprintf("Adding tags to AMI (%s)...", ami)) + + var ec2Tags []*ec2.Tag + for key, value := range s.Tags { + ui.Message(fmt.Sprintf("Adding tag: \"%s\": \"%s\"", key, value)) + ec2Tags = append(ec2Tags, &ec2.Tag{ + Key: aws.String(key), + Value: aws.String(value), + }) + } // Declare list of resources to tag - resourceIds := []string{ami} + resourceIds := []*string{&ami} // Retrieve image list for given AMI - imageResp, err := ec2conn.Images([]string{ami}, ec2.NewFilter()) + imageResp, err := ec2conn.DescribeImages(&ec2.DescribeImagesInput{ + ImageIDs: resourceIds, + }) + if err != nil { err := fmt.Errorf("Error retrieving details for AMI (%s): %s", ami, err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - image := &imageResp.Images[0] + + if len(imageResp.Images) == 0 { + err := fmt.Errorf("Error retrieving details for AMI (%s), no images found", ami) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + image := imageResp.Images[0] // Add only those with a Snapshot ID, i.e. not Ephemeral - for _, device := range image.BlockDevices { - if device.SnapshotId != "" { - ui.Say(fmt.Sprintf("Tagging snapshot: %s", device.SnapshotId)) - resourceIds = append(resourceIds, device.SnapshotId) + for _, device := range image.BlockDeviceMappings { + if device.EBS != nil && device.EBS.SnapshotID != nil { + ui.Say(fmt.Sprintf("Tagging snapshot: %s", *device.EBS.SnapshotID)) + resourceIds = append(resourceIds, device.EBS.SnapshotID) } } - var ec2Tags []*ec2.Tag - for key, value := range s.Tags { - ui.Message(fmt.Sprintf("Adding tag: \"%s\": \"%s\"", key, value)) - ec2Tags = append(ec2Tags, &ec2.Tag{Key: &key, Value: &value}) - } + regionconn := ec2.New(&aws.Config{ + Credentials: ec2conn.Config.Credentials, + Region: region, + }) - _, err := regionconn.CreateTags(&ec2.CreateTagsInput{ + _, err = regionconn.CreateTags(&ec2.CreateTagsInput{ Resources: resourceIds, Tags: ec2Tags, }) + if err != nil { - err := fmt.Errorf("Error adding tags to AMI (%s): %s", ami, err) + err := fmt.Errorf("Error adding tags to Resources (%#v): %s", resourceIds, err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt From 16f8866728d81acf43925427d2b89db581a55fb1 Mon Sep 17 00:00:00 2001 From: Clint Date: Thu, 11 Jun 2015 10:57:07 -0500 Subject: [PATCH 094/208] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index abb0d46d0..4705c3523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ BUG FIXES: * core: Fix potential panic for post-processor plugin exits [GH-2098] * builder/amazon: Allow spaces in AMI names when using `clean_ami_name` [GH-2182] * builder/amazon: Remove deprecated ec2-upload-bundle paramger [GH-1931] + * builder/amazon: Use IAM Profile to upload bundle if provided [GH-1985] * builder/amazon: Retry finding created instance for eventual consistency. [GH-2129] * builder/amazon: If no AZ is specified, use AZ chosen automatically by From 724d591ba4f29817b4f97085b463ba728936bfe1 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Thu, 11 Jun 2015 14:02:00 -0500 Subject: [PATCH 095/208] documentation: subnet_id is required for non-default VPC --- website/Gemfile.lock | 9 ++++++--- website/source/docs/builders/amazon-ebs.html.markdown | 3 ++- .../source/docs/builders/amazon-instance.html.markdown | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/website/Gemfile.lock b/website/Gemfile.lock index 7366999a6..b49383f77 100644 --- a/website/Gemfile.lock +++ b/website/Gemfile.lock @@ -79,18 +79,18 @@ GEM celluloid (~> 0.16.0) rb-fsevent (>= 0.9.3) rb-inotify (>= 0.9) - middleman (3.3.13) + middleman (3.3.12) coffee-script (~> 2.2) compass (>= 1.0.0, < 2.0.0) compass-import-once (= 1.0.5) execjs (~> 2.0) haml (>= 4.0.5) kramdown (~> 1.2) - middleman-core (= 3.3.13) + middleman-core (= 3.3.12) middleman-sprockets (>= 3.1.2) sass (>= 3.4.0, < 4.0) uglifier (~> 2.5) - middleman-core (3.3.13) + middleman-core (3.3.12) activesupport (~> 4.1.0) bundler (~> 1.1) erubis @@ -175,3 +175,6 @@ PLATFORMS DEPENDENCIES middleman-hashicorp! + +BUNDLED WITH + 1.10.2 diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index 5e2c31e90..0ff9522df 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -144,7 +144,8 @@ each category, the available configuration keys are alphabetized. or "5m". The default SSH timeout is "5m", or five minutes. * `subnet_id` (string) - If using VPC, the ID of the subnet, such as - "subnet-12345def", where Packer will launch the EC2 instance. + "subnet-12345def", where Packer will launch the EC2 instance. This field is + required if you are using an non-default VPC. * `tags` (object of key/value strings) - Tags applied to the AMI. diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown index 8360322de..ae5fbff27 100644 --- a/website/source/docs/builders/amazon-instance.html.markdown +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -184,7 +184,8 @@ each category, the available configuration keys are alphabetized. or "5m". The default SSH timeout is "5m", or five minutes. * `subnet_id` (string) - If using VPC, the ID of the subnet, such as - "subnet-12345def", where Packer will launch the EC2 instance. + "subnet-12345def", where Packer will launch the EC2 instance. This field is + required if you are using an non-default VPC. * `tags` (object of key/value strings) - Tags applied to the AMI. From e88fa43cfe9bdb785176c6b9c1944ebecb43157b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 16:45:24 -0400 Subject: [PATCH 096/208] find proper extension --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 2ebc66422..efb4e7d31 100644 --- a/config.go +++ b/config.go @@ -175,7 +175,7 @@ func (c *config) discoverSingle(glob string, m *map[string]string) error { // One Windows, ignore any plugins that don't end in .exe. // We could do a full PATHEXT parse, but this is probably good enough. - if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != "exe" { + if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" { log.Printf( "[DEBUG] Ignoring plugin match %s, no exe extension", match) From ae386d0a36c1485276d76ac8a7c30b8fb77c97dd Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 16:55:32 -0400 Subject: [PATCH 097/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4705c3523..bef4170f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ BUG FIXES: * post-processor/vagrant-cloud: Don't delete version on error [GH-2014] * provisioner/puppet-masterless: Allow manifest_file to be a directory * provisioner/salt-masterless: Add `--retcode-passthrough` to salt-call + * provisioner/shell: chmod executable script to 0755, not 0777 [GH-1708] ## 0.7.5 (December 9, 2014) From 8ca8bd7866e29bfcf38e84c5393146922e5a404e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 16:57:44 -0400 Subject: [PATCH 098/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bef4170f1..ded839503 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] * builder/digitalocean: Save SSH key to pwd if debug mode is on. [GH-1829] + * builder/digitalocean: User data support [GH-2113] * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear From 3ed73852be35e04cb69c05459a298e7b1eb4a37c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 17:19:23 -0400 Subject: [PATCH 099/208] provisioner/shell: set -e on the shebang itself --- provisioner/shell/provisioner.go | 3 +-- website/source/docs/provisioners/shell.html.markdown | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index 8b6ceb705..0d55c981b 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -94,7 +94,7 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { } if p.config.InlineShebang == "" { - p.config.InlineShebang = "/bin/sh" + p.config.InlineShebang = "/bin/sh -e" } if p.config.RawStartRetryTimeout == "" { @@ -184,7 +184,6 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { // Write our contents to it writer := bufio.NewWriter(tf) writer.WriteString(fmt.Sprintf("#!%s\n", p.config.InlineShebang)) - writer.WriteString("set -e\n") for _, command := range p.config.Inline { if _, err := writer.WriteString(command + "\n"); err != nil { return fmt.Errorf("Error preparing shell script: %s", err) diff --git a/website/source/docs/provisioners/shell.html.markdown b/website/source/docs/provisioners/shell.html.markdown index 7fcbe885b..e57910cb0 100644 --- a/website/source/docs/provisioners/shell.html.markdown +++ b/website/source/docs/provisioners/shell.html.markdown @@ -66,8 +66,10 @@ Optional parameters: * `inline_shebang` (string) - The [shebang](http://en.wikipedia.org/wiki/Shebang_%28Unix%29) value to use when - running commands specified by `inline`. By default, this is `/bin/sh`. + running commands specified by `inline`. By default, this is `/bin/sh -e`. If you're not using `inline`, then this configuration has no effect. + **Important:** If you customize this, be sure to include something like + the `-e` flag, otherwise individual steps failing won't fail the provisioner. * `remote_path` (string) - The path where the script will be uploaded to in the machine. This defaults to "/tmp/script.sh". This value must be From 2a912f7013703e263816513914ec374a9edaba6a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 17:20:13 -0400 Subject: [PATCH 100/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ded839503..207920b3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ BUG FIXES: * provisioner/puppet-masterless: Allow manifest_file to be a directory * provisioner/salt-masterless: Add `--retcode-passthrough` to salt-call * provisioner/shell: chmod executable script to 0755, not 0777 [GH-1708] + * provisioner/shell: inline commands failing will fail the provisioner [GH-2069] ## 0.7.5 (December 9, 2014) From 04e174fae899e80cb1d5d5a9b558e6c23a01a930 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Thu, 11 Jun 2015 16:21:29 -0500 Subject: [PATCH 101/208] builder/amazon: Properly return error code on ssh errors --- common/step_connect_ssh.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/common/step_connect_ssh.go b/common/step_connect_ssh.go index c3d8aac2d..d45767ee5 100644 --- a/common/step_connect_ssh.go +++ b/common/step_connect_ssh.go @@ -3,13 +3,14 @@ package common import ( "errors" "fmt" + "log" + "strings" + "time" + "github.com/mitchellh/multistep" "github.com/mitchellh/packer/communicator/ssh" "github.com/mitchellh/packer/packer" gossh "golang.org/x/crypto/ssh" - "log" - "strings" - "time" ) // StepConnectSSH is a multistep Step implementation that waits for SSH @@ -64,6 +65,7 @@ WaitLoop: case <-waitDone: if err != nil { ui.Error(fmt.Sprintf("Error waiting for SSH: %s", err)) + state.Put("error", err) return multistep.ActionHalt } From 7830d78d063f5ade6d6b9b929f8b2c2a1a06999e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 17:24:02 -0400 Subject: [PATCH 102/208] provisioner/shell: fix tests --- provisioner/shell/provisioner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioner/shell/provisioner_test.go b/provisioner/shell/provisioner_test.go index 4fd5a5cad..54c41c956 100644 --- a/provisioner/shell/provisioner_test.go +++ b/provisioner/shell/provisioner_test.go @@ -45,7 +45,7 @@ func TestProvisionerPrepare_InlineShebang(t *testing.T) { t.Fatalf("should not have error: %s", err) } - if p.config.InlineShebang != "/bin/sh" { + if p.config.InlineShebang != "/bin/sh -e" { t.Fatalf("bad value: %s", p.config.InlineShebang) } From 25f5d6dba318d4894ca83e9fdc8a2fcd10273826 Mon Sep 17 00:00:00 2001 From: Clint Date: Thu, 11 Jun 2015 16:29:15 -0500 Subject: [PATCH 103/208] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 207920b3a..b1bc247f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ BUG FIXES: * builder/amazon: Allow spaces in AMI names when using `clean_ami_name` [GH-2182] * builder/amazon: Remove deprecated ec2-upload-bundle paramger [GH-1931] * builder/amazon: Use IAM Profile to upload bundle if provided [GH-1985] + * builder/amazon: Use correct exit code after SSH authentication failed [GH-2004] * builder/amazon: Retry finding created instance for eventual consistency. [GH-2129] * builder/amazon: If no AZ is specified, use AZ chosen automatically by From 17555ff21ad8c00f03101dd25743b25e716606a7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 17:38:25 -0400 Subject: [PATCH 104/208] update version for dev --- version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version.go b/version.go index f7e44a1d0..dbfc8da12 100644 --- a/version.go +++ b/version.go @@ -4,9 +4,9 @@ package main var GitCommit string // The main version number that is being run at the moment. -const Version = "0.7.5" +const Version = "0.8.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -const VersionPrerelease = "" +const VersionPrerelease = "dev" From c903579aaacc66c01c893e4dad21480f1188f415 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 23:43:36 -0400 Subject: [PATCH 105/208] builder/openstack-new --- builder/openstack-new/access_config.go | 109 ++++++++++++++ builder/openstack-new/artifact.go | 47 ++++++ builder/openstack-new/artifact_test.go | 35 +++++ builder/openstack-new/builder.go | 134 ++++++++++++++++++ builder/openstack-new/builder_test.go | 94 ++++++++++++ builder/openstack-new/image_config.go | 25 ++++ builder/openstack-new/image_config_test.go | 23 +++ builder/openstack-new/run_config.go | 75 ++++++++++ builder/openstack-new/run_config_test.go | 88 ++++++++++++ builder/openstack-new/server.go | 100 +++++++++++++ builder/openstack-new/ssh.go | 89 ++++++++++++ builder/openstack-new/step_allocate_ip.go | 94 ++++++++++++ builder/openstack-new/step_create_image.go | 82 +++++++++++ builder/openstack-new/step_key_pair.go | 106 ++++++++++++++ .../openstack-new/step_run_source_server.go | 110 ++++++++++++++ .../step_wait_for_rackconnect.go | 52 +++++++ plugin/builder-openstack-new/main.go | 15 ++ 17 files changed, 1278 insertions(+) create mode 100644 builder/openstack-new/access_config.go create mode 100644 builder/openstack-new/artifact.go create mode 100644 builder/openstack-new/artifact_test.go create mode 100644 builder/openstack-new/builder.go create mode 100644 builder/openstack-new/builder_test.go create mode 100644 builder/openstack-new/image_config.go create mode 100644 builder/openstack-new/image_config_test.go create mode 100644 builder/openstack-new/run_config.go create mode 100644 builder/openstack-new/run_config_test.go create mode 100644 builder/openstack-new/server.go create mode 100644 builder/openstack-new/ssh.go create mode 100644 builder/openstack-new/step_allocate_ip.go create mode 100644 builder/openstack-new/step_create_image.go create mode 100644 builder/openstack-new/step_key_pair.go create mode 100644 builder/openstack-new/step_run_source_server.go create mode 100644 builder/openstack-new/step_wait_for_rackconnect.go create mode 100644 plugin/builder-openstack-new/main.go diff --git a/builder/openstack-new/access_config.go b/builder/openstack-new/access_config.go new file mode 100644 index 000000000..e0f962c50 --- /dev/null +++ b/builder/openstack-new/access_config.go @@ -0,0 +1,109 @@ +package openstack + +import ( + "crypto/tls" + "fmt" + "net/http" + "os" + + "github.com/mitchellh/packer/template/interpolate" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack" +) + +// AccessConfig is for common configuration related to openstack access +type AccessConfig struct { + Username string `mapstructure:"username"` + UserID string `mapstructure:"user_id"` + Password string `mapstructure:"password"` + APIKey string `mapstructure:"api_key"` + IdentityEndpoint string `mapstructure:"identity_endpoint"` + TenantID string `mapstructure:"tenant_id"` + TenantName string `mapstructure:"tenant_name"` + DomainID string `mapstructure:"domain_id"` + DomainName string `mapstructure:"domain_name"` + Insecure bool `mapstructure:"insecure"` + Region string `mapstructure:"region"` + EndpointType string `mapstructure:"endpoint_type"` + + osClient *gophercloud.ProviderClient +} + +func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { + if c.EndpointType != "internal" && c.EndpointType != "internalURL" && + c.EndpointType != "admin" && c.EndpointType != "adminURL" && + c.EndpointType != "public" && c.EndpointType != "publicURL" && + c.EndpointType != "" { + return []error{fmt.Errorf("Invalid endpoint type provided")} + } + + if c.Region == "" { + c.Region = os.Getenv("OS_REGION_NAME") + } + + // Get as much as possible from the end + ao, err := openstack.AuthOptionsFromEnv() + if err != nil { + return []error{err} + } + + // Override values if we have them in our config + overrides := []struct { + From, To *string + }{ + {&c.Username, &ao.Username}, + {&c.UserID, &ao.UserID}, + {&c.Password, &ao.Password}, + {&c.APIKey, &ao.APIKey}, + {&c.IdentityEndpoint, &ao.IdentityEndpoint}, + {&c.TenantID, &ao.TenantID}, + {&c.TenantName, &ao.TenantName}, + {&c.DomainID, &ao.DomainID}, + {&c.DomainName, &ao.DomainName}, + } + for _, s := range overrides { + if *s.From != "" { + *s.To = *s.From + } + } + + // Build the client itself + client, err := openstack.NewClient(ao.IdentityEndpoint) + if err != nil { + return []error{err} + } + + // If we have insecure set, then create a custom HTTP client that + // ignores SSL errors. + if c.Insecure { + config := &tls.Config{InsecureSkipVerify: true} + transport := &http.Transport{TLSClientConfig: config} + client.HTTPClient.Transport = transport + } + + // Auth + err = openstack.Authenticate(client, ao) + if err != nil { + return []error{err} + } + + c.osClient = client + return nil +} + +func (c *AccessConfig) computeV2Client() (*gophercloud.ServiceClient, error) { + return openstack.NewComputeV2(c.osClient, gophercloud.EndpointOpts{ + Region: c.Region, + Availability: c.getEndpointType(), + }) +} + +func (c *AccessConfig) getEndpointType() gophercloud.Availability { + if c.EndpointType == "internal" || c.EndpointType == "internalURL" { + return gophercloud.AvailabilityInternal + } + if c.EndpointType == "admin" || c.EndpointType == "adminURL" { + return gophercloud.AvailabilityAdmin + } + return gophercloud.AvailabilityPublic +} diff --git a/builder/openstack-new/artifact.go b/builder/openstack-new/artifact.go new file mode 100644 index 000000000..aa60d2641 --- /dev/null +++ b/builder/openstack-new/artifact.go @@ -0,0 +1,47 @@ +package openstack + +import ( + "fmt" + "log" + + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/images" +) + +// Artifact is an artifact implementation that contains built images. +type Artifact struct { + // ImageId of built image + ImageId string + + // BuilderId is the unique ID for the builder that created this image + BuilderIdValue string + + // OpenStack connection for performing API stuff. + Client *gophercloud.ServiceClient +} + +func (a *Artifact) BuilderId() string { + return a.BuilderIdValue +} + +func (*Artifact) Files() []string { + // We have no files + return nil +} + +func (a *Artifact) Id() string { + return a.ImageId +} + +func (a *Artifact) String() string { + return fmt.Sprintf("An image was created: %v", a.ImageId) +} + +func (a *Artifact) State(name string) interface{} { + return nil +} + +func (a *Artifact) Destroy() error { + log.Printf("Destroying image: %s", a.ImageId) + return images.Delete(a.Client, a.ImageId).ExtractErr() +} diff --git a/builder/openstack-new/artifact_test.go b/builder/openstack-new/artifact_test.go new file mode 100644 index 000000000..313fea7cf --- /dev/null +++ b/builder/openstack-new/artifact_test.go @@ -0,0 +1,35 @@ +package openstack + +import ( + "github.com/mitchellh/packer/packer" + "testing" +) + +func TestArtifact_Impl(t *testing.T) { + var _ packer.Artifact = new(Artifact) +} + +func TestArtifactId(t *testing.T) { + expected := `b8cdf55b-c916-40bd-b190-389ec144c4ed` + + a := &Artifact{ + ImageId: "b8cdf55b-c916-40bd-b190-389ec144c4ed", + } + + result := a.Id() + if result != expected { + t.Fatalf("bad: %s", result) + } +} + +func TestArtifactString(t *testing.T) { + expected := "An image was created: b8cdf55b-c916-40bd-b190-389ec144c4ed" + + a := &Artifact{ + ImageId: "b8cdf55b-c916-40bd-b190-389ec144c4ed", + } + result := a.String() + if result != expected { + t.Fatalf("bad: %s", result) + } +} diff --git a/builder/openstack-new/builder.go b/builder/openstack-new/builder.go new file mode 100644 index 000000000..bebb28452 --- /dev/null +++ b/builder/openstack-new/builder.go @@ -0,0 +1,134 @@ +// The openstack package contains a packer.Builder implementation that +// builds Images for openstack. + +package openstack + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/common" + "log" + + "github.com/mitchellh/packer/helper/config" + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/template/interpolate" +) + +// The unique ID for this builder +const BuilderId = "mitchellh.openstack" + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + AccessConfig `mapstructure:",squash"` + ImageConfig `mapstructure:",squash"` + RunConfig `mapstructure:",squash"` + + ctx interpolate.Context +} + +type Builder struct { + config Config + runner multistep.Runner +} + +func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { + err := config.Decode(&b.config, &config.DecodeOpts{ + Interpolate: true, + }, raws...) + if err != nil { + return nil, err + } + + // Accumulate any errors + var errs *packer.MultiError + errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) + errs = packer.MultiErrorAppend(errs, b.config.ImageConfig.Prepare(&b.config.ctx)...) + errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...) + + if errs != nil && len(errs.Errors) > 0 { + return nil, errs + } + + log.Println(common.ScrubConfig(b.config, b.config.Password)) + return nil, nil +} + +func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { + computeClient, err := b.config.computeV2Client() + if err != nil { + return nil, fmt.Errorf("Error initializing compute client: %s", err) + } + + // Setup the state bag and initial state for the steps + state := new(multistep.BasicStateBag) + state.Put("config", b.config) + state.Put("hook", hook) + state.Put("ui", ui) + + // Build the steps + steps := []multistep.Step{ + &StepKeyPair{ + Debug: b.config.PackerDebug, + DebugKeyPath: fmt.Sprintf("os_%s.pem", b.config.PackerBuildName), + }, + &StepRunSourceServer{ + Name: b.config.ImageName, + Flavor: b.config.Flavor, + SourceImage: b.config.SourceImage, + SecurityGroups: b.config.SecurityGroups, + Networks: b.config.Networks, + }, + &StepWaitForRackConnect{ + Wait: b.config.RackconnectWait, + }, + &StepAllocateIp{ + FloatingIpPool: b.config.FloatingIpPool, + FloatingIp: b.config.FloatingIp, + }, + &common.StepConnectSSH{ + SSHAddress: SSHAddress(computeClient, b.config.SSHInterface, b.config.SSHPort), + SSHConfig: SSHConfig(b.config.SSHUsername), + SSHWaitTimeout: b.config.SSHTimeout(), + }, + &common.StepProvision{}, + &stepCreateImage{}, + } + + // Run! + if b.config.PackerDebug { + b.runner = &multistep.DebugRunner{ + Steps: steps, + PauseFn: common.MultistepDebugFn(ui), + } + } else { + b.runner = &multistep.BasicRunner{Steps: steps} + } + + b.runner.Run(state) + + // If there was an error, return that + if rawErr, ok := state.GetOk("error"); ok { + return nil, rawErr.(error) + } + + // If there are no images, then just return + if _, ok := state.GetOk("image"); !ok { + return nil, nil + } + + // Build the artifact and return it + artifact := &Artifact{ + ImageId: state.Get("image").(string), + BuilderIdValue: BuilderId, + Client: computeClient, + } + + return artifact, nil +} + +func (b *Builder) Cancel() { + if b.runner != nil { + log.Println("Cancelling the step runner...") + b.runner.Cancel() + } +} diff --git a/builder/openstack-new/builder_test.go b/builder/openstack-new/builder_test.go new file mode 100644 index 000000000..badf9784d --- /dev/null +++ b/builder/openstack-new/builder_test.go @@ -0,0 +1,94 @@ +package openstack + +import ( + "github.com/mitchellh/packer/packer" + "testing" +) + +func testConfig() map[string]interface{} { + return map[string]interface{}{ + "username": "foo", + "password": "bar", + "provider": "foo", + "region": "DFW", + "image_name": "foo", + "source_image": "foo", + "flavor": "foo", + "ssh_username": "root", + } +} + +func TestBuilder_ImplementsBuilder(t *testing.T) { + var raw interface{} + raw = &Builder{} + if _, ok := raw.(packer.Builder); !ok { + t.Fatalf("Builder should be a builder") + } +} + +func TestBuilder_Prepare_BadType(t *testing.T) { + b := &Builder{} + c := map[string]interface{}{ + "password": []string{}, + } + + warns, err := b.Prepare(c) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err == nil { + t.Fatalf("prepare should fail") + } +} + +func TestBuilderPrepare_ImageName(t *testing.T) { + var b Builder + config := testConfig() + + // Test good + config["image_name"] = "foo" + warns, err := b.Prepare(config) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + // Test bad + config["image_name"] = "foo {{" + b = Builder{} + warns, err = b.Prepare(config) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err == nil { + t.Fatal("should have error") + } + + // Test bad + delete(config, "image_name") + b = Builder{} + warns, err = b.Prepare(config) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err == nil { + t.Fatal("should have error") + } +} + +func TestBuilderPrepare_InvalidKey(t *testing.T) { + var b Builder + config := testConfig() + + // Add a random key + config["i_should_not_be_valid"] = true + warns, err := b.Prepare(config) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err == nil { + t.Fatal("should have error") + } +} diff --git a/builder/openstack-new/image_config.go b/builder/openstack-new/image_config.go new file mode 100644 index 000000000..124449eab --- /dev/null +++ b/builder/openstack-new/image_config.go @@ -0,0 +1,25 @@ +package openstack + +import ( + "fmt" + + "github.com/mitchellh/packer/template/interpolate" +) + +// ImageConfig is for common configuration related to creating Images. +type ImageConfig struct { + ImageName string `mapstructure:"image_name"` +} + +func (c *ImageConfig) Prepare(ctx *interpolate.Context) []error { + errs := make([]error, 0) + if c.ImageName == "" { + errs = append(errs, fmt.Errorf("An image_name must be specified")) + } + + if len(errs) > 0 { + return errs + } + + return nil +} diff --git a/builder/openstack-new/image_config_test.go b/builder/openstack-new/image_config_test.go new file mode 100644 index 000000000..4d81ecd94 --- /dev/null +++ b/builder/openstack-new/image_config_test.go @@ -0,0 +1,23 @@ +package openstack + +import ( + "testing" +) + +func testImageConfig() *ImageConfig { + return &ImageConfig{ + ImageName: "foo", + } +} + +func TestImageConfigPrepare_Region(t *testing.T) { + c := testImageConfig() + if err := c.Prepare(nil); err != nil { + t.Fatalf("shouldn't have err: %s", err) + } + + c.ImageName = "" + if err := c.Prepare(nil); err == nil { + t.Fatal("should have error") + } +} diff --git a/builder/openstack-new/run_config.go b/builder/openstack-new/run_config.go new file mode 100644 index 000000000..e5d73c9c1 --- /dev/null +++ b/builder/openstack-new/run_config.go @@ -0,0 +1,75 @@ +package openstack + +import ( + "errors" + "fmt" + "time" + + "github.com/mitchellh/packer/template/interpolate" +) + +// RunConfig contains configuration for running an instance from a source +// image and details on how to access that launched image. +type RunConfig struct { + SourceImage string `mapstructure:"source_image"` + Flavor string `mapstructure:"flavor"` + RawSSHTimeout string `mapstructure:"ssh_timeout"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort int `mapstructure:"ssh_port"` + SSHInterface string `mapstructure:"ssh_interface"` + OpenstackProvider string `mapstructure:"openstack_provider"` + UseFloatingIp bool `mapstructure:"use_floating_ip"` + RackconnectWait bool `mapstructure:"rackconnect_wait"` + FloatingIpPool string `mapstructure:"floating_ip_pool"` + FloatingIp string `mapstructure:"floating_ip"` + SecurityGroups []string `mapstructure:"security_groups"` + Networks []string `mapstructure:"networks"` + + // Unexported fields that are calculated from others + sshTimeout time.Duration +} + +func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { + // Defaults + if c.SSHUsername == "" { + c.SSHUsername = "root" + } + + if c.SSHPort == 0 { + c.SSHPort = 22 + } + + if c.RawSSHTimeout == "" { + c.RawSSHTimeout = "5m" + } + + if c.UseFloatingIp && c.FloatingIpPool == "" { + c.FloatingIpPool = "public" + } + + // Validation + var err error + errs := make([]error, 0) + if c.SourceImage == "" { + errs = append(errs, errors.New("A source_image must be specified")) + } + + if c.Flavor == "" { + errs = append(errs, errors.New("A flavor must be specified")) + } + + if c.SSHUsername == "" { + errs = append(errs, errors.New("An ssh_username must be specified")) + } + + c.sshTimeout, err = time.ParseDuration(c.RawSSHTimeout) + if err != nil { + errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) + } + + return errs +} + +func (c *RunConfig) SSHTimeout() time.Duration { + return c.sshTimeout +} diff --git a/builder/openstack-new/run_config_test.go b/builder/openstack-new/run_config_test.go new file mode 100644 index 000000000..16b89b352 --- /dev/null +++ b/builder/openstack-new/run_config_test.go @@ -0,0 +1,88 @@ +package openstack + +import ( + "os" + "testing" +) + +func init() { + // Clear out the openstack env vars so they don't + // affect our tests. + os.Setenv("SDK_USERNAME", "") + os.Setenv("SDK_PASSWORD", "") + os.Setenv("SDK_PROVIDER", "") +} + +func testRunConfig() *RunConfig { + return &RunConfig{ + SourceImage: "abcd", + Flavor: "m1.small", + SSHUsername: "root", + } +} + +func TestRunConfigPrepare(t *testing.T) { + c := testRunConfig() + err := c.Prepare(nil) + if len(err) > 0 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_InstanceType(t *testing.T) { + c := testRunConfig() + c.Flavor = "" + if err := c.Prepare(nil); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SourceImage(t *testing.T) { + c := testRunConfig() + c.SourceImage = "" + if err := c.Prepare(nil); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SSHPort(t *testing.T) { + c := testRunConfig() + c.SSHPort = 0 + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.SSHPort != 22 { + t.Fatalf("invalid value: %d", c.SSHPort) + } + + c.SSHPort = 44 + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.SSHPort != 44 { + t.Fatalf("invalid value: %d", c.SSHPort) + } +} + +func TestRunConfigPrepare_SSHTimeout(t *testing.T) { + c := testRunConfig() + c.RawSSHTimeout = "" + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + c.RawSSHTimeout = "bad" + if err := c.Prepare(nil); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SSHUsername(t *testing.T) { + c := testRunConfig() + c.SSHUsername = "" + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } +} diff --git a/builder/openstack-new/server.go b/builder/openstack-new/server.go new file mode 100644 index 000000000..a87ef0110 --- /dev/null +++ b/builder/openstack-new/server.go @@ -0,0 +1,100 @@ +package openstack + +import ( + "errors" + "fmt" + "log" + "time" + + "github.com/mitchellh/multistep" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +// StateRefreshFunc is a function type used for StateChangeConf that is +// responsible for refreshing the item being watched for a state change. +// +// It returns three results. `result` is any object that will be returned +// as the final object after waiting for state change. This allows you to +// return the final updated object, for example an openstack instance after +// refreshing it. +// +// `state` is the latest state of that object. And `err` is any error that +// may have happened while refreshing the state. +type StateRefreshFunc func() (result interface{}, state string, progress int, err error) + +// StateChangeConf is the configuration struct used for `WaitForState`. +type StateChangeConf struct { + Pending []string + Refresh StateRefreshFunc + StepState multistep.StateBag + Target string +} + +// ServerStateRefreshFunc returns a StateRefreshFunc that is used to watch +// an openstack server. +func ServerStateRefreshFunc( + client *gophercloud.ServiceClient, s *servers.Server) StateRefreshFunc { + return func() (interface{}, string, int, error) { + var serverNew *servers.Server + result := servers.Get(client, s.ID) + err := result.Err + if err == nil { + serverNew, err = result.Extract() + } + if result.Err != nil { + errCode, ok := result.Err.(*gophercloud.UnexpectedResponseCodeError) + if ok && errCode.Actual == 404 { + log.Printf("[INFO] 404 on ServerStateRefresh, returning DELETED") + return nil, "DELETED", 0, nil + } else { + log.Printf("[ERROR] Error on ServerStateRefresh: %s", result.Err) + return nil, "", 0, result.Err + } + } + + return serverNew, serverNew.Status, serverNew.Progress, nil + } +} + +// WaitForState watches an object and waits for it to achieve a certain +// state. +func WaitForState(conf *StateChangeConf) (i interface{}, err error) { + log.Printf("Waiting for state to become: %s", conf.Target) + + for { + var currentProgress int + var currentState string + i, currentState, currentProgress, err = conf.Refresh() + if err != nil { + return + } + + if currentState == conf.Target { + return + } + + if conf.StepState != nil { + if _, ok := conf.StepState.GetOk(multistep.StateCancelled); ok { + return nil, errors.New("interrupted") + } + } + + found := false + for _, allowed := range conf.Pending { + if currentState == allowed { + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target) + } + + log.Printf("Waiting for state to become: %s currently %s (%d%%)", conf.Target, currentState, currentProgress) + time.Sleep(2 * time.Second) + } + + return +} diff --git a/builder/openstack-new/ssh.go b/builder/openstack-new/ssh.go new file mode 100644 index 000000000..a3de654f6 --- /dev/null +++ b/builder/openstack-new/ssh.go @@ -0,0 +1,89 @@ +package openstack + +import ( + "errors" + "fmt" + "time" + + "github.com/mitchellh/multistep" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" + "golang.org/x/crypto/ssh" +) + +// SSHAddress returns a function that can be given to the SSH communicator +// for determining the SSH address based on the server AccessIPv4 setting.. +func SSHAddress( + client *gophercloud.ServiceClient, + sshinterface string, port int) func(multistep.StateBag) (string, error) { + return func(state multistep.StateBag) (string, error) { + s := state.Get("server").(*servers.Server) + + // If we have a floating IP, use that + if ip := state.Get("access_ip").(*floatingip.FloatingIP); ip.FixedIP != "" { + return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil + } + + if s.AccessIPv4 != "" { + return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil + } + + // Get all the addresses associated with this server + /* + ip_pools, err := s.AllAddressPools() + if err != nil { + return "", errors.New("Error parsing SSH addresses") + } + for pool, addresses := range ip_pools { + if sshinterface != "" { + if pool != sshinterface { + continue + } + } + if pool != "" { + for _, address := range addresses { + if address.Addr != "" && address.Version == 4 { + return fmt.Sprintf("%s:%d", address.Addr, port), nil + } + } + } + } + */ + + result := servers.Get(client, s.ID) + err := result.Err + if err == nil { + s, err = result.Extract() + } + if err != nil { + return "", err + } + + state.Put("server", s) + time.Sleep(1 * time.Second) + + return "", errors.New("couldn't determine IP address for server") + } +} + +// SSHConfig returns a function that can be used for the SSH communicator +// config for connecting to the instance created over SSH using the generated +// private key. +func SSHConfig(username string) func(multistep.StateBag) (*ssh.ClientConfig, error) { + return func(state multistep.StateBag) (*ssh.ClientConfig, error) { + privateKey := state.Get("privateKey").(string) + + signer, err := ssh.ParsePrivateKey([]byte(privateKey)) + if err != nil { + return nil, fmt.Errorf("Error setting up SSH config: %s", err) + } + + return &ssh.ClientConfig{ + User: username, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + }, nil + } +} diff --git a/builder/openstack-new/step_allocate_ip.go b/builder/openstack-new/step_allocate_ip.go new file mode 100644 index 000000000..adb15eb5b --- /dev/null +++ b/builder/openstack-new/step_allocate_ip.go @@ -0,0 +1,94 @@ +package openstack + +import ( + "fmt" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +type StepAllocateIp struct { + FloatingIpPool string + FloatingIp string +} + +func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { + ui := state.Get("ui").(packer.Ui) + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + var instanceIp *floatingip.FloatingIP + // This is here in case we error out before putting instanceIp into the + // statebag below, because it is requested by Cleanup() + state.Put("access_ip", instanceIp) + + if s.FloatingIp != "" { + instanceIp.FixedIP = s.FloatingIp + } else if s.FloatingIpPool != "" { + newIp, err := floatingip.Create(client, floatingip.CreateOpts{ + Pool: s.FloatingIpPool, + }).Extract() + if err != nil { + err := fmt.Errorf("Error creating floating ip from pool '%s'", s.FloatingIpPool) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + instanceIp = newIp + ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) + } + + if instanceIp.FixedIP != "" { + err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() + if err != nil { + err := fmt.Errorf( + "Error associating floating IP %s with instance.", + instanceIp.FixedIP) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + ui.Say(fmt.Sprintf( + "Added floating IP %s to instance...", instanceIp.FixedIP)) + } + + state.Put("access_ip", instanceIp) + + return multistep.ActionContinue +} + +func (s *StepAllocateIp) Cleanup(state multistep.StateBag) { + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + instanceIp := state.Get("access_ip").(*floatingip.FloatingIP) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf( + "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + return + } + + if s.FloatingIpPool != "" && instanceIp.ID != "" { + if err := floatingip.Delete(client, instanceIp.ID).ExtractErr(); err != nil { + ui.Error(fmt.Sprintf( + "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + return + } + + ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.FixedIP)) + } +} diff --git a/builder/openstack-new/step_create_image.go b/builder/openstack-new/step_create_image.go new file mode 100644 index 000000000..df540e311 --- /dev/null +++ b/builder/openstack-new/step_create_image.go @@ -0,0 +1,82 @@ +package openstack + +import ( + "fmt" + "log" + "time" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/images" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +type stepCreateImage struct{} + +func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + // Create the image + ui.Say(fmt.Sprintf("Creating the image: %s", config.ImageName)) + imageId, err := servers.CreateImage(client, server.ID, servers.CreateImageOpts{ + Name: config.ImageName, + }).ExtractImageID() + if err != nil { + err := fmt.Errorf("Error creating image: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Set the Image ID in the state + ui.Say(fmt.Sprintf("Image: %s", imageId)) + state.Put("image", imageId) + + // Wait for the image to become ready + ui.Say("Waiting for image to become ready...") + if err := WaitForImage(client, imageId); err != nil { + err := fmt.Errorf("Error waiting for image: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *stepCreateImage) Cleanup(multistep.StateBag) { + // No cleanup... +} + +// WaitForImage waits for the given Image ID to become ready. +func WaitForImage(client *gophercloud.ServiceClient, imageId string) error { + for { + var image *images.Image + result := images.Get(client, imageId) + err := result.Err + if err == nil { + image, err = result.Extract() + } + if err != nil { + return err + } + + if image.Status == "ACTIVE" { + return nil + } + + log.Printf("Waiting for image creation status: %s (%d%%)", image.Status, image.Progress) + time.Sleep(2 * time.Second) + } +} diff --git a/builder/openstack-new/step_key_pair.go b/builder/openstack-new/step_key_pair.go new file mode 100644 index 000000000..06bcbf9ea --- /dev/null +++ b/builder/openstack-new/step_key_pair.go @@ -0,0 +1,106 @@ +package openstack + +import ( + "fmt" + "os" + "runtime" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" +) + +type StepKeyPair struct { + Debug bool + DebugKeyPath string + keyName string +} + +func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Say("Creating temporary keypair for this instance...") + keyName := fmt.Sprintf("packer %s", uuid.TimeOrderedUUID()) + keypair, err := keypairs.Create(computeClient, keypairs.CreateOpts{ + Name: keyName, + }).Extract() + if err != nil { + state.Put("error", fmt.Errorf("Error creating temporary keypair: %s", err)) + return multistep.ActionHalt + } + + if keypair.PrivateKey == "" { + state.Put("error", fmt.Errorf("The temporary keypair returned was blank")) + return multistep.ActionHalt + } + + // If we're in debug mode, output the private key to the working + // directory. + if s.Debug { + ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath)) + f, err := os.Create(s.DebugKeyPath) + if err != nil { + state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) + return multistep.ActionHalt + } + defer f.Close() + + // Write the key out + if _, err := f.Write([]byte(keypair.PrivateKey)); err != nil { + state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) + return multistep.ActionHalt + } + + // Chmod it so that it is SSH ready + if runtime.GOOS != "windows" { + if err := f.Chmod(0600); err != nil { + state.Put("error", fmt.Errorf("Error setting permissions of debug key: %s", err)) + return multistep.ActionHalt + } + } + } + + // Set the keyname so we know to delete it later + s.keyName = keyName + + // Set some state data for use in future steps + state.Put("keyPair", keyName) + state.Put("privateKey", keypair.PrivateKey) + + return multistep.ActionContinue +} + +func (s *StepKeyPair) Cleanup(state multistep.StateBag) { + // If no key name is set, then we never created it, so just return + if s.keyName == "" { + return + } + + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf( + "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) + return + } + + ui.Say("Deleting temporary keypair...") + err = keypairs.Delete(computeClient, s.keyName).ExtractErr() + if err != nil { + ui.Error(fmt.Sprintf( + "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) + } +} diff --git a/builder/openstack-new/step_run_source_server.go b/builder/openstack-new/step_run_source_server.go new file mode 100644 index 000000000..e58e2c46b --- /dev/null +++ b/builder/openstack-new/step_run_source_server.go @@ -0,0 +1,110 @@ +package openstack + +import ( + "fmt" + "log" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +type StepRunSourceServer struct { + Flavor string + Name string + SourceImage string + SecurityGroups []string + Networks []string + + server *servers.Server +} + +func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(Config) + keyName := state.Get("keyPair").(string) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + networks := make([]servers.Network, len(s.Networks)) + for i, networkUuid := range s.Networks { + networks[i].UUID = networkUuid + } + + s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ + CreateOptsBuilder: servers.CreateOpts{ + Name: s.Name, + ImageRef: s.SourceImage, + FlavorRef: s.Flavor, + SecurityGroups: s.SecurityGroups, + Networks: networks, + }, + + KeyName: keyName, + }).Extract() + if err != nil { + err := fmt.Errorf("Error launching source server: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + log.Printf("server id: %s", s.server.ID) + + ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.ID)) + stateChange := StateChangeConf{ + Pending: []string{"BUILD"}, + Target: "ACTIVE", + Refresh: ServerStateRefreshFunc(computeClient, s.server), + StepState: state, + } + latestServer, err := WaitForState(&stateChange) + if err != nil { + err := fmt.Errorf("Error waiting for server (%s) to become ready: %s", s.server.ID, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + s.server = latestServer.(*servers.Server) + state.Put("server", s.server) + + return multistep.ActionContinue +} + +func (s *StepRunSourceServer) Cleanup(state multistep.StateBag) { + if s.server == nil { + return + } + + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) + return + } + + ui.Say("Terminating the source server...") + if err := servers.Delete(computeClient, s.server.ID).ExtractErr(); err != nil { + ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) + return + } + + stateChange := StateChangeConf{ + Pending: []string{"ACTIVE", "BUILD", "REBUILD", "SUSPENDED"}, + Refresh: ServerStateRefreshFunc(computeClient, s.server), + Target: "DELETED", + } + + WaitForState(&stateChange) +} diff --git a/builder/openstack-new/step_wait_for_rackconnect.go b/builder/openstack-new/step_wait_for_rackconnect.go new file mode 100644 index 000000000..6263bd17d --- /dev/null +++ b/builder/openstack-new/step_wait_for_rackconnect.go @@ -0,0 +1,52 @@ +package openstack + +import ( + "fmt" + "time" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +type StepWaitForRackConnect struct { + Wait bool +} + +func (s *StepWaitForRackConnect) Run(state multistep.StateBag) multistep.StepAction { + if !s.Wait { + return multistep.ActionContinue + } + + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Say(fmt.Sprintf( + "Waiting for server (%s) to become RackConnect ready...", server.ID)) + for { + server, err = servers.Get(computeClient, server.ID).Extract() + if err != nil { + return multistep.ActionHalt + } + + if server.Metadata["rackconnect_automation_status"] == "DEPLOYED" { + break + } + + time.Sleep(2 * time.Second) + } + + return multistep.ActionContinue +} + +func (s *StepWaitForRackConnect) Cleanup(state multistep.StateBag) { +} diff --git a/plugin/builder-openstack-new/main.go b/plugin/builder-openstack-new/main.go new file mode 100644 index 000000000..d8075c78d --- /dev/null +++ b/plugin/builder-openstack-new/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/mitchellh/packer/builder/openstack-new" + "github.com/mitchellh/packer/packer/plugin" +) + +func main() { + server, err := plugin.Server() + if err != nil { + panic(err) + } + server.RegisterBuilder(new(openstack.Builder)) + server.Serve() +} From 551e80774d0b0f28547a5373a59821fbf0cfeaf8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:03:17 -0400 Subject: [PATCH 106/208] builder/openstack-new: fix some issues --- builder/openstack-new/ssh.go | 53 ++++++++++--------- builder/openstack-new/step_allocate_ip.go | 7 ++- .../openstack-new/step_run_source_server.go | 2 +- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/builder/openstack-new/ssh.go b/builder/openstack-new/ssh.go index a3de654f6..7b0510f98 100644 --- a/builder/openstack-new/ssh.go +++ b/builder/openstack-new/ssh.go @@ -3,6 +3,7 @@ package openstack import ( "errors" "fmt" + "log" "time" "github.com/mitchellh/multistep" @@ -21,7 +22,8 @@ func SSHAddress( s := state.Get("server").(*servers.Server) // If we have a floating IP, use that - if ip := state.Get("access_ip").(*floatingip.FloatingIP); ip.FixedIP != "" { + ip := state.Get("access_ip").(*floatingip.FloatingIP) + if ip != nil && ip.FixedIP != "" { return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil } @@ -29,33 +31,34 @@ func SSHAddress( return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil } - // Get all the addresses associated with this server - /* - ip_pools, err := s.AllAddressPools() - if err != nil { - return "", errors.New("Error parsing SSH addresses") + // Get all the addresses associated with this server. This + // was taken directly from Terraform. + for _, networkAddresses := range s.Addresses { + elements, ok := networkAddresses.([]interface{}) + if !ok { + log.Printf( + "[ERROR] Unknown return type for address field: %#v", + networkAddresses) + continue } - for pool, addresses := range ip_pools { - if sshinterface != "" { - if pool != sshinterface { - continue - } - } - if pool != "" { - for _, address := range addresses { - if address.Addr != "" && address.Version == 4 { - return fmt.Sprintf("%s:%d", address.Addr, port), nil - } - } - } - } - */ - result := servers.Get(client, s.ID) - err := result.Err - if err == nil { - s, err = result.Extract() + for _, element := range elements { + var addr string + address := element.(map[string]interface{}) + if address["OS-EXT-IPS:type"] == "floating" { + addr = address["addr"].(string) + } else { + if address["version"].(float64) == 4 { + addr = address["addr"].(string) + } + } + if addr != "" { + return fmt.Sprintf("%s:%d", addr, port), nil + } + } } + + s, err := servers.Get(client, s.ID).Extract() if err != nil { return "", err } diff --git a/builder/openstack-new/step_allocate_ip.go b/builder/openstack-new/step_allocate_ip.go index adb15eb5b..16efe8d38 100644 --- a/builder/openstack-new/step_allocate_ip.go +++ b/builder/openstack-new/step_allocate_ip.go @@ -33,7 +33,7 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { state.Put("access_ip", instanceIp) if s.FloatingIp != "" { - instanceIp.FixedIP = s.FloatingIp + *instanceIp = floatingip.FloatingIP{FixedIP: s.FloatingIp} } else if s.FloatingIpPool != "" { newIp, err := floatingip.Create(client, floatingip.CreateOpts{ Pool: s.FloatingIpPool, @@ -45,11 +45,11 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - instanceIp = newIp + *instanceIp = *newIp ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) } - if instanceIp.FixedIP != "" { + if instanceIp != nil && instanceIp.FixedIP != "" { err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() if err != nil { err := fmt.Errorf( @@ -65,7 +65,6 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { } state.Put("access_ip", instanceIp) - return multistep.ActionContinue } diff --git a/builder/openstack-new/step_run_source_server.go b/builder/openstack-new/step_run_source_server.go index e58e2c46b..4432d5860 100644 --- a/builder/openstack-new/step_run_source_server.go +++ b/builder/openstack-new/step_run_source_server.go @@ -42,7 +42,7 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction CreateOptsBuilder: servers.CreateOpts{ Name: s.Name, ImageRef: s.SourceImage, - FlavorRef: s.Flavor, + FlavorName: s.Flavor, SecurityGroups: s.SecurityGroups, Networks: networks, }, From 7a46b80cfb24aa307286a2b67c253f7716b3d3fc Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:05:24 -0400 Subject: [PATCH 107/208] builder/openstack-new: better UI --- builder/openstack-new/step_create_image.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/openstack-new/step_create_image.go b/builder/openstack-new/step_create_image.go index df540e311..989db81e5 100644 --- a/builder/openstack-new/step_create_image.go +++ b/builder/openstack-new/step_create_image.go @@ -40,7 +40,7 @@ func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { } // Set the Image ID in the state - ui.Say(fmt.Sprintf("Image: %s", imageId)) + ui.Message(fmt.Sprintf("Image: %s", imageId)) state.Put("image", imageId) // Wait for the image to become ready From 46f518f21df6a127d3e9e64eabfdc31441aa4a1c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:09:01 -0400 Subject: [PATCH 108/208] builder/openstack: proper error extraction --- builder/openstack-new/server.go | 15 +++++---------- builder/openstack-new/step_create_image.go | 14 ++++++++------ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/builder/openstack-new/server.go b/builder/openstack-new/server.go index a87ef0110..de8c9d103 100644 --- a/builder/openstack-new/server.go +++ b/builder/openstack-new/server.go @@ -36,20 +36,15 @@ type StateChangeConf struct { func ServerStateRefreshFunc( client *gophercloud.ServiceClient, s *servers.Server) StateRefreshFunc { return func() (interface{}, string, int, error) { - var serverNew *servers.Server - result := servers.Get(client, s.ID) - err := result.Err - if err == nil { - serverNew, err = result.Extract() - } - if result.Err != nil { - errCode, ok := result.Err.(*gophercloud.UnexpectedResponseCodeError) + serverNew, err := servers.Get(client, s.ID).Extract() + if err != nil { + errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) if ok && errCode.Actual == 404 { log.Printf("[INFO] 404 on ServerStateRefresh, returning DELETED") return nil, "DELETED", 0, nil } else { - log.Printf("[ERROR] Error on ServerStateRefresh: %s", result.Err) - return nil, "", 0, result.Err + log.Printf("[ERROR] Error on ServerStateRefresh: %s", err) + return nil, "", 0, err } } diff --git a/builder/openstack-new/step_create_image.go b/builder/openstack-new/step_create_image.go index 989db81e5..b777e8b0b 100644 --- a/builder/openstack-new/step_create_image.go +++ b/builder/openstack-new/step_create_image.go @@ -62,13 +62,15 @@ func (s *stepCreateImage) Cleanup(multistep.StateBag) { // WaitForImage waits for the given Image ID to become ready. func WaitForImage(client *gophercloud.ServiceClient, imageId string) error { for { - var image *images.Image - result := images.Get(client, imageId) - err := result.Err - if err == nil { - image, err = result.Extract() - } + image, err := images.Get(client, imageId).Extract() if err != nil { + errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) + if ok && errCode.Actual == 500 { + log.Printf("[ERROR] 500 error received, will ignore and retry: %s", err) + time.Sleep(2 * time.Second) + continue + } + return err } From a0d41fcd14f2525df351d1764e3401d1f016ff53 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:16:43 -0400 Subject: [PATCH 109/208] builder/openstack --- builder/openstack-new/access_config.go | 109 ------------ builder/openstack-new/artifact.go | 47 ----- builder/openstack-new/artifact_test.go | 35 ---- builder/openstack-new/builder.go | 134 -------------- builder/openstack-new/builder_test.go | 94 ---------- builder/openstack-new/image_config.go | 25 --- builder/openstack-new/image_config_test.go | 23 --- builder/openstack-new/run_config.go | 75 -------- builder/openstack-new/run_config_test.go | 88 ---------- builder/openstack-new/server.go | 95 ---------- builder/openstack-new/ssh.go | 92 ---------- builder/openstack-new/step_allocate_ip.go | 93 ---------- builder/openstack-new/step_create_image.go | 84 --------- builder/openstack-new/step_key_pair.go | 106 ------------ .../openstack-new/step_run_source_server.go | 110 ------------ .../step_wait_for_rackconnect.go | 52 ------ builder/openstack/access_config.go | 163 +++++++++--------- builder/openstack/access_config_test.go | 77 --------- builder/openstack/artifact.go | 7 +- builder/openstack/builder.go | 23 +-- builder/openstack/server.go | 23 +-- builder/openstack/ssh.go | 62 ++++--- builder/openstack/step_allocate_ip.go | 68 +++++--- builder/openstack/step_create_image.go | 39 +++-- builder/openstack/step_key_pair.go | 43 +++-- builder/openstack/step_run_source_server.go | 70 ++++---- .../openstack/step_wait_for_rackconnect.go | 22 ++- 27 files changed, 303 insertions(+), 1556 deletions(-) delete mode 100644 builder/openstack-new/access_config.go delete mode 100644 builder/openstack-new/artifact.go delete mode 100644 builder/openstack-new/artifact_test.go delete mode 100644 builder/openstack-new/builder.go delete mode 100644 builder/openstack-new/builder_test.go delete mode 100644 builder/openstack-new/image_config.go delete mode 100644 builder/openstack-new/image_config_test.go delete mode 100644 builder/openstack-new/run_config.go delete mode 100644 builder/openstack-new/run_config_test.go delete mode 100644 builder/openstack-new/server.go delete mode 100644 builder/openstack-new/ssh.go delete mode 100644 builder/openstack-new/step_allocate_ip.go delete mode 100644 builder/openstack-new/step_create_image.go delete mode 100644 builder/openstack-new/step_key_pair.go delete mode 100644 builder/openstack-new/step_run_source_server.go delete mode 100644 builder/openstack-new/step_wait_for_rackconnect.go delete mode 100644 builder/openstack/access_config_test.go diff --git a/builder/openstack-new/access_config.go b/builder/openstack-new/access_config.go deleted file mode 100644 index e0f962c50..000000000 --- a/builder/openstack-new/access_config.go +++ /dev/null @@ -1,109 +0,0 @@ -package openstack - -import ( - "crypto/tls" - "fmt" - "net/http" - "os" - - "github.com/mitchellh/packer/template/interpolate" - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack" -) - -// AccessConfig is for common configuration related to openstack access -type AccessConfig struct { - Username string `mapstructure:"username"` - UserID string `mapstructure:"user_id"` - Password string `mapstructure:"password"` - APIKey string `mapstructure:"api_key"` - IdentityEndpoint string `mapstructure:"identity_endpoint"` - TenantID string `mapstructure:"tenant_id"` - TenantName string `mapstructure:"tenant_name"` - DomainID string `mapstructure:"domain_id"` - DomainName string `mapstructure:"domain_name"` - Insecure bool `mapstructure:"insecure"` - Region string `mapstructure:"region"` - EndpointType string `mapstructure:"endpoint_type"` - - osClient *gophercloud.ProviderClient -} - -func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { - if c.EndpointType != "internal" && c.EndpointType != "internalURL" && - c.EndpointType != "admin" && c.EndpointType != "adminURL" && - c.EndpointType != "public" && c.EndpointType != "publicURL" && - c.EndpointType != "" { - return []error{fmt.Errorf("Invalid endpoint type provided")} - } - - if c.Region == "" { - c.Region = os.Getenv("OS_REGION_NAME") - } - - // Get as much as possible from the end - ao, err := openstack.AuthOptionsFromEnv() - if err != nil { - return []error{err} - } - - // Override values if we have them in our config - overrides := []struct { - From, To *string - }{ - {&c.Username, &ao.Username}, - {&c.UserID, &ao.UserID}, - {&c.Password, &ao.Password}, - {&c.APIKey, &ao.APIKey}, - {&c.IdentityEndpoint, &ao.IdentityEndpoint}, - {&c.TenantID, &ao.TenantID}, - {&c.TenantName, &ao.TenantName}, - {&c.DomainID, &ao.DomainID}, - {&c.DomainName, &ao.DomainName}, - } - for _, s := range overrides { - if *s.From != "" { - *s.To = *s.From - } - } - - // Build the client itself - client, err := openstack.NewClient(ao.IdentityEndpoint) - if err != nil { - return []error{err} - } - - // If we have insecure set, then create a custom HTTP client that - // ignores SSL errors. - if c.Insecure { - config := &tls.Config{InsecureSkipVerify: true} - transport := &http.Transport{TLSClientConfig: config} - client.HTTPClient.Transport = transport - } - - // Auth - err = openstack.Authenticate(client, ao) - if err != nil { - return []error{err} - } - - c.osClient = client - return nil -} - -func (c *AccessConfig) computeV2Client() (*gophercloud.ServiceClient, error) { - return openstack.NewComputeV2(c.osClient, gophercloud.EndpointOpts{ - Region: c.Region, - Availability: c.getEndpointType(), - }) -} - -func (c *AccessConfig) getEndpointType() gophercloud.Availability { - if c.EndpointType == "internal" || c.EndpointType == "internalURL" { - return gophercloud.AvailabilityInternal - } - if c.EndpointType == "admin" || c.EndpointType == "adminURL" { - return gophercloud.AvailabilityAdmin - } - return gophercloud.AvailabilityPublic -} diff --git a/builder/openstack-new/artifact.go b/builder/openstack-new/artifact.go deleted file mode 100644 index aa60d2641..000000000 --- a/builder/openstack-new/artifact.go +++ /dev/null @@ -1,47 +0,0 @@ -package openstack - -import ( - "fmt" - "log" - - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack/compute/v2/images" -) - -// Artifact is an artifact implementation that contains built images. -type Artifact struct { - // ImageId of built image - ImageId string - - // BuilderId is the unique ID for the builder that created this image - BuilderIdValue string - - // OpenStack connection for performing API stuff. - Client *gophercloud.ServiceClient -} - -func (a *Artifact) BuilderId() string { - return a.BuilderIdValue -} - -func (*Artifact) Files() []string { - // We have no files - return nil -} - -func (a *Artifact) Id() string { - return a.ImageId -} - -func (a *Artifact) String() string { - return fmt.Sprintf("An image was created: %v", a.ImageId) -} - -func (a *Artifact) State(name string) interface{} { - return nil -} - -func (a *Artifact) Destroy() error { - log.Printf("Destroying image: %s", a.ImageId) - return images.Delete(a.Client, a.ImageId).ExtractErr() -} diff --git a/builder/openstack-new/artifact_test.go b/builder/openstack-new/artifact_test.go deleted file mode 100644 index 313fea7cf..000000000 --- a/builder/openstack-new/artifact_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package openstack - -import ( - "github.com/mitchellh/packer/packer" - "testing" -) - -func TestArtifact_Impl(t *testing.T) { - var _ packer.Artifact = new(Artifact) -} - -func TestArtifactId(t *testing.T) { - expected := `b8cdf55b-c916-40bd-b190-389ec144c4ed` - - a := &Artifact{ - ImageId: "b8cdf55b-c916-40bd-b190-389ec144c4ed", - } - - result := a.Id() - if result != expected { - t.Fatalf("bad: %s", result) - } -} - -func TestArtifactString(t *testing.T) { - expected := "An image was created: b8cdf55b-c916-40bd-b190-389ec144c4ed" - - a := &Artifact{ - ImageId: "b8cdf55b-c916-40bd-b190-389ec144c4ed", - } - result := a.String() - if result != expected { - t.Fatalf("bad: %s", result) - } -} diff --git a/builder/openstack-new/builder.go b/builder/openstack-new/builder.go deleted file mode 100644 index bebb28452..000000000 --- a/builder/openstack-new/builder.go +++ /dev/null @@ -1,134 +0,0 @@ -// The openstack package contains a packer.Builder implementation that -// builds Images for openstack. - -package openstack - -import ( - "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/common" - "log" - - "github.com/mitchellh/packer/helper/config" - "github.com/mitchellh/packer/packer" - "github.com/mitchellh/packer/template/interpolate" -) - -// The unique ID for this builder -const BuilderId = "mitchellh.openstack" - -type Config struct { - common.PackerConfig `mapstructure:",squash"` - AccessConfig `mapstructure:",squash"` - ImageConfig `mapstructure:",squash"` - RunConfig `mapstructure:",squash"` - - ctx interpolate.Context -} - -type Builder struct { - config Config - runner multistep.Runner -} - -func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { - err := config.Decode(&b.config, &config.DecodeOpts{ - Interpolate: true, - }, raws...) - if err != nil { - return nil, err - } - - // Accumulate any errors - var errs *packer.MultiError - errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) - errs = packer.MultiErrorAppend(errs, b.config.ImageConfig.Prepare(&b.config.ctx)...) - errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...) - - if errs != nil && len(errs.Errors) > 0 { - return nil, errs - } - - log.Println(common.ScrubConfig(b.config, b.config.Password)) - return nil, nil -} - -func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { - computeClient, err := b.config.computeV2Client() - if err != nil { - return nil, fmt.Errorf("Error initializing compute client: %s", err) - } - - // Setup the state bag and initial state for the steps - state := new(multistep.BasicStateBag) - state.Put("config", b.config) - state.Put("hook", hook) - state.Put("ui", ui) - - // Build the steps - steps := []multistep.Step{ - &StepKeyPair{ - Debug: b.config.PackerDebug, - DebugKeyPath: fmt.Sprintf("os_%s.pem", b.config.PackerBuildName), - }, - &StepRunSourceServer{ - Name: b.config.ImageName, - Flavor: b.config.Flavor, - SourceImage: b.config.SourceImage, - SecurityGroups: b.config.SecurityGroups, - Networks: b.config.Networks, - }, - &StepWaitForRackConnect{ - Wait: b.config.RackconnectWait, - }, - &StepAllocateIp{ - FloatingIpPool: b.config.FloatingIpPool, - FloatingIp: b.config.FloatingIp, - }, - &common.StepConnectSSH{ - SSHAddress: SSHAddress(computeClient, b.config.SSHInterface, b.config.SSHPort), - SSHConfig: SSHConfig(b.config.SSHUsername), - SSHWaitTimeout: b.config.SSHTimeout(), - }, - &common.StepProvision{}, - &stepCreateImage{}, - } - - // Run! - if b.config.PackerDebug { - b.runner = &multistep.DebugRunner{ - Steps: steps, - PauseFn: common.MultistepDebugFn(ui), - } - } else { - b.runner = &multistep.BasicRunner{Steps: steps} - } - - b.runner.Run(state) - - // If there was an error, return that - if rawErr, ok := state.GetOk("error"); ok { - return nil, rawErr.(error) - } - - // If there are no images, then just return - if _, ok := state.GetOk("image"); !ok { - return nil, nil - } - - // Build the artifact and return it - artifact := &Artifact{ - ImageId: state.Get("image").(string), - BuilderIdValue: BuilderId, - Client: computeClient, - } - - return artifact, nil -} - -func (b *Builder) Cancel() { - if b.runner != nil { - log.Println("Cancelling the step runner...") - b.runner.Cancel() - } -} diff --git a/builder/openstack-new/builder_test.go b/builder/openstack-new/builder_test.go deleted file mode 100644 index badf9784d..000000000 --- a/builder/openstack-new/builder_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package openstack - -import ( - "github.com/mitchellh/packer/packer" - "testing" -) - -func testConfig() map[string]interface{} { - return map[string]interface{}{ - "username": "foo", - "password": "bar", - "provider": "foo", - "region": "DFW", - "image_name": "foo", - "source_image": "foo", - "flavor": "foo", - "ssh_username": "root", - } -} - -func TestBuilder_ImplementsBuilder(t *testing.T) { - var raw interface{} - raw = &Builder{} - if _, ok := raw.(packer.Builder); !ok { - t.Fatalf("Builder should be a builder") - } -} - -func TestBuilder_Prepare_BadType(t *testing.T) { - b := &Builder{} - c := map[string]interface{}{ - "password": []string{}, - } - - warns, err := b.Prepare(c) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatalf("prepare should fail") - } -} - -func TestBuilderPrepare_ImageName(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["image_name"] = "foo" - warns, err := b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - // Test bad - config["image_name"] = "foo {{" - b = Builder{} - warns, err = b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } - - // Test bad - delete(config, "image_name") - b = Builder{} - warns, err = b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } -} - -func TestBuilderPrepare_InvalidKey(t *testing.T) { - var b Builder - config := testConfig() - - // Add a random key - config["i_should_not_be_valid"] = true - warns, err := b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } -} diff --git a/builder/openstack-new/image_config.go b/builder/openstack-new/image_config.go deleted file mode 100644 index 124449eab..000000000 --- a/builder/openstack-new/image_config.go +++ /dev/null @@ -1,25 +0,0 @@ -package openstack - -import ( - "fmt" - - "github.com/mitchellh/packer/template/interpolate" -) - -// ImageConfig is for common configuration related to creating Images. -type ImageConfig struct { - ImageName string `mapstructure:"image_name"` -} - -func (c *ImageConfig) Prepare(ctx *interpolate.Context) []error { - errs := make([]error, 0) - if c.ImageName == "" { - errs = append(errs, fmt.Errorf("An image_name must be specified")) - } - - if len(errs) > 0 { - return errs - } - - return nil -} diff --git a/builder/openstack-new/image_config_test.go b/builder/openstack-new/image_config_test.go deleted file mode 100644 index 4d81ecd94..000000000 --- a/builder/openstack-new/image_config_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package openstack - -import ( - "testing" -) - -func testImageConfig() *ImageConfig { - return &ImageConfig{ - ImageName: "foo", - } -} - -func TestImageConfigPrepare_Region(t *testing.T) { - c := testImageConfig() - if err := c.Prepare(nil); err != nil { - t.Fatalf("shouldn't have err: %s", err) - } - - c.ImageName = "" - if err := c.Prepare(nil); err == nil { - t.Fatal("should have error") - } -} diff --git a/builder/openstack-new/run_config.go b/builder/openstack-new/run_config.go deleted file mode 100644 index e5d73c9c1..000000000 --- a/builder/openstack-new/run_config.go +++ /dev/null @@ -1,75 +0,0 @@ -package openstack - -import ( - "errors" - "fmt" - "time" - - "github.com/mitchellh/packer/template/interpolate" -) - -// RunConfig contains configuration for running an instance from a source -// image and details on how to access that launched image. -type RunConfig struct { - SourceImage string `mapstructure:"source_image"` - Flavor string `mapstructure:"flavor"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort int `mapstructure:"ssh_port"` - SSHInterface string `mapstructure:"ssh_interface"` - OpenstackProvider string `mapstructure:"openstack_provider"` - UseFloatingIp bool `mapstructure:"use_floating_ip"` - RackconnectWait bool `mapstructure:"rackconnect_wait"` - FloatingIpPool string `mapstructure:"floating_ip_pool"` - FloatingIp string `mapstructure:"floating_ip"` - SecurityGroups []string `mapstructure:"security_groups"` - Networks []string `mapstructure:"networks"` - - // Unexported fields that are calculated from others - sshTimeout time.Duration -} - -func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { - // Defaults - if c.SSHUsername == "" { - c.SSHUsername = "root" - } - - if c.SSHPort == 0 { - c.SSHPort = 22 - } - - if c.RawSSHTimeout == "" { - c.RawSSHTimeout = "5m" - } - - if c.UseFloatingIp && c.FloatingIpPool == "" { - c.FloatingIpPool = "public" - } - - // Validation - var err error - errs := make([]error, 0) - if c.SourceImage == "" { - errs = append(errs, errors.New("A source_image must be specified")) - } - - if c.Flavor == "" { - errs = append(errs, errors.New("A flavor must be specified")) - } - - if c.SSHUsername == "" { - errs = append(errs, errors.New("An ssh_username must be specified")) - } - - c.sshTimeout, err = time.ParseDuration(c.RawSSHTimeout) - if err != nil { - errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - - return errs -} - -func (c *RunConfig) SSHTimeout() time.Duration { - return c.sshTimeout -} diff --git a/builder/openstack-new/run_config_test.go b/builder/openstack-new/run_config_test.go deleted file mode 100644 index 16b89b352..000000000 --- a/builder/openstack-new/run_config_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package openstack - -import ( - "os" - "testing" -) - -func init() { - // Clear out the openstack env vars so they don't - // affect our tests. - os.Setenv("SDK_USERNAME", "") - os.Setenv("SDK_PASSWORD", "") - os.Setenv("SDK_PROVIDER", "") -} - -func testRunConfig() *RunConfig { - return &RunConfig{ - SourceImage: "abcd", - Flavor: "m1.small", - SSHUsername: "root", - } -} - -func TestRunConfigPrepare(t *testing.T) { - c := testRunConfig() - err := c.Prepare(nil) - if len(err) > 0 { - t.Fatalf("err: %s", err) - } -} - -func TestRunConfigPrepare_InstanceType(t *testing.T) { - c := testRunConfig() - c.Flavor = "" - if err := c.Prepare(nil); len(err) != 1 { - t.Fatalf("err: %s", err) - } -} - -func TestRunConfigPrepare_SourceImage(t *testing.T) { - c := testRunConfig() - c.SourceImage = "" - if err := c.Prepare(nil); len(err) != 1 { - t.Fatalf("err: %s", err) - } -} - -func TestRunConfigPrepare_SSHPort(t *testing.T) { - c := testRunConfig() - c.SSHPort = 0 - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } - - if c.SSHPort != 22 { - t.Fatalf("invalid value: %d", c.SSHPort) - } - - c.SSHPort = 44 - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } - - if c.SSHPort != 44 { - t.Fatalf("invalid value: %d", c.SSHPort) - } -} - -func TestRunConfigPrepare_SSHTimeout(t *testing.T) { - c := testRunConfig() - c.RawSSHTimeout = "" - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } - - c.RawSSHTimeout = "bad" - if err := c.Prepare(nil); len(err) != 1 { - t.Fatalf("err: %s", err) - } -} - -func TestRunConfigPrepare_SSHUsername(t *testing.T) { - c := testRunConfig() - c.SSHUsername = "" - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } -} diff --git a/builder/openstack-new/server.go b/builder/openstack-new/server.go deleted file mode 100644 index de8c9d103..000000000 --- a/builder/openstack-new/server.go +++ /dev/null @@ -1,95 +0,0 @@ -package openstack - -import ( - "errors" - "fmt" - "log" - "time" - - "github.com/mitchellh/multistep" - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -// StateRefreshFunc is a function type used for StateChangeConf that is -// responsible for refreshing the item being watched for a state change. -// -// It returns three results. `result` is any object that will be returned -// as the final object after waiting for state change. This allows you to -// return the final updated object, for example an openstack instance after -// refreshing it. -// -// `state` is the latest state of that object. And `err` is any error that -// may have happened while refreshing the state. -type StateRefreshFunc func() (result interface{}, state string, progress int, err error) - -// StateChangeConf is the configuration struct used for `WaitForState`. -type StateChangeConf struct { - Pending []string - Refresh StateRefreshFunc - StepState multistep.StateBag - Target string -} - -// ServerStateRefreshFunc returns a StateRefreshFunc that is used to watch -// an openstack server. -func ServerStateRefreshFunc( - client *gophercloud.ServiceClient, s *servers.Server) StateRefreshFunc { - return func() (interface{}, string, int, error) { - serverNew, err := servers.Get(client, s.ID).Extract() - if err != nil { - errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) - if ok && errCode.Actual == 404 { - log.Printf("[INFO] 404 on ServerStateRefresh, returning DELETED") - return nil, "DELETED", 0, nil - } else { - log.Printf("[ERROR] Error on ServerStateRefresh: %s", err) - return nil, "", 0, err - } - } - - return serverNew, serverNew.Status, serverNew.Progress, nil - } -} - -// WaitForState watches an object and waits for it to achieve a certain -// state. -func WaitForState(conf *StateChangeConf) (i interface{}, err error) { - log.Printf("Waiting for state to become: %s", conf.Target) - - for { - var currentProgress int - var currentState string - i, currentState, currentProgress, err = conf.Refresh() - if err != nil { - return - } - - if currentState == conf.Target { - return - } - - if conf.StepState != nil { - if _, ok := conf.StepState.GetOk(multistep.StateCancelled); ok { - return nil, errors.New("interrupted") - } - } - - found := false - for _, allowed := range conf.Pending { - if currentState == allowed { - found = true - break - } - } - - if !found { - return nil, fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target) - } - - log.Printf("Waiting for state to become: %s currently %s (%d%%)", conf.Target, currentState, currentProgress) - time.Sleep(2 * time.Second) - } - - return -} diff --git a/builder/openstack-new/ssh.go b/builder/openstack-new/ssh.go deleted file mode 100644 index 7b0510f98..000000000 --- a/builder/openstack-new/ssh.go +++ /dev/null @@ -1,92 +0,0 @@ -package openstack - -import ( - "errors" - "fmt" - "log" - "time" - - "github.com/mitchellh/multistep" - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" - "golang.org/x/crypto/ssh" -) - -// SSHAddress returns a function that can be given to the SSH communicator -// for determining the SSH address based on the server AccessIPv4 setting.. -func SSHAddress( - client *gophercloud.ServiceClient, - sshinterface string, port int) func(multistep.StateBag) (string, error) { - return func(state multistep.StateBag) (string, error) { - s := state.Get("server").(*servers.Server) - - // If we have a floating IP, use that - ip := state.Get("access_ip").(*floatingip.FloatingIP) - if ip != nil && ip.FixedIP != "" { - return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil - } - - if s.AccessIPv4 != "" { - return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil - } - - // Get all the addresses associated with this server. This - // was taken directly from Terraform. - for _, networkAddresses := range s.Addresses { - elements, ok := networkAddresses.([]interface{}) - if !ok { - log.Printf( - "[ERROR] Unknown return type for address field: %#v", - networkAddresses) - continue - } - - for _, element := range elements { - var addr string - address := element.(map[string]interface{}) - if address["OS-EXT-IPS:type"] == "floating" { - addr = address["addr"].(string) - } else { - if address["version"].(float64) == 4 { - addr = address["addr"].(string) - } - } - if addr != "" { - return fmt.Sprintf("%s:%d", addr, port), nil - } - } - } - - s, err := servers.Get(client, s.ID).Extract() - if err != nil { - return "", err - } - - state.Put("server", s) - time.Sleep(1 * time.Second) - - return "", errors.New("couldn't determine IP address for server") - } -} - -// SSHConfig returns a function that can be used for the SSH communicator -// config for connecting to the instance created over SSH using the generated -// private key. -func SSHConfig(username string) func(multistep.StateBag) (*ssh.ClientConfig, error) { - return func(state multistep.StateBag) (*ssh.ClientConfig, error) { - privateKey := state.Get("privateKey").(string) - - signer, err := ssh.ParsePrivateKey([]byte(privateKey)) - if err != nil { - return nil, fmt.Errorf("Error setting up SSH config: %s", err) - } - - return &ssh.ClientConfig{ - User: username, - Auth: []ssh.AuthMethod{ - ssh.PublicKeys(signer), - }, - }, nil - } -} diff --git a/builder/openstack-new/step_allocate_ip.go b/builder/openstack-new/step_allocate_ip.go deleted file mode 100644 index 16efe8d38..000000000 --- a/builder/openstack-new/step_allocate_ip.go +++ /dev/null @@ -1,93 +0,0 @@ -package openstack - -import ( - "fmt" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -type StepAllocateIp struct { - FloatingIpPool string - FloatingIp string -} - -func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { - ui := state.Get("ui").(packer.Ui) - config := state.Get("config").(Config) - server := state.Get("server").(*servers.Server) - - // We need the v2 compute client - client, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - var instanceIp *floatingip.FloatingIP - // This is here in case we error out before putting instanceIp into the - // statebag below, because it is requested by Cleanup() - state.Put("access_ip", instanceIp) - - if s.FloatingIp != "" { - *instanceIp = floatingip.FloatingIP{FixedIP: s.FloatingIp} - } else if s.FloatingIpPool != "" { - newIp, err := floatingip.Create(client, floatingip.CreateOpts{ - Pool: s.FloatingIpPool, - }).Extract() - if err != nil { - err := fmt.Errorf("Error creating floating ip from pool '%s'", s.FloatingIpPool) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - *instanceIp = *newIp - ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) - } - - if instanceIp != nil && instanceIp.FixedIP != "" { - err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() - if err != nil { - err := fmt.Errorf( - "Error associating floating IP %s with instance.", - instanceIp.FixedIP) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - ui.Say(fmt.Sprintf( - "Added floating IP %s to instance...", instanceIp.FixedIP)) - } - - state.Put("access_ip", instanceIp) - return multistep.ActionContinue -} - -func (s *StepAllocateIp) Cleanup(state multistep.StateBag) { - config := state.Get("config").(Config) - ui := state.Get("ui").(packer.Ui) - instanceIp := state.Get("access_ip").(*floatingip.FloatingIP) - - // We need the v2 compute client - client, err := config.computeV2Client() - if err != nil { - ui.Error(fmt.Sprintf( - "Error deleting temporary floating IP %s", instanceIp.FixedIP)) - return - } - - if s.FloatingIpPool != "" && instanceIp.ID != "" { - if err := floatingip.Delete(client, instanceIp.ID).ExtractErr(); err != nil { - ui.Error(fmt.Sprintf( - "Error deleting temporary floating IP %s", instanceIp.FixedIP)) - return - } - - ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.FixedIP)) - } -} diff --git a/builder/openstack-new/step_create_image.go b/builder/openstack-new/step_create_image.go deleted file mode 100644 index b777e8b0b..000000000 --- a/builder/openstack-new/step_create_image.go +++ /dev/null @@ -1,84 +0,0 @@ -package openstack - -import ( - "fmt" - "log" - "time" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack/compute/v2/images" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -type stepCreateImage struct{} - -func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { - config := state.Get("config").(Config) - server := state.Get("server").(*servers.Server) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - client, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - // Create the image - ui.Say(fmt.Sprintf("Creating the image: %s", config.ImageName)) - imageId, err := servers.CreateImage(client, server.ID, servers.CreateImageOpts{ - Name: config.ImageName, - }).ExtractImageID() - if err != nil { - err := fmt.Errorf("Error creating image: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - // Set the Image ID in the state - ui.Message(fmt.Sprintf("Image: %s", imageId)) - state.Put("image", imageId) - - // Wait for the image to become ready - ui.Say("Waiting for image to become ready...") - if err := WaitForImage(client, imageId); err != nil { - err := fmt.Errorf("Error waiting for image: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - return multistep.ActionContinue -} - -func (s *stepCreateImage) Cleanup(multistep.StateBag) { - // No cleanup... -} - -// WaitForImage waits for the given Image ID to become ready. -func WaitForImage(client *gophercloud.ServiceClient, imageId string) error { - for { - image, err := images.Get(client, imageId).Extract() - if err != nil { - errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) - if ok && errCode.Actual == 500 { - log.Printf("[ERROR] 500 error received, will ignore and retry: %s", err) - time.Sleep(2 * time.Second) - continue - } - - return err - } - - if image.Status == "ACTIVE" { - return nil - } - - log.Printf("Waiting for image creation status: %s (%d%%)", image.Status, image.Progress) - time.Sleep(2 * time.Second) - } -} diff --git a/builder/openstack-new/step_key_pair.go b/builder/openstack-new/step_key_pair.go deleted file mode 100644 index 06bcbf9ea..000000000 --- a/builder/openstack-new/step_key_pair.go +++ /dev/null @@ -1,106 +0,0 @@ -package openstack - -import ( - "fmt" - "os" - "runtime" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/common/uuid" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" -) - -type StepKeyPair struct { - Debug bool - DebugKeyPath string - keyName string -} - -func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { - config := state.Get("config").(Config) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - ui.Say("Creating temporary keypair for this instance...") - keyName := fmt.Sprintf("packer %s", uuid.TimeOrderedUUID()) - keypair, err := keypairs.Create(computeClient, keypairs.CreateOpts{ - Name: keyName, - }).Extract() - if err != nil { - state.Put("error", fmt.Errorf("Error creating temporary keypair: %s", err)) - return multistep.ActionHalt - } - - if keypair.PrivateKey == "" { - state.Put("error", fmt.Errorf("The temporary keypair returned was blank")) - return multistep.ActionHalt - } - - // If we're in debug mode, output the private key to the working - // directory. - if s.Debug { - ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath)) - f, err := os.Create(s.DebugKeyPath) - if err != nil { - state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) - return multistep.ActionHalt - } - defer f.Close() - - // Write the key out - if _, err := f.Write([]byte(keypair.PrivateKey)); err != nil { - state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) - return multistep.ActionHalt - } - - // Chmod it so that it is SSH ready - if runtime.GOOS != "windows" { - if err := f.Chmod(0600); err != nil { - state.Put("error", fmt.Errorf("Error setting permissions of debug key: %s", err)) - return multistep.ActionHalt - } - } - } - - // Set the keyname so we know to delete it later - s.keyName = keyName - - // Set some state data for use in future steps - state.Put("keyPair", keyName) - state.Put("privateKey", keypair.PrivateKey) - - return multistep.ActionContinue -} - -func (s *StepKeyPair) Cleanup(state multistep.StateBag) { - // If no key name is set, then we never created it, so just return - if s.keyName == "" { - return - } - - config := state.Get("config").(Config) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - ui.Error(fmt.Sprintf( - "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) - return - } - - ui.Say("Deleting temporary keypair...") - err = keypairs.Delete(computeClient, s.keyName).ExtractErr() - if err != nil { - ui.Error(fmt.Sprintf( - "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) - } -} diff --git a/builder/openstack-new/step_run_source_server.go b/builder/openstack-new/step_run_source_server.go deleted file mode 100644 index 4432d5860..000000000 --- a/builder/openstack-new/step_run_source_server.go +++ /dev/null @@ -1,110 +0,0 @@ -package openstack - -import ( - "fmt" - "log" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -type StepRunSourceServer struct { - Flavor string - Name string - SourceImage string - SecurityGroups []string - Networks []string - - server *servers.Server -} - -func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction { - config := state.Get("config").(Config) - keyName := state.Get("keyPair").(string) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - networks := make([]servers.Network, len(s.Networks)) - for i, networkUuid := range s.Networks { - networks[i].UUID = networkUuid - } - - s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ - CreateOptsBuilder: servers.CreateOpts{ - Name: s.Name, - ImageRef: s.SourceImage, - FlavorName: s.Flavor, - SecurityGroups: s.SecurityGroups, - Networks: networks, - }, - - KeyName: keyName, - }).Extract() - if err != nil { - err := fmt.Errorf("Error launching source server: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - log.Printf("server id: %s", s.server.ID) - - ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.ID)) - stateChange := StateChangeConf{ - Pending: []string{"BUILD"}, - Target: "ACTIVE", - Refresh: ServerStateRefreshFunc(computeClient, s.server), - StepState: state, - } - latestServer, err := WaitForState(&stateChange) - if err != nil { - err := fmt.Errorf("Error waiting for server (%s) to become ready: %s", s.server.ID, err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - s.server = latestServer.(*servers.Server) - state.Put("server", s.server) - - return multistep.ActionContinue -} - -func (s *StepRunSourceServer) Cleanup(state multistep.StateBag) { - if s.server == nil { - return - } - - config := state.Get("config").(Config) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) - return - } - - ui.Say("Terminating the source server...") - if err := servers.Delete(computeClient, s.server.ID).ExtractErr(); err != nil { - ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) - return - } - - stateChange := StateChangeConf{ - Pending: []string{"ACTIVE", "BUILD", "REBUILD", "SUSPENDED"}, - Refresh: ServerStateRefreshFunc(computeClient, s.server), - Target: "DELETED", - } - - WaitForState(&stateChange) -} diff --git a/builder/openstack-new/step_wait_for_rackconnect.go b/builder/openstack-new/step_wait_for_rackconnect.go deleted file mode 100644 index 6263bd17d..000000000 --- a/builder/openstack-new/step_wait_for_rackconnect.go +++ /dev/null @@ -1,52 +0,0 @@ -package openstack - -import ( - "fmt" - "time" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -type StepWaitForRackConnect struct { - Wait bool -} - -func (s *StepWaitForRackConnect) Run(state multistep.StateBag) multistep.StepAction { - if !s.Wait { - return multistep.ActionContinue - } - - config := state.Get("config").(Config) - server := state.Get("server").(*servers.Server) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - ui.Say(fmt.Sprintf( - "Waiting for server (%s) to become RackConnect ready...", server.ID)) - for { - server, err = servers.Get(computeClient, server.ID).Extract() - if err != nil { - return multistep.ActionHalt - } - - if server.Metadata["rackconnect_automation_status"] == "DEPLOYED" { - break - } - - time.Sleep(2 * time.Second) - } - - return multistep.ActionContinue -} - -func (s *StepWaitForRackConnect) Cleanup(state multistep.StateBag) { -} diff --git a/builder/openstack/access_config.go b/builder/openstack/access_config.go index cb1c9d7bd..e0f962c50 100644 --- a/builder/openstack/access_config.go +++ b/builder/openstack/access_config.go @@ -4,99 +4,106 @@ import ( "crypto/tls" "fmt" "net/http" - "net/url" "os" - "strings" - "github.com/mitchellh/gophercloud-fork-40444fb" - "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/template/interpolate" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack" ) // AccessConfig is for common configuration related to openstack access type AccessConfig struct { - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - ApiKey string `mapstructure:"api_key"` - Project string `mapstructure:"project"` - Provider string `mapstructure:"provider"` - RawRegion string `mapstructure:"region"` - ProxyUrl string `mapstructure:"proxy_url"` - TenantId string `mapstructure:"tenant_id"` - Insecure bool `mapstructure:"insecure"` -} + Username string `mapstructure:"username"` + UserID string `mapstructure:"user_id"` + Password string `mapstructure:"password"` + APIKey string `mapstructure:"api_key"` + IdentityEndpoint string `mapstructure:"identity_endpoint"` + TenantID string `mapstructure:"tenant_id"` + TenantName string `mapstructure:"tenant_name"` + DomainID string `mapstructure:"domain_id"` + DomainName string `mapstructure:"domain_name"` + Insecure bool `mapstructure:"insecure"` + Region string `mapstructure:"region"` + EndpointType string `mapstructure:"endpoint_type"` -// Auth returns a valid Auth object for access to openstack services, or -// an error if the authentication couldn't be resolved. -func (c *AccessConfig) Auth() (gophercloud.AccessProvider, error) { - c.Username = common.ChooseString(c.Username, os.Getenv("SDK_USERNAME"), os.Getenv("OS_USERNAME")) - c.Password = common.ChooseString(c.Password, os.Getenv("SDK_PASSWORD"), os.Getenv("OS_PASSWORD")) - c.ApiKey = common.ChooseString(c.ApiKey, os.Getenv("SDK_API_KEY")) - c.Project = common.ChooseString(c.Project, os.Getenv("SDK_PROJECT"), os.Getenv("OS_TENANT_NAME")) - c.Provider = common.ChooseString(c.Provider, os.Getenv("SDK_PROVIDER"), os.Getenv("OS_AUTH_URL")) - c.RawRegion = common.ChooseString(c.RawRegion, os.Getenv("SDK_REGION"), os.Getenv("OS_REGION_NAME")) - c.TenantId = common.ChooseString(c.TenantId, os.Getenv("OS_TENANT_ID")) - - // OpenStack's auto-generated openrc.sh files do not append the suffix - // /tokens to the authentication URL. This ensures it is present when - // specifying the URL. - if strings.Contains(c.Provider, "://") && !strings.HasSuffix(c.Provider, "/tokens") { - c.Provider += "/tokens" - } - - authoptions := gophercloud.AuthOptions{ - AllowReauth: true, - - ApiKey: c.ApiKey, - TenantId: c.TenantId, - TenantName: c.Project, - Username: c.Username, - Password: c.Password, - } - - default_transport := &http.Transport{} - - if c.Insecure { - cfg := new(tls.Config) - cfg.InsecureSkipVerify = true - default_transport.TLSClientConfig = cfg - } - - // For corporate networks it may be the case where we want our API calls - // to be sent through a separate HTTP proxy than external traffic. - if c.ProxyUrl != "" { - url, err := url.Parse(c.ProxyUrl) - if err != nil { - return nil, err - } - - // The gophercloud.Context has a UseCustomClient method which - // would allow us to override with a new instance of http.Client. - default_transport.Proxy = http.ProxyURL(url) - } - - if c.Insecure || c.ProxyUrl != "" { - http.DefaultTransport = default_transport - } - - return gophercloud.Authenticate(c.Provider, authoptions) -} - -func (c *AccessConfig) Region() string { - return common.ChooseString(c.RawRegion, os.Getenv("SDK_REGION"), os.Getenv("OS_REGION_NAME")) + osClient *gophercloud.ProviderClient } func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { - errs := make([]error, 0) - if strings.HasPrefix(c.Provider, "rackspace") { - if c.Region() == "" { - errs = append(errs, fmt.Errorf("region must be specified when using rackspace")) + if c.EndpointType != "internal" && c.EndpointType != "internalURL" && + c.EndpointType != "admin" && c.EndpointType != "adminURL" && + c.EndpointType != "public" && c.EndpointType != "publicURL" && + c.EndpointType != "" { + return []error{fmt.Errorf("Invalid endpoint type provided")} + } + + if c.Region == "" { + c.Region = os.Getenv("OS_REGION_NAME") + } + + // Get as much as possible from the end + ao, err := openstack.AuthOptionsFromEnv() + if err != nil { + return []error{err} + } + + // Override values if we have them in our config + overrides := []struct { + From, To *string + }{ + {&c.Username, &ao.Username}, + {&c.UserID, &ao.UserID}, + {&c.Password, &ao.Password}, + {&c.APIKey, &ao.APIKey}, + {&c.IdentityEndpoint, &ao.IdentityEndpoint}, + {&c.TenantID, &ao.TenantID}, + {&c.TenantName, &ao.TenantName}, + {&c.DomainID, &ao.DomainID}, + {&c.DomainName, &ao.DomainName}, + } + for _, s := range overrides { + if *s.From != "" { + *s.To = *s.From } } - if len(errs) > 0 { - return errs + // Build the client itself + client, err := openstack.NewClient(ao.IdentityEndpoint) + if err != nil { + return []error{err} } + // If we have insecure set, then create a custom HTTP client that + // ignores SSL errors. + if c.Insecure { + config := &tls.Config{InsecureSkipVerify: true} + transport := &http.Transport{TLSClientConfig: config} + client.HTTPClient.Transport = transport + } + + // Auth + err = openstack.Authenticate(client, ao) + if err != nil { + return []error{err} + } + + c.osClient = client return nil } + +func (c *AccessConfig) computeV2Client() (*gophercloud.ServiceClient, error) { + return openstack.NewComputeV2(c.osClient, gophercloud.EndpointOpts{ + Region: c.Region, + Availability: c.getEndpointType(), + }) +} + +func (c *AccessConfig) getEndpointType() gophercloud.Availability { + if c.EndpointType == "internal" || c.EndpointType == "internalURL" { + return gophercloud.AvailabilityInternal + } + if c.EndpointType == "admin" || c.EndpointType == "adminURL" { + return gophercloud.AvailabilityAdmin + } + return gophercloud.AvailabilityPublic +} diff --git a/builder/openstack/access_config_test.go b/builder/openstack/access_config_test.go deleted file mode 100644 index cf37448cc..000000000 --- a/builder/openstack/access_config_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package openstack - -import ( - "os" - "testing" -) - -func init() { - // Clear out the openstack env vars so they don't - // affect our tests. - os.Setenv("SDK_REGION", "") - os.Setenv("OS_REGION_NAME", "") -} - -func testAccessConfig() *AccessConfig { - return &AccessConfig{} -} - -func TestAccessConfigPrepare_NoRegion_Rackspace(t *testing.T) { - c := testAccessConfig() - c.Provider = "rackspace-us" - if err := c.Prepare(nil); err == nil { - t.Fatalf("shouldn't have err: %s", err) - } -} - -func TestAccessConfigRegionWithEmptyEnv(t *testing.T) { - c := testAccessConfig() - c.Prepare(nil) - if c.Region() != "" { - t.Fatalf("Region should be empty") - } -} - -func TestAccessConfigRegionWithSdkRegionEnv(t *testing.T) { - c := testAccessConfig() - c.Prepare(nil) - - expectedRegion := "sdk_region" - os.Setenv("SDK_REGION", expectedRegion) - os.Setenv("OS_REGION_NAME", "") - if c.Region() != expectedRegion { - t.Fatalf("Region should be: %s", expectedRegion) - } -} - -func TestAccessConfigRegionWithOsRegionNameEnv(t *testing.T) { - c := testAccessConfig() - c.Prepare(nil) - - expectedRegion := "os_region_name" - os.Setenv("SDK_REGION", "") - os.Setenv("OS_REGION_NAME", expectedRegion) - if c.Region() != expectedRegion { - t.Fatalf("Region should be: %s", expectedRegion) - } -} - -func TestAccessConfigPrepare_NoRegion_PrivateCloud(t *testing.T) { - c := testAccessConfig() - c.Provider = "http://some-keystone-server:5000/v2.0" - if err := c.Prepare(nil); err != nil { - t.Fatalf("shouldn't have err: %s", err) - } -} - -func TestAccessConfigPrepare_Region(t *testing.T) { - dfw := "DFW" - c := testAccessConfig() - c.RawRegion = dfw - if err := c.Prepare(nil); err != nil { - t.Fatalf("shouldn't have err: %s", err) - } - if dfw != c.Region() { - t.Fatalf("Regions do not match: %s %s", dfw, c.Region()) - } -} diff --git a/builder/openstack/artifact.go b/builder/openstack/artifact.go index 6e75fad3e..aa60d2641 100644 --- a/builder/openstack/artifact.go +++ b/builder/openstack/artifact.go @@ -4,7 +4,8 @@ import ( "fmt" "log" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/images" ) // Artifact is an artifact implementation that contains built images. @@ -16,7 +17,7 @@ type Artifact struct { BuilderIdValue string // OpenStack connection for performing API stuff. - Conn gophercloud.CloudServersProvider + Client *gophercloud.ServiceClient } func (a *Artifact) BuilderId() string { @@ -42,5 +43,5 @@ func (a *Artifact) State(name string) interface{} { func (a *Artifact) Destroy() error { log.Printf("Destroying image: %s", a.ImageId) - return a.Conn.DeleteImageById(a.ImageId) + return images.Delete(a.Client, a.ImageId).ExtractErr() } diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index e6e5c6675..bebb28452 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -9,7 +9,6 @@ import ( "github.com/mitchellh/packer/common" "log" - "github.com/mitchellh/gophercloud-fork-40444fb" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -55,28 +54,14 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { } func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { - auth, err := b.config.AccessConfig.Auth() + computeClient, err := b.config.computeV2Client() if err != nil { - return nil, err - } - //fetches the api requisites from gophercloud for the appropriate - //openstack variant - api, err := gophercloud.PopulateApi(b.config.RunConfig.OpenstackProvider) - if err != nil { - return nil, err - } - api.Region = b.config.AccessConfig.Region() - - csp, err := gophercloud.ServersApi(auth, api) - if err != nil { - log.Printf("Region: %s", b.config.AccessConfig.Region()) - return nil, err + return nil, fmt.Errorf("Error initializing compute client: %s", err) } // Setup the state bag and initial state for the steps state := new(multistep.BasicStateBag) state.Put("config", b.config) - state.Put("csp", csp) state.Put("hook", hook) state.Put("ui", ui) @@ -101,7 +86,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe FloatingIp: b.config.FloatingIp, }, &common.StepConnectSSH{ - SSHAddress: SSHAddress(csp, b.config.SSHInterface, b.config.SSHPort), + SSHAddress: SSHAddress(computeClient, b.config.SSHInterface, b.config.SSHPort), SSHConfig: SSHConfig(b.config.SSHUsername), SSHWaitTimeout: b.config.SSHTimeout(), }, @@ -135,7 +120,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe artifact := &Artifact{ ImageId: state.Get("image").(string), BuilderIdValue: BuilderId, - Conn: csp, + Client: computeClient, } return artifact, nil diff --git a/builder/openstack/server.go b/builder/openstack/server.go index ba22dd3e2..de8c9d103 100644 --- a/builder/openstack/server.go +++ b/builder/openstack/server.go @@ -3,12 +3,12 @@ package openstack import ( "errors" "fmt" - "github.com/mitchellh/multistep" - "github.com/racker/perigee" "log" "time" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) // StateRefreshFunc is a function type used for StateChangeConf that is @@ -33,21 +33,22 @@ type StateChangeConf struct { // ServerStateRefreshFunc returns a StateRefreshFunc that is used to watch // an openstack server. -func ServerStateRefreshFunc(csp gophercloud.CloudServersProvider, s *gophercloud.Server) StateRefreshFunc { +func ServerStateRefreshFunc( + client *gophercloud.ServiceClient, s *servers.Server) StateRefreshFunc { return func() (interface{}, string, int, error) { - resp, err := csp.ServerById(s.Id) + serverNew, err := servers.Get(client, s.ID).Extract() if err != nil { - urce, ok := err.(*perigee.UnexpectedResponseCodeError) - if ok && (urce.Actual == 404) { - log.Printf("404 on ServerStateRefresh, returning DELETED") - + errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) + if ok && errCode.Actual == 404 { + log.Printf("[INFO] 404 on ServerStateRefresh, returning DELETED") return nil, "DELETED", 0, nil } else { - log.Printf("Error on ServerStateRefresh: %s", err) + log.Printf("[ERROR] Error on ServerStateRefresh: %s", err) return nil, "", 0, err } } - return resp, resp.Status, resp.Progress, nil + + return serverNew, serverNew.Status, serverNew.Progress, nil } } diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go index d20f24170..7b0510f98 100644 --- a/builder/openstack/ssh.go +++ b/builder/openstack/ssh.go @@ -3,49 +3,67 @@ package openstack import ( "errors" "fmt" - "github.com/mitchellh/multistep" - "golang.org/x/crypto/ssh" + "log" "time" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" + "golang.org/x/crypto/ssh" ) // SSHAddress returns a function that can be given to the SSH communicator // for determining the SSH address based on the server AccessIPv4 setting.. -func SSHAddress(csp gophercloud.CloudServersProvider, sshinterface string, port int) func(multistep.StateBag) (string, error) { +func SSHAddress( + client *gophercloud.ServiceClient, + sshinterface string, port int) func(multistep.StateBag) (string, error) { return func(state multistep.StateBag) (string, error) { - s := state.Get("server").(*gophercloud.Server) + s := state.Get("server").(*servers.Server) - if ip := state.Get("access_ip").(gophercloud.FloatingIp); ip.Ip != "" { - return fmt.Sprintf("%s:%d", ip.Ip, port), nil + // If we have a floating IP, use that + ip := state.Get("access_ip").(*floatingip.FloatingIP) + if ip != nil && ip.FixedIP != "" { + return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil } - ip_pools, err := s.AllAddressPools() - if err != nil { - return "", errors.New("Error parsing SSH addresses") + if s.AccessIPv4 != "" { + return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil } - for pool, addresses := range ip_pools { - if sshinterface != "" { - if pool != sshinterface { - continue - } + + // Get all the addresses associated with this server. This + // was taken directly from Terraform. + for _, networkAddresses := range s.Addresses { + elements, ok := networkAddresses.([]interface{}) + if !ok { + log.Printf( + "[ERROR] Unknown return type for address field: %#v", + networkAddresses) + continue } - if pool != "" { - for _, address := range addresses { - if address.Addr != "" && address.Version == 4 { - return fmt.Sprintf("%s:%d", address.Addr, port), nil + + for _, element := range elements { + var addr string + address := element.(map[string]interface{}) + if address["OS-EXT-IPS:type"] == "floating" { + addr = address["addr"].(string) + } else { + if address["version"].(float64) == 4 { + addr = address["addr"].(string) } } + if addr != "" { + return fmt.Sprintf("%s:%d", addr, port), nil + } } } - serverState, err := csp.ServerById(s.Id) - + s, err := servers.Get(client, s.ID).Extract() if err != nil { return "", err } - state.Put("server", serverState) + state.Put("server", s) time.Sleep(1 * time.Second) return "", errors.New("couldn't determine IP address for server") diff --git a/builder/openstack/step_allocate_ip.go b/builder/openstack/step_allocate_ip.go index b64f8b617..16efe8d38 100644 --- a/builder/openstack/step_allocate_ip.go +++ b/builder/openstack/step_allocate_ip.go @@ -2,10 +2,11 @@ package openstack import ( "fmt" + "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" - - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) type StepAllocateIp struct { @@ -15,53 +16,78 @@ type StepAllocateIp struct { func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) - csp := state.Get("csp").(gophercloud.CloudServersProvider) - server := state.Get("server").(*gophercloud.Server) + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) - var instanceIp gophercloud.FloatingIp + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + var instanceIp *floatingip.FloatingIP // This is here in case we error out before putting instanceIp into the // statebag below, because it is requested by Cleanup() state.Put("access_ip", instanceIp) if s.FloatingIp != "" { - instanceIp.Ip = s.FloatingIp + *instanceIp = floatingip.FloatingIP{FixedIP: s.FloatingIp} } else if s.FloatingIpPool != "" { - newIp, err := csp.CreateFloatingIp(s.FloatingIpPool) + newIp, err := floatingip.Create(client, floatingip.CreateOpts{ + Pool: s.FloatingIpPool, + }).Extract() if err != nil { err := fmt.Errorf("Error creating floating ip from pool '%s'", s.FloatingIpPool) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - instanceIp = newIp - ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.Ip)) + + *instanceIp = *newIp + ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) } - if instanceIp.Ip != "" { - if err := csp.AssociateFloatingIp(server.Id, instanceIp); err != nil { - err := fmt.Errorf("Error associating floating IP %s with instance.", instanceIp.Ip) + if instanceIp != nil && instanceIp.FixedIP != "" { + err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() + if err != nil { + err := fmt.Errorf( + "Error associating floating IP %s with instance.", + instanceIp.FixedIP) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt - } else { - ui.Say(fmt.Sprintf("Added floating IP %s to instance...", instanceIp.Ip)) } + + ui.Say(fmt.Sprintf( + "Added floating IP %s to instance...", instanceIp.FixedIP)) } state.Put("access_ip", instanceIp) - return multistep.ActionContinue } func (s *StepAllocateIp) Cleanup(state multistep.StateBag) { + config := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) - csp := state.Get("csp").(gophercloud.CloudServersProvider) - instanceIp := state.Get("access_ip").(gophercloud.FloatingIp) - if s.FloatingIpPool != "" && instanceIp.Id != 0 { - if err := csp.DeleteFloatingIp(instanceIp); err != nil { - ui.Error(fmt.Sprintf("Error deleting temporary floating IP %s", instanceIp.Ip)) + instanceIp := state.Get("access_ip").(*floatingip.FloatingIP) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf( + "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + return + } + + if s.FloatingIpPool != "" && instanceIp.ID != "" { + if err := floatingip.Delete(client, instanceIp.ID).ExtractErr(); err != nil { + ui.Error(fmt.Sprintf( + "Error deleting temporary floating IP %s", instanceIp.FixedIP)) return } - ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.Ip)) + + ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.FixedIP)) } } diff --git a/builder/openstack/step_create_image.go b/builder/openstack/step_create_image.go index 52a2ec4d1..b777e8b0b 100644 --- a/builder/openstack/step_create_image.go +++ b/builder/openstack/step_create_image.go @@ -2,28 +2,36 @@ package openstack import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" "log" "time" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/images" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) type stepCreateImage struct{} func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { - csp := state.Get("csp").(gophercloud.CloudServersProvider) config := state.Get("config").(Config) - server := state.Get("server").(*gophercloud.Server) + server := state.Get("server").(*servers.Server) ui := state.Get("ui").(packer.Ui) + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + // Create the image ui.Say(fmt.Sprintf("Creating the image: %s", config.ImageName)) - createOpts := gophercloud.CreateImage{ + imageId, err := servers.CreateImage(client, server.ID, servers.CreateImageOpts{ Name: config.ImageName, - } - imageId, err := csp.CreateImage(server.Id, createOpts) + }).ExtractImageID() if err != nil { err := fmt.Errorf("Error creating image: %s", err) state.Put("error", err) @@ -32,12 +40,12 @@ func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { } // Set the Image ID in the state - ui.Say(fmt.Sprintf("Image: %s", imageId)) + ui.Message(fmt.Sprintf("Image: %s", imageId)) state.Put("image", imageId) // Wait for the image to become ready ui.Say("Waiting for image to become ready...") - if err := WaitForImage(csp, imageId); err != nil { + if err := WaitForImage(client, imageId); err != nil { err := fmt.Errorf("Error waiting for image: %s", err) state.Put("error", err) ui.Error(err.Error()) @@ -52,10 +60,17 @@ func (s *stepCreateImage) Cleanup(multistep.StateBag) { } // WaitForImage waits for the given Image ID to become ready. -func WaitForImage(csp gophercloud.CloudServersProvider, imageId string) error { +func WaitForImage(client *gophercloud.ServiceClient, imageId string) error { for { - image, err := csp.ImageById(imageId) + image, err := images.Get(client, imageId).Extract() if err != nil { + errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) + if ok && errCode.Actual == 500 { + log.Printf("[ERROR] 500 error received, will ignore and retry: %s", err) + time.Sleep(2 * time.Second) + continue + } + return err } diff --git a/builder/openstack/step_key_pair.go b/builder/openstack/step_key_pair.go index 9c46b4377..06bcbf9ea 100644 --- a/builder/openstack/step_key_pair.go +++ b/builder/openstack/step_key_pair.go @@ -2,14 +2,13 @@ package openstack import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/common/uuid" - "github.com/mitchellh/packer/packer" - "log" "os" "runtime" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" ) type StepKeyPair struct { @@ -19,18 +18,28 @@ type StepKeyPair struct { } func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { - csp := state.Get("csp").(gophercloud.CloudServersProvider) + config := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + ui.Say("Creating temporary keypair for this instance...") keyName := fmt.Sprintf("packer %s", uuid.TimeOrderedUUID()) - log.Printf("temporary keypair name: %s", keyName) - keyResp, err := csp.CreateKeyPair(gophercloud.NewKeyPair{Name: keyName}) + keypair, err := keypairs.Create(computeClient, keypairs.CreateOpts{ + Name: keyName, + }).Extract() if err != nil { state.Put("error", fmt.Errorf("Error creating temporary keypair: %s", err)) return multistep.ActionHalt } - if keyResp.PrivateKey == "" { + + if keypair.PrivateKey == "" { state.Put("error", fmt.Errorf("The temporary keypair returned was blank")) return multistep.ActionHalt } @@ -47,7 +56,7 @@ func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { defer f.Close() // Write the key out - if _, err := f.Write([]byte(keyResp.PrivateKey)); err != nil { + if _, err := f.Write([]byte(keypair.PrivateKey)); err != nil { state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) return multistep.ActionHalt } @@ -66,7 +75,7 @@ func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { // Set some state data for use in future steps state.Put("keyPair", keyName) - state.Put("privateKey", keyResp.PrivateKey) + state.Put("privateKey", keypair.PrivateKey) return multistep.ActionContinue } @@ -77,11 +86,19 @@ func (s *StepKeyPair) Cleanup(state multistep.StateBag) { return } - csp := state.Get("csp").(gophercloud.CloudServersProvider) + config := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf( + "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) + return + } + ui.Say("Deleting temporary keypair...") - err := csp.DeleteKeyPair(s.keyName) + err = keypairs.Delete(computeClient, s.keyName).ExtractErr() if err != nil { ui.Error(fmt.Sprintf( "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) diff --git a/builder/openstack/step_run_source_server.go b/builder/openstack/step_run_source_server.go index 19e7f024d..4432d5860 100644 --- a/builder/openstack/step_run_source_server.go +++ b/builder/openstack/step_run_source_server.go @@ -2,11 +2,12 @@ package openstack import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" "log" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) type StepRunSourceServer struct { @@ -16,37 +17,38 @@ type StepRunSourceServer struct { SecurityGroups []string Networks []string - server *gophercloud.Server + server *servers.Server } func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction { - csp := state.Get("csp").(gophercloud.CloudServersProvider) + config := state.Get("config").(Config) keyName := state.Get("keyPair").(string) ui := state.Get("ui").(packer.Ui) - // XXX - validate image and flavor is available - - securityGroups := make([]map[string]interface{}, len(s.SecurityGroups)) - for i, groupName := range s.SecurityGroups { - securityGroups[i] = make(map[string]interface{}) - securityGroups[i]["name"] = groupName + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt } - networks := make([]gophercloud.NetworkConfig, len(s.Networks)) + networks := make([]servers.Network, len(s.Networks)) for i, networkUuid := range s.Networks { - networks[i].Uuid = networkUuid + networks[i].UUID = networkUuid } - server := gophercloud.NewServer{ - Name: s.Name, - ImageRef: s.SourceImage, - FlavorRef: s.Flavor, - KeyPairName: keyName, - SecurityGroup: securityGroups, - Networks: networks, - } + s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ + CreateOptsBuilder: servers.CreateOpts{ + Name: s.Name, + ImageRef: s.SourceImage, + FlavorName: s.Flavor, + SecurityGroups: s.SecurityGroups, + Networks: networks, + }, - serverResp, err := csp.CreateServer(server) + KeyName: keyName, + }).Extract() if err != nil { err := fmt.Errorf("Error launching source server: %s", err) state.Put("error", err) @@ -54,25 +56,24 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction return multistep.ActionHalt } - s.server, err = csp.ServerById(serverResp.Id) - log.Printf("server id: %s", s.server.Id) + log.Printf("server id: %s", s.server.ID) - ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.Id)) + ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.ID)) stateChange := StateChangeConf{ Pending: []string{"BUILD"}, Target: "ACTIVE", - Refresh: ServerStateRefreshFunc(csp, s.server), + Refresh: ServerStateRefreshFunc(computeClient, s.server), StepState: state, } latestServer, err := WaitForState(&stateChange) if err != nil { - err := fmt.Errorf("Error waiting for server (%s) to become ready: %s", s.server.Id, err) + err := fmt.Errorf("Error waiting for server (%s) to become ready: %s", s.server.ID, err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - s.server = latestServer.(*gophercloud.Server) + s.server = latestServer.(*servers.Server) state.Put("server", s.server) return multistep.ActionContinue @@ -83,18 +84,25 @@ func (s *StepRunSourceServer) Cleanup(state multistep.StateBag) { return } - csp := state.Get("csp").(gophercloud.CloudServersProvider) + config := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) + return + } + ui.Say("Terminating the source server...") - if err := csp.DeleteServerById(s.server.Id); err != nil { + if err := servers.Delete(computeClient, s.server.ID).ExtractErr(); err != nil { ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) return } stateChange := StateChangeConf{ Pending: []string{"ACTIVE", "BUILD", "REBUILD", "SUSPENDED"}, - Refresh: ServerStateRefreshFunc(csp, s.server), + Refresh: ServerStateRefreshFunc(computeClient, s.server), Target: "DELETED", } diff --git a/builder/openstack/step_wait_for_rackconnect.go b/builder/openstack/step_wait_for_rackconnect.go index ee6ee6138..6263bd17d 100644 --- a/builder/openstack/step_wait_for_rackconnect.go +++ b/builder/openstack/step_wait_for_rackconnect.go @@ -2,11 +2,11 @@ package openstack import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" "time" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) type StepWaitForRackConnect struct { @@ -18,14 +18,22 @@ func (s *StepWaitForRackConnect) Run(state multistep.StateBag) multistep.StepAct return multistep.ActionContinue } - csp := state.Get("csp").(gophercloud.CloudServersProvider) - server := state.Get("server").(*gophercloud.Server) + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) ui := state.Get("ui").(packer.Ui) - ui.Say(fmt.Sprintf("Waiting for server (%s) to become RackConnect ready...", server.Id)) + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + ui.Say(fmt.Sprintf( + "Waiting for server (%s) to become RackConnect ready...", server.ID)) for { - server, err := csp.ServerById(server.Id) + server, err = servers.Get(computeClient, server.ID).Extract() if err != nil { return multistep.ActionHalt } From c80d1ab46b609637399a2e696f1e184812b92067 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:16:56 -0400 Subject: [PATCH 110/208] remove the new plugin --- plugin/builder-openstack-new/main.go | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 plugin/builder-openstack-new/main.go diff --git a/plugin/builder-openstack-new/main.go b/plugin/builder-openstack-new/main.go deleted file mode 100644 index d8075c78d..000000000 --- a/plugin/builder-openstack-new/main.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -import ( - "github.com/mitchellh/packer/builder/openstack-new" - "github.com/mitchellh/packer/packer/plugin" -) - -func main() { - server, err := plugin.Server() - if err != nil { - panic(err) - } - server.RegisterBuilder(new(openstack.Builder)) - server.Serve() -} From 50e2eb30e6f2c0cb2313b91c1ccb9db62c7eb2e9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:28:38 -0400 Subject: [PATCH 111/208] builder/openstack: modifications to work with rackspace --- builder/openstack/access_config.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/builder/openstack/access_config.go b/builder/openstack/access_config.go index e0f962c50..71679b979 100644 --- a/builder/openstack/access_config.go +++ b/builder/openstack/access_config.go @@ -42,10 +42,7 @@ func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { } // Get as much as possible from the end - ao, err := openstack.AuthOptionsFromEnv() - if err != nil { - return []error{err} - } + ao, _ := openstack.AuthOptionsFromEnv() // Override values if we have them in our config overrides := []struct { From e724b5fe80de907da0dc8aa15b10ae0b83e0a812 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:33:52 -0400 Subject: [PATCH 112/208] builder/openstack: support legacy env vars --- builder/openstack/access_config.go | 17 ++++++ .../docs/builders/openstack.html.markdown | 57 +++++-------------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/builder/openstack/access_config.go b/builder/openstack/access_config.go index 71679b979..ca33495a8 100644 --- a/builder/openstack/access_config.go +++ b/builder/openstack/access_config.go @@ -41,6 +41,23 @@ func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { c.Region = os.Getenv("OS_REGION_NAME") } + // Legacy RackSpace stuff. We're keeping this around to keep things BC. + if c.APIKey == "" { + c.APIKey = os.Getenv("SDK_API_KEY") + } + if c.Password == "" { + c.Password = os.Getenv("SDK_PASSWORD") + } + if c.Region == "" { + c.Region = os.Getenv("SDK_REGION") + } + if c.TenantName == "" { + c.TenantName = os.Getenv("SDK_PROJECT") + } + if c.Username == "" { + c.Username = os.Getenv("SDK_USERNAME") + } + // Get as much as possible from the end ao, _ := openstack.AuthOptionsFromEnv() diff --git a/website/source/docs/builders/openstack.html.markdown b/website/source/docs/builders/openstack.html.markdown index d5dbbf249..10f38a445 100644 --- a/website/source/docs/builders/openstack.html.markdown +++ b/website/source/docs/builders/openstack.html.markdown @@ -34,23 +34,21 @@ each category, the available configuration keys are alphabetized. * `image_name` (string) - The name of the resulting image. -* `password` (string) - The password used to connect to the OpenStack service. - If not specified, Packer will use the environment variables - `SDK_PASSWORD` or `OS_PASSWORD` (in that order), if set. - * `source_image` (string) - The ID or full URL to the base image to use. This is the image that will be used to launch a new server and provision it. * `username` (string) - The username used to connect to the OpenStack service. + If not specified, Packer will use the environment variable + `OS_USERNAME`, if set. + +* `password` (string) - The password used to connect to the OpenStack service. If not specified, Packer will use the environment variables - `SDK_USERNAME` or `OS_USERNAME` (in that order), if set. + `OS_PASSWORD`, if set. ### Optional: * `api_key` (string) - The API key used to access OpenStack. Some OpenStack installations require this. - If not specified, Packer will use the environment variables - `SDK_API_KEY`, if set. * `floating_ip` (string) - A specific floating IP to assign to this instance. `use_floating_ip` must also be set to true for this to have an affect. @@ -65,32 +63,18 @@ each category, the available configuration keys are alphabetized. * `networks` (array of strings) - A list of networks by UUID to attach to this instance. -* `openstack_provider` (string) - A name of a provider that has a slightly - different API model. Currently supported values are "openstack" (default), - and "rackspace". - -* `project` (string) - The project name to boot the instance into. Some - OpenStack installations require this. - If not specified, Packer will use the environment variables - `SDK_PROJECT` or `OS_TENANT_NAME` (in that order), if set. - -* `provider` (string) - The provider used to connect to the OpenStack service. - If not specified, Packer will use the environment variables `SDK_PROVIDER` - or `OS_AUTH_URL` (in that order), if set. - For Rackspace this should be `rackspace-us` or `rackspace-uk`. - -* `proxy_url` (string) +* `tenant_id` or `tenant_name` (string) - The tenant ID or name to boot the + instance into. Some OpenStack installations require this. + If not specified, Packer will use the environment variable + `OS_TENANT_NAME`, if set. * `security_groups` (array of strings) - A list of security groups by name to add to this instance. * `region` (string) - The name of the region, such as "DFW", in which to launch the server to create the AMI. - If not specified, Packer will use the environment variables - `SDK_REGION` or `OS_REGION_NAME` (in that order), if set. - For a `provider` of "rackspace", it is required to specify a region, - either using this option or with an environment variable. For other - providers, including a private cloud, specifying a region is optional. + If not specified, Packer will use the environment variable + `OS_REGION_NAME`, if set. * `ssh_port` (integer) - The port that SSH will be available on. Defaults to port 22. @@ -106,9 +90,6 @@ each category, the available configuration keys are alphabetized. useful for Rackspace are "public" or "private", and the default behavior is to connect via whichever is returned first from the OpenStack API. -* `tenant_id` (string) - Tenant ID for accessing OpenStack if your - installation requires this. - * `use_floating_ip` (boolean) - Whether or not to use a floating IP for the instance. Defaults to false. @@ -124,10 +105,8 @@ Ubuntu 12.04 LTS (Precise Pangolin) on Rackspace OpenStack cloud offering. ```javascript { "type": "openstack", - "username": "", - "api_key": "", - "openstack_provider": "rackspace", - "provider": "rackspace-us", + "username": "foo", + "password": "foo", "region": "DFW", "ssh_username": "root", "image_name": "Test image", @@ -160,13 +139,3 @@ script is setting environment variables like: * `OS_TENANT_ID` * `OS_USERNAME` * `OS_PASSWORD` - -## Troubleshooting - -*I get the error "Missing or incorrect provider"* - -* Verify your "username", "password" and "provider" settings. - -*I get the error "Missing endpoint, or insufficient privileges to access endpoint"* - -* Verify your "region" setting. From 590177ea4b0b5057c5ec9a4da4b2811651d62c8b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:35:54 -0400 Subject: [PATCH 113/208] builder/openstack: fix unit tests --- builder/openstack/builder_test.go | 53 ------------------------------- 1 file changed, 53 deletions(-) diff --git a/builder/openstack/builder_test.go b/builder/openstack/builder_test.go index badf9784d..ce15873eb 100644 --- a/builder/openstack/builder_test.go +++ b/builder/openstack/builder_test.go @@ -9,7 +9,6 @@ func testConfig() map[string]interface{} { return map[string]interface{}{ "username": "foo", "password": "bar", - "provider": "foo", "region": "DFW", "image_name": "foo", "source_image": "foo", @@ -40,55 +39,3 @@ func TestBuilder_Prepare_BadType(t *testing.T) { t.Fatalf("prepare should fail") } } - -func TestBuilderPrepare_ImageName(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["image_name"] = "foo" - warns, err := b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - // Test bad - config["image_name"] = "foo {{" - b = Builder{} - warns, err = b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } - - // Test bad - delete(config, "image_name") - b = Builder{} - warns, err = b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } -} - -func TestBuilderPrepare_InvalidKey(t *testing.T) { - var b Builder - config := testConfig() - - // Add a random key - config["i_should_not_be_valid"] = true - warns, err := b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } -} From 5d32a1f6e059b107a112ec7811dd8a16c2e6101f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:02:04 -0400 Subject: [PATCH 114/208] builder/openstack: use IP not FixedIP --- builder/openstack/ssh.go | 4 ++-- builder/openstack/step_allocate_ip.go | 27 ++++++++++++++------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go index 7b0510f98..76c2686b1 100644 --- a/builder/openstack/ssh.go +++ b/builder/openstack/ssh.go @@ -23,8 +23,8 @@ func SSHAddress( // If we have a floating IP, use that ip := state.Get("access_ip").(*floatingip.FloatingIP) - if ip != nil && ip.FixedIP != "" { - return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil + if ip != nil && ip.IP != "" { + return fmt.Sprintf("%s:%d", ip.IP, port), nil } if s.AccessIPv4 != "" { diff --git a/builder/openstack/step_allocate_ip.go b/builder/openstack/step_allocate_ip.go index 16efe8d38..0ab9b3529 100644 --- a/builder/openstack/step_allocate_ip.go +++ b/builder/openstack/step_allocate_ip.go @@ -27,13 +27,14 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - var instanceIp *floatingip.FloatingIP + var instanceIp floatingip.FloatingIP + // This is here in case we error out before putting instanceIp into the // statebag below, because it is requested by Cleanup() - state.Put("access_ip", instanceIp) + state.Put("access_ip", &instanceIp) if s.FloatingIp != "" { - *instanceIp = floatingip.FloatingIP{FixedIP: s.FloatingIp} + instanceIp.IP = s.FloatingIp } else if s.FloatingIpPool != "" { newIp, err := floatingip.Create(client, floatingip.CreateOpts{ Pool: s.FloatingIpPool, @@ -45,26 +46,26 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - *instanceIp = *newIp - ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) + instanceIp = *newIp + ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.IP)) } - if instanceIp != nil && instanceIp.FixedIP != "" { - err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() + if instanceIp.IP != "" { + err := floatingip.Associate(client, server.ID, instanceIp.IP).ExtractErr() if err != nil { err := fmt.Errorf( "Error associating floating IP %s with instance.", - instanceIp.FixedIP) + instanceIp.IP) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } ui.Say(fmt.Sprintf( - "Added floating IP %s to instance...", instanceIp.FixedIP)) + "Added floating IP %s to instance...", instanceIp.IP)) } - state.Put("access_ip", instanceIp) + state.Put("access_ip", &instanceIp) return multistep.ActionContinue } @@ -77,17 +78,17 @@ func (s *StepAllocateIp) Cleanup(state multistep.StateBag) { client, err := config.computeV2Client() if err != nil { ui.Error(fmt.Sprintf( - "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + "Error deleting temporary floating IP %s", instanceIp.IP)) return } if s.FloatingIpPool != "" && instanceIp.ID != "" { if err := floatingip.Delete(client, instanceIp.ID).ExtractErr(); err != nil { ui.Error(fmt.Sprintf( - "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + "Error deleting temporary floating IP %s", instanceIp.IP)) return } - ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.FixedIP)) + ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.IP)) } } From ad374e82afe184249687a8e0993e7376b9d92a7c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:05:03 -0400 Subject: [PATCH 115/208] builder/openstack: shuffle some fields to note unused fields --- builder/openstack/run_config.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/builder/openstack/run_config.go b/builder/openstack/run_config.go index e5d73c9c1..de76fad20 100644 --- a/builder/openstack/run_config.go +++ b/builder/openstack/run_config.go @@ -11,19 +11,21 @@ import ( // RunConfig contains configuration for running an instance from a source // image and details on how to access that launched image. type RunConfig struct { - SourceImage string `mapstructure:"source_image"` - Flavor string `mapstructure:"flavor"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort int `mapstructure:"ssh_port"` - SSHInterface string `mapstructure:"ssh_interface"` - OpenstackProvider string `mapstructure:"openstack_provider"` - UseFloatingIp bool `mapstructure:"use_floating_ip"` - RackconnectWait bool `mapstructure:"rackconnect_wait"` - FloatingIpPool string `mapstructure:"floating_ip_pool"` - FloatingIp string `mapstructure:"floating_ip"` - SecurityGroups []string `mapstructure:"security_groups"` - Networks []string `mapstructure:"networks"` + SourceImage string `mapstructure:"source_image"` + Flavor string `mapstructure:"flavor"` + RawSSHTimeout string `mapstructure:"ssh_timeout"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort int `mapstructure:"ssh_port"` + SSHInterface string `mapstructure:"ssh_interface"` + RackconnectWait bool `mapstructure:"rackconnect_wait"` + FloatingIpPool string `mapstructure:"floating_ip_pool"` + FloatingIp string `mapstructure:"floating_ip"` + SecurityGroups []string `mapstructure:"security_groups"` + Networks []string `mapstructure:"networks"` + + // Not really used, but here for BC + OpenstackProvider string `mapstructure:"openstack_provider"` + UseFloatingIp bool `mapstructure:"use_floating_ip"` // Unexported fields that are calculated from others sshTimeout time.Duration From 92b6b5c387b4f090d55450b44ba1619d5a3eddef Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:32:31 -0400 Subject: [PATCH 116/208] builder/openstack: can ref flavor by name --- builder/openstack/builder.go | 4 +- builder/openstack/step_load_flavor.go | 61 +++++++++++++++++++ builder/openstack/step_run_source_server.go | 8 ++- .../docs/builders/openstack.html.markdown | 2 +- 4 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 builder/openstack/step_load_flavor.go diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index bebb28452..d30ede8d3 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -67,13 +67,15 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ + &StepLoadFlavor{ + Flavor: b.config.Flavor, + }, &StepKeyPair{ Debug: b.config.PackerDebug, DebugKeyPath: fmt.Sprintf("os_%s.pem", b.config.PackerBuildName), }, &StepRunSourceServer{ Name: b.config.ImageName, - Flavor: b.config.Flavor, SourceImage: b.config.SourceImage, SecurityGroups: b.config.SecurityGroups, Networks: b.config.Networks, diff --git a/builder/openstack/step_load_flavor.go b/builder/openstack/step_load_flavor.go new file mode 100644 index 000000000..8b8cae994 --- /dev/null +++ b/builder/openstack/step_load_flavor.go @@ -0,0 +1,61 @@ +package openstack + +import ( + "fmt" + "log" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/flavors" +) + +// StepLoadFlavor gets the FlavorRef from a Flavor. It first assumes +// that the Flavor is a ref and verifies it. Otherwise, it tries to find +// the flavor by name. +type StepLoadFlavor struct { + Flavor string +} + +func (s *StepLoadFlavor) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Say(fmt.Sprintf("Loading flavor: %s", s.Flavor)) + log.Printf("[INFO] Loading flavor by ID: %s", s.Flavor) + flavor, err := flavors.Get(client, s.Flavor).Extract() + if err != nil { + log.Printf("[ERROR] Failed to find flavor by ID: %s", err) + geterr := err + + log.Printf("[INFO] Loading flavor by name: %s", s.Flavor) + id, err := flavors.IDFromName(client, s.Flavor) + if err != nil { + log.Printf("[ERROR] Failed to find flavor by name: %s", err) + err = fmt.Errorf( + "Unable to find specified flavor by ID or name!\n\n"+ + "Error from ID lookup: %s\n\n"+ + "Error from name lookup: %s", + geterr, + err) + state.Put("error", err) + return multistep.ActionHalt + } + + flavor = &flavors.Flavor{ID: id} + } + + ui.Message(fmt.Sprintf("Verified flavor. ID: %s", flavor.ID)) + state.Put("flavor_id", flavor.ID) + return multistep.ActionContinue +} + +func (s *StepLoadFlavor) Cleanup(state multistep.StateBag) { +} diff --git a/builder/openstack/step_run_source_server.go b/builder/openstack/step_run_source_server.go index 4432d5860..89d816297 100644 --- a/builder/openstack/step_run_source_server.go +++ b/builder/openstack/step_run_source_server.go @@ -11,7 +11,6 @@ import ( ) type StepRunSourceServer struct { - Flavor string Name string SourceImage string SecurityGroups []string @@ -22,6 +21,7 @@ type StepRunSourceServer struct { func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction { config := state.Get("config").(Config) + flavor := state.Get("flavor_id").(string) keyName := state.Get("keyPair").(string) ui := state.Get("ui").(packer.Ui) @@ -38,11 +38,12 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction networks[i].UUID = networkUuid } + ui.Say("Launching server...") s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ CreateOptsBuilder: servers.CreateOpts{ Name: s.Name, ImageRef: s.SourceImage, - FlavorName: s.Flavor, + FlavorRef: flavor, SecurityGroups: s.SecurityGroups, Networks: networks, }, @@ -56,9 +57,10 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction return multistep.ActionHalt } + ui.Message(fmt.Sprintf("Server ID: %s", s.server.ID)) log.Printf("server id: %s", s.server.ID) - ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.ID)) + ui.Say("Waiting for server to become ready...") stateChange := StateChangeConf{ Pending: []string{"BUILD"}, Target: "ACTIVE", diff --git a/website/source/docs/builders/openstack.html.markdown b/website/source/docs/builders/openstack.html.markdown index 10f38a445..3b5bf791d 100644 --- a/website/source/docs/builders/openstack.html.markdown +++ b/website/source/docs/builders/openstack.html.markdown @@ -29,7 +29,7 @@ each category, the available configuration keys are alphabetized. ### Required: -* `flavor` (string) - The ID or full URL for the desired flavor for the +* `flavor` (string) - The ID, name, or full URL for the desired flavor for the server to be created. * `image_name` (string) - The name of the resulting image. From 53b117e223ab27f3dec38ca34b22caea4587273e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:54:12 -0400 Subject: [PATCH 117/208] update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1bc247f5..15ce0290a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,12 @@ BACKWARDS INCOMPATIBILITIES: - * The DigitalOcean builder no longer supports the v1 API which has been + * builder/digitalocean: no longer supports the v1 API which has been deprecated for some time. Most configurations should continue to work as long as you use the `api_token` field for auth. * builder/digitalocean: `image`, `region`, and `size` are now required. + * builder/openstack: auth parameters have been changed to better + reflect OS terminology. Existing environment variables still work. FEATURES: From 44d86c2e90faa8876b3ac99e7beaf67021c0001a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:54:36 -0400 Subject: [PATCH 118/208] update CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15ce0290a..d077ceba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,9 @@ IMPROVEMENTS: * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear - * buidler/openstakc: Add `ssh_interface` option for rackconnect for users that + * buidler/openstack: Add `ssh_interface` option for rackconnect for users that have prohibitive firewalls + * builder/openstack: Flavor names can be used as well as refs * builder/virtualbox: Added option: `ssh_skip_nat_mapping` to skip the automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support From 693f04afccd604e41b07b261ecea873990bc046c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 11:10:10 -0400 Subject: [PATCH 119/208] builder/openstack: AZ support --- builder/openstack/builder.go | 9 ++++---- builder/openstack/run_config.go | 23 ++++++++++--------- builder/openstack/step_run_source_server.go | 20 ++++++++-------- .../docs/builders/openstack.html.markdown | 4 ++++ 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index d30ede8d3..ab60afc0e 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -75,10 +75,11 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe DebugKeyPath: fmt.Sprintf("os_%s.pem", b.config.PackerBuildName), }, &StepRunSourceServer{ - Name: b.config.ImageName, - SourceImage: b.config.SourceImage, - SecurityGroups: b.config.SecurityGroups, - Networks: b.config.Networks, + Name: b.config.ImageName, + SourceImage: b.config.SourceImage, + SecurityGroups: b.config.SecurityGroups, + Networks: b.config.Networks, + AvailabilityZone: b.config.AvailabilityZone, }, &StepWaitForRackConnect{ Wait: b.config.RackconnectWait, diff --git a/builder/openstack/run_config.go b/builder/openstack/run_config.go index de76fad20..4a6a1b81f 100644 --- a/builder/openstack/run_config.go +++ b/builder/openstack/run_config.go @@ -11,17 +11,18 @@ import ( // RunConfig contains configuration for running an instance from a source // image and details on how to access that launched image. type RunConfig struct { - SourceImage string `mapstructure:"source_image"` - Flavor string `mapstructure:"flavor"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort int `mapstructure:"ssh_port"` - SSHInterface string `mapstructure:"ssh_interface"` - RackconnectWait bool `mapstructure:"rackconnect_wait"` - FloatingIpPool string `mapstructure:"floating_ip_pool"` - FloatingIp string `mapstructure:"floating_ip"` - SecurityGroups []string `mapstructure:"security_groups"` - Networks []string `mapstructure:"networks"` + SourceImage string `mapstructure:"source_image"` + Flavor string `mapstructure:"flavor"` + RawSSHTimeout string `mapstructure:"ssh_timeout"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort int `mapstructure:"ssh_port"` + SSHInterface string `mapstructure:"ssh_interface"` + AvailabilityZone string `mapstructure:"availability_zone"` + RackconnectWait bool `mapstructure:"rackconnect_wait"` + FloatingIpPool string `mapstructure:"floating_ip_pool"` + FloatingIp string `mapstructure:"floating_ip"` + SecurityGroups []string `mapstructure:"security_groups"` + Networks []string `mapstructure:"networks"` // Not really used, but here for BC OpenstackProvider string `mapstructure:"openstack_provider"` diff --git a/builder/openstack/step_run_source_server.go b/builder/openstack/step_run_source_server.go index 89d816297..4014f1d95 100644 --- a/builder/openstack/step_run_source_server.go +++ b/builder/openstack/step_run_source_server.go @@ -11,10 +11,11 @@ import ( ) type StepRunSourceServer struct { - Name string - SourceImage string - SecurityGroups []string - Networks []string + Name string + SourceImage string + SecurityGroups []string + Networks []string + AvailabilityZone string server *servers.Server } @@ -41,11 +42,12 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction ui.Say("Launching server...") s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ CreateOptsBuilder: servers.CreateOpts{ - Name: s.Name, - ImageRef: s.SourceImage, - FlavorRef: flavor, - SecurityGroups: s.SecurityGroups, - Networks: networks, + Name: s.Name, + ImageRef: s.SourceImage, + FlavorRef: flavor, + SecurityGroups: s.SecurityGroups, + Networks: networks, + AvailabilityZone: s.AvailabilityZone, }, KeyName: keyName, diff --git a/website/source/docs/builders/openstack.html.markdown b/website/source/docs/builders/openstack.html.markdown index 3b5bf791d..a7fef29dc 100644 --- a/website/source/docs/builders/openstack.html.markdown +++ b/website/source/docs/builders/openstack.html.markdown @@ -50,6 +50,10 @@ each category, the available configuration keys are alphabetized. * `api_key` (string) - The API key used to access OpenStack. Some OpenStack installations require this. +* `availability_zone` (string) - The availability zone to launch the + server in. If this isn't specified, the default enforced by your OpenStack + cluster will be used. This may be required for some OpenStack clusters. + * `floating_ip` (string) - A specific floating IP to assign to this instance. `use_floating_ip` must also be set to true for this to have an affect. From 9bb722c99f3f2e3da83bfaf72d4411e754202644 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 11:10:34 -0400 Subject: [PATCH 120/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d077ceba6..b96154c2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ IMPROVEMENTS: * buidler/openstack: Add `ssh_interface` option for rackconnect for users that have prohibitive firewalls * builder/openstack: Flavor names can be used as well as refs + * builder/openstack: Add `availability_zone` [GH-2016] * builder/virtualbox: Added option: `ssh_skip_nat_mapping` to skip the automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support From 35b8df18162f8849eb28f1bed53f67c1421aacd4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 11:14:28 -0400 Subject: [PATCH 121/208] website: note cloud-init req for openstack [GH-1750] --- website/source/docs/builders/openstack.html.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/source/docs/builders/openstack.html.markdown b/website/source/docs/builders/openstack.html.markdown index a7fef29dc..fcd210dec 100644 --- a/website/source/docs/builders/openstack.html.markdown +++ b/website/source/docs/builders/openstack.html.markdown @@ -36,6 +36,8 @@ each category, the available configuration keys are alphabetized. * `source_image` (string) - The ID or full URL to the base image to use. This is the image that will be used to launch a new server and provision it. + Unless you specify completely custom SSH settings, the source image must + have `cloud-init` installed so that the keypair gets assigned properly. * `username` (string) - The username used to connect to the OpenStack service. If not specified, Packer will use the environment variable From 86206e316db6b1a23dc67c43324af75e7b2860a5 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Fri, 12 Jun 2015 10:39:37 -0500 Subject: [PATCH 122/208] add tags test --- builder/amazon/ebs/tags_acc_test.go | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 builder/amazon/ebs/tags_acc_test.go diff --git a/builder/amazon/ebs/tags_acc_test.go b/builder/amazon/ebs/tags_acc_test.go new file mode 100644 index 000000000..606bb89ee --- /dev/null +++ b/builder/amazon/ebs/tags_acc_test.go @@ -0,0 +1,114 @@ +package ebs + +import ( + "fmt" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/packer/builder/amazon/common" + builderT "github.com/mitchellh/packer/helper/builder/testing" + "github.com/mitchellh/packer/packer" +) + +func TestBuilderTagsAcc_basic(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Builder: &Builder{}, + Template: testBuilderTagsAccBasic, + Check: checkTags(), + }) +} + +func checkTags() builderT.TestCheckFunc { + return func(artifacts []packer.Artifact) error { + if len(artifacts) > 1 { + return fmt.Errorf("more than 1 artifact") + } + + tags := make(map[string]string) + tags["OS_Version"] = "Ubuntu" + tags["Release"] = "Latest" + + // Get the actual *Artifact pointer so we can access the AMIs directly + artifactRaw := artifacts[0] + artifact, ok := artifactRaw.(*common.Artifact) + if !ok { + return fmt.Errorf("unknown artifact: %#v", artifactRaw) + } + + // describe the image, get block devices with a snapshot + ec2conn, _ := testEC2Conn() + imageResp, err := ec2conn.DescribeImages(&ec2.DescribeImagesInput{ + ImageIDs: []*string{aws.String(artifact.Amis["us-east-1"])}, + }) + + if err != nil { + return fmt.Errorf("Error retrieving details for AMI Artifcat (%#v) in Tags Test: %s", artifact, err) + } + + if len(imageResp.Images) == 0 { + return fmt.Errorf("No images found for AMI Artifcat (%#v) in Tags Test: %s", artifact, err) + } + + image := imageResp.Images[0] + + // Check only those with a Snapshot ID, i.e. not Ephemeral + var snapshots []*string + for _, device := range image.BlockDeviceMappings { + if device.EBS != nil && device.EBS.SnapshotID != nil { + snapshots = append(snapshots, device.EBS.SnapshotID) + } + } + + // grab matching snapshot info + resp, err := ec2conn.DescribeSnapshots(&ec2.DescribeSnapshotsInput{ + SnapshotIDs: snapshots, + }) + + if err != nil { + return fmt.Errorf("Error retreiving Snapshots for AMI Artifcat (%#v) in Tags Test: %s", artifact, err) + } + + if len(resp.Snapshots) == 0 { + return fmt.Errorf("No Snapshots found for AMI Artifcat (%#v) in Tags Test", artifact) + } + + // grab the snapshots, check the tags + for _, s := range resp.Snapshots { + expected := len(tags) + for _, t := range s.Tags { + for key, value := range tags { + if key == *t.Key && value == *t.Value { + expected-- + } + } + } + + if expected > 0 { + return fmt.Errorf("Not all tags found") + } + } + + return nil + } +} + +const testBuilderTagsAccBasic = ` +{ + "builders": [ + { + "type": "test", + "region": "us-east-1", + "source_ami": "ami-9eaa1cf6", + "instance_type": "t2.micro", + "ssh_username": "ubuntu", + "ami_name": "packer-tags-testing-{{timestamp}}", + "tags": { + "OS_Version": "Ubuntu", + "Release": "Latest" + } + } + ] +} +` From 85db8abe8daab3413868aa48591356ea015342c1 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Fri, 12 Jun 2015 10:40:55 -0500 Subject: [PATCH 123/208] remove old bats test --- test/builder_amazon_ebs.bats | 54 ------------------------------------ 1 file changed, 54 deletions(-) delete mode 100755 test/builder_amazon_ebs.bats diff --git a/test/builder_amazon_ebs.bats b/test/builder_amazon_ebs.bats deleted file mode 100755 index 89f32a4a0..000000000 --- a/test/builder_amazon_ebs.bats +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bats -# -# This tests the amazon-ebs builder. The teardown function will automatically -# delete any AMIs with a tag of `packer-test` being equal to "true" so -# be sure any test cases set this. - -load test_helper -fixtures amazon-ebs - -# This counts how many AMIs were copied to another region -aws_ami_region_copy_count() { - aws ec2 describe-images --region $1 --owners self --output text \ - --filters 'Name=tag:packer-id,Values=ami_region_copy' \ - --query "Images[*].ImageId" \ - | wc -l -} - -# This verifies AMI tags are correctly applied to relevant snapshots -aws_ami_snapshot_tags_count() { - filter='Name=tag:packer-id,Values=ami_snapshot_tags' - aws ec2 describe-images --region $1 --owners self --output text \ - --filters "$filter" \ - --query "Images[*].BlockDeviceMappings[*].Ebs.SnapshotId" \ - | aws ec2 describe-snapshots --region $1 --owners self --output text \ - --filters "$filter" \ - --snapshot-ids \ - | wc -l -} - -teardown() { - aws_ami_cleanup 'us-east-1' - aws_ami_cleanup 'us-west-1' - aws_ami_cleanup 'us-west-2' -} - -@test "amazon-ebs: build minimal.json" { - run packer build $FIXTURE_ROOT/minimal.json - [ "$status" -eq 0 ] -} - -# @unit-testable -@test "amazon-ebs: AMI region copy" { - run packer build $FIXTURE_ROOT/ami_region_copy.json - [ "$status" -eq 0 ] - [ "$(aws_ami_region_copy_count 'us-east-1')" -eq "1" ] - [ "$(aws_ami_region_copy_count 'us-west-1')" -eq "1" ] - [ "$(aws_ami_region_copy_count 'us-west-2')" -eq "1" ] -} - -@test "amazon-ebs: AMI snapshot tags" { - run packer build $FIXTURE_ROOT/ami_snapshot_tags.json - [ "$status" -eq 0 ] - [ "$(aws_ami_snapshot_tags)" -eq "2" ] -} From c875d40b2cbf6407379cf2b4f0ecba24c86e4f2f Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Fri, 12 Jun 2015 10:41:44 -0500 Subject: [PATCH 124/208] remove bats test fixture --- .../amazon-ebs/ami_snapshot_tags.json | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 test/fixtures/amazon-ebs/ami_snapshot_tags.json diff --git a/test/fixtures/amazon-ebs/ami_snapshot_tags.json b/test/fixtures/amazon-ebs/ami_snapshot_tags.json deleted file mode 100644 index 278474a32..000000000 --- a/test/fixtures/amazon-ebs/ami_snapshot_tags.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "builders": [{ - "type": "amazon-ebs", - "ami_name": "packer-test {{timestamp}}", - "instance_type": "m1.small", - "region": "us-east-1", - "ssh_username": "ubuntu", - "source_ami": "ami-0568456c", - "tags": { - "packer-test": "true", - "packer-id": "ami_snapshot_tags" - }, - "ami_block_device_mappings": [ - { - "device_name": "/dev/sde", - "volume_type": "standard" - } - ] - }] -} From 31abc93f50d2389b1237a216af3616db1dc31d7c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 14:22:32 -0400 Subject: [PATCH 125/208] builder/openstack: support ssh_interface [GH-2087] --- builder/openstack/ssh.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go index 76c2686b1..519ccd406 100644 --- a/builder/openstack/ssh.go +++ b/builder/openstack/ssh.go @@ -33,7 +33,15 @@ func SSHAddress( // Get all the addresses associated with this server. This // was taken directly from Terraform. - for _, networkAddresses := range s.Addresses { + for pool, networkAddresses := range s.Addresses { + // If we have an SSH interface specified, skip it if no match + if sshinterface != "" && pool != sshinterface { + log.Printf( + "[INFO] Skipping pool %s, doesn't match requested %s", + pool, sshinterface) + continue + } + elements, ok := networkAddresses.([]interface{}) if !ok { log.Printf( From bec59b535df5dab334ecbe978bfa846036b00e86 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Fri, 12 Jun 2015 13:05:15 -0500 Subject: [PATCH 126/208] builder/amazon: Add force_deregister option, to automatically deregister artifacts with name conflicts --- builder/amazon/chroot/builder.go | 8 +++ builder/amazon/common/ami_config.go | 1 + builder/amazon/common/step_deregister_ami.go | 56 ++++++++++++++++++++ builder/amazon/common/step_pre_validate.go | 10 +++- builder/amazon/ebs/builder.go | 7 ++- builder/amazon/ebs/builder_acc_test.go | 34 ++++++++++++ builder/amazon/instance/builder.go | 8 +++ helper/builder/testing/testing.go | 18 ++++--- 8 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 builder/amazon/common/step_deregister_ami.go diff --git a/builder/amazon/chroot/builder.go b/builder/amazon/chroot/builder.go index 9e7452182..18b07b81c 100644 --- a/builder/amazon/chroot/builder.go +++ b/builder/amazon/chroot/builder.go @@ -147,6 +147,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ + &awscommon.StepPreValidate{ + DestAmiName: b.config.AMIName, + ForceDeregister: b.config.AMIForceDeregister, + }, &StepInstanceInfo{}, &awscommon.StepSourceAMIInfo{ SourceAmi: b.config.SourceAmi, @@ -164,6 +168,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &StepChrootProvision{}, &StepEarlyCleanup{}, &StepSnapshot{}, + &awscommon.StepDeregisterAMI{ + ForceDeregister: b.config.AMIForceDeregister, + AMIName: b.config.AMIName, + }, &StepRegisterAMI{}, &awscommon.StepAMIRegionCopy{ AccessConfig: &b.config.AccessConfig, diff --git a/builder/amazon/common/ami_config.go b/builder/amazon/common/ami_config.go index 14b880f4c..377201902 100644 --- a/builder/amazon/common/ami_config.go +++ b/builder/amazon/common/ami_config.go @@ -17,6 +17,7 @@ type AMIConfig struct { AMIRegions []string `mapstructure:"ami_regions"` AMITags map[string]string `mapstructure:"tags"` AMIEnhancedNetworking bool `mapstructure:"enhanced_networking"` + AMIForceDeregister bool `mapstructure:"force_deregister"` } func (c *AMIConfig) Prepare(ctx *interpolate.Context) []error { diff --git a/builder/amazon/common/step_deregister_ami.go b/builder/amazon/common/step_deregister_ami.go new file mode 100644 index 000000000..ce20a5d90 --- /dev/null +++ b/builder/amazon/common/step_deregister_ami.go @@ -0,0 +1,56 @@ +package common + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" +) + +type StepDeregisterAMI struct { + ForceDeregister bool + AMIName string +} + +func (s *StepDeregisterAMI) Run(state multistep.StateBag) multistep.StepAction { + ec2conn := state.Get("ec2").(*ec2.EC2) + ui := state.Get("ui").(packer.Ui) + + // check for force deregister + if s.ForceDeregister { + resp, err := ec2conn.DescribeImages(&ec2.DescribeImagesInput{ + Filters: []*ec2.Filter{&ec2.Filter{ + Name: aws.String("name"), + Values: []*string{aws.String(s.AMIName)}, + }}}) + + if err != nil { + err := fmt.Errorf("Error creating AMI: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // deregister image(s) by that name + for _, i := range resp.Images { + _, err := ec2conn.DeregisterImage(&ec2.DeregisterImageInput{ + ImageID: i.ImageID, + }) + + if err != nil { + err := fmt.Errorf("Error deregistering existing AMI: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + ui.Say(fmt.Sprintf("Deregistered AMI %s, id: %s", s.AMIName, *i.ImageID)) + } + } + + return multistep.ActionContinue +} + +func (s *StepDeregisterAMI) Cleanup(state multistep.StateBag) { +} diff --git a/builder/amazon/common/step_pre_validate.go b/builder/amazon/common/step_pre_validate.go index 5eb263eca..bbeacea43 100644 --- a/builder/amazon/common/step_pre_validate.go +++ b/builder/amazon/common/step_pre_validate.go @@ -13,12 +13,18 @@ import ( // the build before actually doing any time consuming work // type StepPreValidate struct { - DestAmiName string + DestAmiName string + ForceDeregister bool } func (s *StepPreValidate) Run(state multistep.StateBag) multistep.StepAction { - ec2conn := state.Get("ec2").(*ec2.EC2) ui := state.Get("ui").(packer.Ui) + if s.ForceDeregister { + ui.Say("Force Deregister flag found, skipping prevalidating AMI Name") + return multistep.ActionContinue + } + + ec2conn := state.Get("ec2").(*ec2.EC2) ui.Say("Prevalidating AMI Name...") resp, err := ec2conn.DescribeImages(&ec2.DescribeImagesInput{ diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index c689accee..356c91e2b 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -79,7 +79,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ &awscommon.StepPreValidate{ - DestAmiName: b.config.AMIName, + DestAmiName: b.config.AMIName, + ForceDeregister: b.config.AMIForceDeregister, }, &awscommon.StepSourceAMIInfo{ SourceAmi: b.config.SourceAmi, @@ -122,6 +123,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &stepStopInstance{SpotPrice: b.config.SpotPrice}, // TODO(mitchellh): verify works with spots &stepModifyInstance{}, + &awscommon.StepDeregisterAMI{ + ForceDeregister: b.config.AMIForceDeregister, + AMIName: b.config.AMIName, + }, &stepCreateAMI{}, &awscommon.StepAMIRegionCopy{ AccessConfig: &b.config.AccessConfig, diff --git a/builder/amazon/ebs/builder_acc_test.go b/builder/amazon/ebs/builder_acc_test.go index b70f7f7b9..1b4de70ce 100644 --- a/builder/amazon/ebs/builder_acc_test.go +++ b/builder/amazon/ebs/builder_acc_test.go @@ -28,6 +28,22 @@ func TestBuilderAcc_regionCopy(t *testing.T) { }) } +func TestBuilderAcc_forceDeregister(t *testing.T) { + // Build the same AMI name twice, with force_deregister on the second run + builderT.Test(t, builderT.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Builder: &Builder{}, + Template: buildForceDeregisterConfig("false", "dereg"), + SkipArtifactTeardown: true, + }) + + builderT.Test(t, builderT.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Builder: &Builder{}, + Template: buildForceDeregisterConfig("true", "dereg"), + }) +} + func checkRegionCopy(regions []string) builderT.TestCheckFunc { return func(artifacts []packer.Artifact) error { if len(artifacts) > 1 { @@ -107,3 +123,21 @@ const testBuilderAccRegionCopy = ` }] } ` + +const testBuilderAccForceDeregister = ` +{ + "builders": [{ + "type": "test", + "region": "us-east-1", + "instance_type": "m3.medium", + "source_ami": "ami-76b2a71e", + "ssh_username": "ubuntu", + "force_deregister": "%s", + "ami_name": "packer-test-%s" + }] +} +` + +func buildForceDeregisterConfig(name, flag string) string { + return fmt.Sprintf(testBuilderAccForceDeregister, name, flag) +} diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 91355b913..385544d61 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -167,6 +167,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ + &awscommon.StepPreValidate{ + DestAmiName: b.config.AMIName, + ForceDeregister: b.config.AMIForceDeregister, + }, &awscommon.StepSourceAMIInfo{ SourceAmi: b.config.SourceAmi, EnhancedNetworking: b.config.AMIEnhancedNetworking, @@ -211,6 +215,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &StepUploadBundle{ Debug: b.config.PackerDebug, }, + &awscommon.StepDeregisterAMI{ + ForceDeregister: b.config.AMIForceDeregister, + AMIName: b.config.AMIName, + }, &StepRegisterAMI{}, &awscommon.StepAMIRegionCopy{ AccessConfig: &b.config.AccessConfig, diff --git a/helper/builder/testing/testing.go b/helper/builder/testing/testing.go index 29200e108..522d7a265 100644 --- a/helper/builder/testing/testing.go +++ b/helper/builder/testing/testing.go @@ -41,6 +41,10 @@ type TestCase struct { // in the case that the test can't guarantee all resources were // properly cleaned up. Teardown TestTeardownFunc + + // If SkipArtifactTeardown is true, we will not attempt to destroy the + // artifact created in this test run. + SkipArtifactTeardown bool } // TestCheckFunc is the callback used for Check in TestStep. @@ -163,12 +167,14 @@ func Test(t TestT, c TestCase) { } TEARDOWN: - // Delete all artifacts - for _, a := range artifacts { - if err := a.Destroy(); err != nil { - t.Error(fmt.Sprintf( - "!!! ERROR REMOVING ARTIFACT '%s': %s !!!", - a.String(), err)) + if !c.SkipArtifactTeardown { + // Delete all artifacts + for _, a := range artifacts { + if err := a.Destroy(); err != nil { + t.Error(fmt.Sprintf( + "!!! ERROR REMOVING ARTIFACT '%s': %s !!!", + a.String(), err)) + } } } From 2c683c50573982d2bb0cb3be745b9f52140b2010 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 14:28:27 -0400 Subject: [PATCH 127/208] builder/openstack: prioritize ssh interfaces --- builder/openstack/ssh.go | 82 ++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go index 519ccd406..65e057084 100644 --- a/builder/openstack/ssh.go +++ b/builder/openstack/ssh.go @@ -21,6 +21,13 @@ func SSHAddress( return func(state multistep.StateBag) (string, error) { s := state.Get("server").(*servers.Server) + // If we have a specific interface, try that + if sshinterface != "" { + if addr := sshAddrFromPool(s, sshinterface, port); addr != "" { + return addr, nil + } + } + // If we have a floating IP, use that ip := state.Get("access_ip").(*floatingip.FloatingIP) if ip != nil && ip.IP != "" { @@ -31,39 +38,9 @@ func SSHAddress( return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil } - // Get all the addresses associated with this server. This - // was taken directly from Terraform. - for pool, networkAddresses := range s.Addresses { - // If we have an SSH interface specified, skip it if no match - if sshinterface != "" && pool != sshinterface { - log.Printf( - "[INFO] Skipping pool %s, doesn't match requested %s", - pool, sshinterface) - continue - } - - elements, ok := networkAddresses.([]interface{}) - if !ok { - log.Printf( - "[ERROR] Unknown return type for address field: %#v", - networkAddresses) - continue - } - - for _, element := range elements { - var addr string - address := element.(map[string]interface{}) - if address["OS-EXT-IPS:type"] == "floating" { - addr = address["addr"].(string) - } else { - if address["version"].(float64) == 4 { - addr = address["addr"].(string) - } - } - if addr != "" { - return fmt.Sprintf("%s:%d", addr, port), nil - } - } + // Try to get it from the requested interface + if addr := sshAddrFromPool(s, sshinterface, port); addr != "" { + return addr, nil } s, err := servers.Get(client, s.ID).Extract() @@ -98,3 +75,42 @@ func SSHConfig(username string) func(multistep.StateBag) (*ssh.ClientConfig, err }, nil } } + +func sshAddrFromPool(s *servers.Server, desired string, port int) string { + // Get all the addresses associated with this server. This + // was taken directly from Terraform. + for pool, networkAddresses := range s.Addresses { + // If we have an SSH interface specified, skip it if no match + if desired != "" && pool != desired { + log.Printf( + "[INFO] Skipping pool %s, doesn't match requested %s", + pool, desired) + continue + } + + elements, ok := networkAddresses.([]interface{}) + if !ok { + log.Printf( + "[ERROR] Unknown return type for address field: %#v", + networkAddresses) + continue + } + + for _, element := range elements { + var addr string + address := element.(map[string]interface{}) + if address["OS-EXT-IPS:type"] == "floating" { + addr = address["addr"].(string) + } else { + if address["version"].(float64) == 4 { + addr = address["addr"].(string) + } + } + if addr != "" { + return fmt.Sprintf("%s:%d", addr, port) + } + } + } + + return "" +} From a3863c3495db25eed06da1a2a5741f31dd9c8935 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 15:23:05 -0400 Subject: [PATCH 128/208] builder/openstack: update floating IP messaging --- builder/openstack/step_allocate_ip.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/builder/openstack/step_allocate_ip.go b/builder/openstack/step_allocate_ip.go index 0ab9b3529..fc386082b 100644 --- a/builder/openstack/step_allocate_ip.go +++ b/builder/openstack/step_allocate_ip.go @@ -36,6 +36,8 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { if s.FloatingIp != "" { instanceIp.IP = s.FloatingIp } else if s.FloatingIpPool != "" { + ui.Say(fmt.Sprintf("Creating floating IP...")) + ui.Message(fmt.Sprintf("Pool: %s", s.FloatingIpPool)) newIp, err := floatingip.Create(client, floatingip.CreateOpts{ Pool: s.FloatingIpPool, }).Extract() @@ -47,22 +49,24 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { } instanceIp = *newIp - ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.IP)) + ui.Message(fmt.Sprintf("Created floating IP: %s", instanceIp.IP)) } if instanceIp.IP != "" { + ui.Say(fmt.Sprintf("Associating floating IP with server...")) + ui.Message(fmt.Sprintf("IP: %s", instanceIp.IP)) err := floatingip.Associate(client, server.ID, instanceIp.IP).ExtractErr() if err != nil { err := fmt.Errorf( - "Error associating floating IP %s with instance.", - instanceIp.IP) + "Error associating floating IP %s with instance: %s", + instanceIp.IP, err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - ui.Say(fmt.Sprintf( - "Added floating IP %s to instance...", instanceIp.IP)) + ui.Message(fmt.Sprintf( + "Added floating IP %s to instance!", instanceIp.IP)) } state.Put("access_ip", &instanceIp) From f398352996bc5e696ee17c47460099c3fcbd0ffb Mon Sep 17 00:00:00 2001 From: Mark Peek Date: Fri, 12 Jun 2015 14:00:59 -0700 Subject: [PATCH 129/208] Fix a bug where interpolation was broken in some builders --- builder/googlecompute/config.go | 2 +- builder/null/config.go | 2 +- builder/parallels/pvm/config.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/googlecompute/config.go b/builder/googlecompute/config.go index 743e4745c..69d223a75 100644 --- a/builder/googlecompute/config.go +++ b/builder/googlecompute/config.go @@ -47,7 +47,7 @@ type Config struct { func NewConfig(raws ...interface{}) (*Config, []string, error) { c := new(Config) - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ diff --git a/builder/null/config.go b/builder/null/config.go index 7665aec51..9ccc32282 100644 --- a/builder/null/config.go +++ b/builder/null/config.go @@ -21,7 +21,7 @@ type Config struct { func NewConfig(raws ...interface{}) (*Config, []string, error) { c := new(Config) - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ diff --git a/builder/parallels/pvm/config.go b/builder/parallels/pvm/config.go index c3fab4446..8f6d9a915 100644 --- a/builder/parallels/pvm/config.go +++ b/builder/parallels/pvm/config.go @@ -33,7 +33,7 @@ type Config struct { func NewConfig(raws ...interface{}) (*Config, []string, error) { c := new(Config) - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ From 16320372d64adacf1c172af0535fe56114d3b611 Mon Sep 17 00:00:00 2001 From: Mark Peek Date: Fri, 12 Jun 2015 14:02:09 -0700 Subject: [PATCH 130/208] Make some builder config usage more consistent with other builders --- builder/digitalocean/config.go | 6 +++--- builder/docker/config.go | 6 +++--- builder/virtualbox/ovf/config.go | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index a19dabda2..178b54049 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -42,10 +42,10 @@ type Config struct { } func NewConfig(raws ...interface{}) (*Config, []string, error) { - var c Config + c := new(Config) var md mapstructure.Metadata - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Metadata: &md, Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ @@ -142,5 +142,5 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { } common.ScrubConfig(c, c.APIToken) - return &c, nil, nil + return c, nil, nil } diff --git a/builder/docker/config.go b/builder/docker/config.go index 024b915af..d5801c8ba 100644 --- a/builder/docker/config.go +++ b/builder/docker/config.go @@ -31,10 +31,10 @@ type Config struct { } func NewConfig(raws ...interface{}) (*Config, []string, error) { - var c Config + c := new(Config) var md mapstructure.Metadata - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Metadata: &md, Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ @@ -91,5 +91,5 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { return nil, nil, errs } - return &c, nil, nil + return c, nil, nil } diff --git a/builder/virtualbox/ovf/config.go b/builder/virtualbox/ovf/config.go index de60bcc7b..837c3f37a 100644 --- a/builder/virtualbox/ovf/config.go +++ b/builder/virtualbox/ovf/config.go @@ -40,8 +40,8 @@ type Config struct { } func NewConfig(raws ...interface{}) (*Config, []string, error) { - var c Config - err := config.Decode(&c, &config.DecodeOpts{ + c := new(Config) + err := config.Decode(c, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ @@ -132,5 +132,5 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { c.ImportFlags = append(c.ImportFlags, "--options", c.ImportOpts) } - return &c, warnings, nil + return c, warnings, nil } From 8b4f980123bbb09d0a4a0b404d41070db17c8fe4 Mon Sep 17 00:00:00 2001 From: Mark Peek Date: Fri, 12 Jun 2015 15:00:53 -0700 Subject: [PATCH 131/208] website: fix missing comma in parallels-pvm example --- website/source/docs/builders/parallels-pvm.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/builders/parallels-pvm.html.markdown b/website/source/docs/builders/parallels-pvm.html.markdown index 355e325c5..434bda8e7 100644 --- a/website/source/docs/builders/parallels-pvm.html.markdown +++ b/website/source/docs/builders/parallels-pvm.html.markdown @@ -27,7 +27,7 @@ the settings here. ```javascript { "type": "parallels-pvm", - "parallels_tools_flavor": "lin" + "parallels_tools_flavor": "lin", "source_path": "source.pvm", "ssh_username": "packer", "ssh_password": "packer", From 48b674d331c22d1dc75ea80fb8605acf5a3c3a7c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 18:04:46 -0400 Subject: [PATCH 132/208] builder/openstack: load extensions, stop server if supported --- builder/openstack/builder.go | 2 + builder/openstack/step_load_extensions.go | 58 +++++++++++++++++++++++ builder/openstack/step_stop_server.go | 58 +++++++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 builder/openstack/step_load_extensions.go create mode 100644 builder/openstack/step_stop_server.go diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index ab60afc0e..2256ad80e 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -67,6 +67,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ + &StepLoadExtensions{}, &StepLoadFlavor{ Flavor: b.config.Flavor, }, @@ -94,6 +95,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SSHWaitTimeout: b.config.SSHTimeout(), }, &common.StepProvision{}, + &StepStopServer{}, &stepCreateImage{}, } diff --git a/builder/openstack/step_load_extensions.go b/builder/openstack/step_load_extensions.go new file mode 100644 index 000000000..095863612 --- /dev/null +++ b/builder/openstack/step_load_extensions.go @@ -0,0 +1,58 @@ +package openstack + +import ( + "fmt" + "log" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions" + "github.com/rackspace/gophercloud/pagination" +) + +// StepLoadExtensions gets the FlavorRef from a Flavor. It first assumes +// that the Flavor is a ref and verifies it. Otherwise, it tries to find +// the flavor by name. +type StepLoadExtensions struct{} + +func (s *StepLoadExtensions) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Say("Discovering enabled extensions...") + result := make(map[string]struct{}, 15) + pager := extensions.List(client) + err = pager.EachPage(func(p pagination.Page) (bool, error) { + // Extract the extensions from this page + exts, err := extensions.ExtractExtensions(p) + if err != nil { + return false, err + } + + for _, ext := range exts { + log.Printf("[DEBUG] Discovered extension: %s", ext.Alias) + result[ext.Alias] = struct{}{} + } + + return true, nil + }) + if err != nil { + err = fmt.Errorf("Error loading extensions: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + state.Put("extensions", result) + return multistep.ActionContinue +} + +func (s *StepLoadExtensions) Cleanup(state multistep.StateBag) { +} diff --git a/builder/openstack/step_stop_server.go b/builder/openstack/step_stop_server.go new file mode 100644 index 000000000..9b83fd89b --- /dev/null +++ b/builder/openstack/step_stop_server.go @@ -0,0 +1,58 @@ +package openstack + +import ( + "fmt" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/startstop" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +type StepStopServer struct{} + +func (s *StepStopServer) Run(state multistep.StateBag) multistep.StepAction { + ui := state.Get("ui").(packer.Ui) + config := state.Get("config").(Config) + extensions := state.Get("extensions").(map[string]struct{}) + server := state.Get("server").(*servers.Server) + + // Verify we have the extension + if _, ok := extensions["os-server-start-stop"]; !ok { + ui.Say("OpenStack cluster doesn't support stop, skipping...") + return multistep.ActionContinue + } + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Say("Stopping server...") + if err := startstop.Stop(client, server.ID).ExtractErr(); err != nil { + err = fmt.Errorf("Error stopping server: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Message("Waiting for server to stop...") + stateChange := StateChangeConf{ + Pending: []string{"ACTIVE"}, + Target: "STOPPED", + Refresh: ServerStateRefreshFunc(client, server), + StepState: state, + } + if _, err := WaitForState(&stateChange); err != nil { + err := fmt.Errorf("Error waiting for server (%s) to stop: %s", server.ID, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *StepStopServer) Cleanup(state multistep.StateBag) {} From b6d6a71c6e47b3e99e84c749372c11f227371df6 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 22:50:59 -0400 Subject: [PATCH 133/208] builder/openstack: wait for more states --- builder/openstack/server.go | 8 +++++--- builder/openstack/step_run_source_server.go | 6 +++--- builder/openstack/step_stop_server.go | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/builder/openstack/server.go b/builder/openstack/server.go index de8c9d103..482657c03 100644 --- a/builder/openstack/server.go +++ b/builder/openstack/server.go @@ -28,7 +28,7 @@ type StateChangeConf struct { Pending []string Refresh StateRefreshFunc StepState multistep.StateBag - Target string + Target []string } // ServerStateRefreshFunc returns a StateRefreshFunc that is used to watch @@ -65,8 +65,10 @@ func WaitForState(conf *StateChangeConf) (i interface{}, err error) { return } - if currentState == conf.Target { - return + for _, t := range conf.Target { + if currentState == t { + return + } } if conf.StepState != nil { diff --git a/builder/openstack/step_run_source_server.go b/builder/openstack/step_run_source_server.go index 4014f1d95..596348def 100644 --- a/builder/openstack/step_run_source_server.go +++ b/builder/openstack/step_run_source_server.go @@ -65,7 +65,7 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction ui.Say("Waiting for server to become ready...") stateChange := StateChangeConf{ Pending: []string{"BUILD"}, - Target: "ACTIVE", + Target: []string{"ACTIVE"}, Refresh: ServerStateRefreshFunc(computeClient, s.server), StepState: state, } @@ -105,9 +105,9 @@ func (s *StepRunSourceServer) Cleanup(state multistep.StateBag) { } stateChange := StateChangeConf{ - Pending: []string{"ACTIVE", "BUILD", "REBUILD", "SUSPENDED"}, + Pending: []string{"ACTIVE", "BUILD", "REBUILD", "SUSPENDED", "SHUTOFF", "STOPPED"}, Refresh: ServerStateRefreshFunc(computeClient, s.server), - Target: "DELETED", + Target: []string{"DELETED"}, } WaitForState(&stateChange) diff --git a/builder/openstack/step_stop_server.go b/builder/openstack/step_stop_server.go index 9b83fd89b..298d0bc0a 100644 --- a/builder/openstack/step_stop_server.go +++ b/builder/openstack/step_stop_server.go @@ -41,7 +41,7 @@ func (s *StepStopServer) Run(state multistep.StateBag) multistep.StepAction { ui.Message("Waiting for server to stop...") stateChange := StateChangeConf{ Pending: []string{"ACTIVE"}, - Target: "STOPPED", + Target: []string{"SHUTOFF", "STOPPED"}, Refresh: ServerStateRefreshFunc(client, server), StepState: state, } From b3a97124023b48a36d05d2aa8fc1fa0def0ad0c9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 22:55:39 -0400 Subject: [PATCH 134/208] builder/openstack: support user data [GH-1867] --- builder/openstack/builder.go | 2 ++ builder/openstack/run_config.go | 2 ++ builder/openstack/step_run_source_server.go | 14 ++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index ab60afc0e..cf1cfc8c5 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -80,6 +80,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SecurityGroups: b.config.SecurityGroups, Networks: b.config.Networks, AvailabilityZone: b.config.AvailabilityZone, + UserData: b.config.UserData, + UserDataFile: b.config.UserDataFile, }, &StepWaitForRackConnect{ Wait: b.config.RackconnectWait, diff --git a/builder/openstack/run_config.go b/builder/openstack/run_config.go index 4a6a1b81f..ca0360a10 100644 --- a/builder/openstack/run_config.go +++ b/builder/openstack/run_config.go @@ -23,6 +23,8 @@ type RunConfig struct { FloatingIp string `mapstructure:"floating_ip"` SecurityGroups []string `mapstructure:"security_groups"` Networks []string `mapstructure:"networks"` + UserData string `mapstructure:"user_data"` + UserDataFile string `mapstructure:"user_data_file"` // Not really used, but here for BC OpenstackProvider string `mapstructure:"openstack_provider"` diff --git a/builder/openstack/step_run_source_server.go b/builder/openstack/step_run_source_server.go index 4014f1d95..9e55dea0e 100644 --- a/builder/openstack/step_run_source_server.go +++ b/builder/openstack/step_run_source_server.go @@ -2,6 +2,7 @@ package openstack import ( "fmt" + "io/ioutil" "log" "github.com/mitchellh/multistep" @@ -16,6 +17,8 @@ type StepRunSourceServer struct { SecurityGroups []string Networks []string AvailabilityZone string + UserData string + UserDataFile string server *servers.Server } @@ -39,6 +42,16 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction networks[i].UUID = networkUuid } + userData := []byte(s.UserData) + if s.UserDataFile != "" { + userData, err = ioutil.ReadFile(s.UserDataFile) + if err != nil { + err = fmt.Errorf("Error reading user data file: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + } + ui.Say("Launching server...") s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ CreateOptsBuilder: servers.CreateOpts{ @@ -48,6 +61,7 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction SecurityGroups: s.SecurityGroups, Networks: networks, AvailabilityZone: s.AvailabilityZone, + UserData: userData, }, KeyName: keyName, From 0dd80c0eca33164dbbd7749533f3a2318e2675eb Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 10:17:42 -0400 Subject: [PATCH 135/208] config file doesn't need to exist if set [GH-2225] --- main.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/main.go b/main.go index d00336283..aab28c141 100644 --- a/main.go +++ b/main.go @@ -217,12 +217,10 @@ func loadConfig() (*config, error) { return nil, err } - mustExist := true configFilePath := os.Getenv("PACKER_CONFIG") if configFilePath == "" { var err error configFilePath, err = configFile() - mustExist = false if err != nil { log.Printf("Error detecting default config file path: %s", err) @@ -240,11 +238,7 @@ func loadConfig() (*config, error) { return nil, err } - if mustExist { - return nil, err - } - - log.Println("File doesn't exist, but doesn't need to. Ignoring.") + log.Println("[WARN] Config file doesn't exist: %s", configFilePath) return &config, nil } defer f.Close() From 1e853f9f1fcf36c54114db39039a50df7d4271c2 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 10:20:54 -0400 Subject: [PATCH 136/208] common: revert some changes from #2121 for Windows --- common/config.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/common/config.go b/common/config.go index ebbfa6d8c..e6c6477b5 100644 --- a/common/config.go +++ b/common/config.go @@ -99,6 +99,14 @@ func DownloadableURL(original string) (string, error) { // Make sure it is lowercased url.Scheme = strings.ToLower(url.Scheme) + // This is to work around issue #5927. This can safely be removed once + // we distribute with a version of Go that fixes that bug. + // + // See: https://code.google.com/p/go/issues/detail?id=5927 + if url.Path != "" && url.Path[0] != '/' { + url.Path = "/" + url.Path + } + // Verify that the scheme is something we support in our common downloader. supported := []string{"file", "http", "https"} found := false From 1999c83a0c00ec7e1ad6a06ce76b544ab2e7897f Mon Sep 17 00:00:00 2001 From: Emil Hessman Date: Sat, 13 Jun 2015 18:58:13 +0200 Subject: [PATCH 137/208] post-processor/atlas: adjust test for cross-platform filepath separator Make TestLongestCommonPrefix cross-platform friendly by defining the test cases with filepath.Separator. Fixes test failure on Windows. --- post-processor/atlas/util_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/post-processor/atlas/util_test.go b/post-processor/atlas/util_test.go index b6b9da3d9..d0bed0a5d 100644 --- a/post-processor/atlas/util_test.go +++ b/post-processor/atlas/util_test.go @@ -1,10 +1,12 @@ package atlas import ( + "path/filepath" "testing" ) func TestLongestCommonPrefix(t *testing.T) { + sep := string(filepath.Separator) cases := []struct { Input []string Output string @@ -18,12 +20,12 @@ func TestLongestCommonPrefix(t *testing.T) { "", }, { - []string{"foo/", "foo/bar"}, - "foo/", + []string{"foo" + sep, "foo" + sep + "bar"}, + "foo" + sep, }, { - []string{"/foo/", "/bar"}, - "/", + []string{sep + "foo" + sep, sep + "bar"}, + sep, }, } From 1bcb52a093e79e66ecc1370f6567757c7912bab0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 13:56:09 -0400 Subject: [PATCH 138/208] command/fix: validate resulting template [GH-2075] --- command/fix.go | 29 ++++++++++ command/fix_test.go | 58 +++++++++++++++++++ .../test-fixtures/fix-invalid/template.json | 3 + command/test-fixtures/fix/template.json | 7 +++ 4 files changed, 97 insertions(+) create mode 100644 command/fix_test.go create mode 100644 command/test-fixtures/fix-invalid/template.json create mode 100644 command/test-fixtures/fix/template.json diff --git a/command/fix.go b/command/fix.go index e908dc52e..2d9bcce37 100644 --- a/command/fix.go +++ b/command/fix.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/mitchellh/packer/fix" + "github.com/mitchellh/packer/template" ) type FixCommand struct { @@ -16,7 +17,9 @@ type FixCommand struct { } func (c *FixCommand) Run(args []string) int { + var flagValidate bool flags := c.Meta.FlagSet("fix", FlagSetNone) + flags.BoolVar(&flagValidate, "validate", true, "") flags.Usage = func() { c.Ui.Say(c.Help()) } if err := flags.Parse(args); err != nil { return 1 @@ -80,6 +83,28 @@ func (c *FixCommand) Run(args []string) int { result = strings.Replace(result, `\u003c`, "<", -1) result = strings.Replace(result, `\u003e`, ">", -1) c.Ui.Say(result) + + if flagValidate { + // Attemot to parse and validate the template + tpl, err := template.Parse(strings.NewReader(result)) + if err != nil { + c.Ui.Error(fmt.Sprintf( + "Error! Fixed template fails to parse: %s\n\n"+ + "This is usually caused by an error in the input template.\n"+ + "Please fix the error and try again.", + err)) + return 1 + } + if err := tpl.Validate(); err != nil { + c.Ui.Error(fmt.Sprintf( + "Error! Fixed template failed to validate: %s\n\n"+ + "This is usually caused by an error in the input template.\n"+ + "Please fix the error and try again.", + err)) + return 1 + } + } + return 0 } @@ -102,6 +127,10 @@ Fixes that are run: pp-vagrant-override Replaces old-style provider overrides for the Vagrant post-processor to new-style as of Packer 0.5.0. virtualbox-rename Updates "virtualbox" builders to "virtualbox-iso" + +Options: + + -validate=true If true (default), validates the fixed template. ` return strings.TrimSpace(helpText) diff --git a/command/fix_test.go b/command/fix_test.go new file mode 100644 index 000000000..1bf6f1900 --- /dev/null +++ b/command/fix_test.go @@ -0,0 +1,58 @@ +package command + +import ( + "path/filepath" + "testing" +) + +func TestFix_noArgs(t *testing.T) { + c := &PushCommand{Meta: testMeta(t)} + code := c.Run(nil) + if code != 1 { + t.Fatalf("bad: %#v", code) + } +} + +func TestFix_multiArgs(t *testing.T) { + c := &PushCommand{Meta: testMeta(t)} + code := c.Run([]string{"one", "two"}) + if code != 1 { + t.Fatalf("bad: %#v", code) + } +} + +func TestFix(t *testing.T) { + c := &FixCommand{ + Meta: testMeta(t), + } + + args := []string{filepath.Join(testFixture("fix"), "template.json")} + if code := c.Run(args); code != 0 { + fatalCommand(t, c.Meta) + } +} + +func TestFix_invalidTemplate(t *testing.T) { + c := &FixCommand{ + Meta: testMeta(t), + } + + args := []string{filepath.Join(testFixture("fix-invalid"), "template.json")} + if code := c.Run(args); code != 1 { + fatalCommand(t, c.Meta) + } +} + +func TestFix_invalidTemplateDisableValidation(t *testing.T) { + c := &FixCommand{ + Meta: testMeta(t), + } + + args := []string{ + "-validate=false", + filepath.Join(testFixture("fix-invalid"), "template.json"), + } + if code := c.Run(args); code != 0 { + fatalCommand(t, c.Meta) + } +} diff --git a/command/test-fixtures/fix-invalid/template.json b/command/test-fixtures/fix-invalid/template.json new file mode 100644 index 000000000..ea50c5dcf --- /dev/null +++ b/command/test-fixtures/fix-invalid/template.json @@ -0,0 +1,3 @@ +{ + "hello": "world" +} diff --git a/command/test-fixtures/fix/template.json b/command/test-fixtures/fix/template.json new file mode 100644 index 000000000..63b0f9037 --- /dev/null +++ b/command/test-fixtures/fix/template.json @@ -0,0 +1,7 @@ +{ + "builders": [{"type": "dummy"}], + + "push": { + "name": "foo/bar" + } +} From f1cef0baaee6a572831e9f6e61a983f32126ed2a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 14:07:43 -0400 Subject: [PATCH 139/208] builder/null: fix config parsing --- builder/null/config.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/builder/null/config.go b/builder/null/config.go index 9ccc32282..aa3f15120 100644 --- a/builder/null/config.go +++ b/builder/null/config.go @@ -2,6 +2,7 @@ package null import ( "fmt" + "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" @@ -19,9 +20,9 @@ type Config struct { } func NewConfig(raws ...interface{}) (*Config, []string, error) { - c := new(Config) + var c Config - err := config.Decode(c, &config.DecodeOpts{ + err := config.Decode(&c, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ @@ -62,5 +63,5 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { return nil, nil, errs } - return c, nil, nil + return &c, nil, nil } From c549fce85e9e09be6c072bfdbc44a430219663b1 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 14:11:20 -0400 Subject: [PATCH 140/208] provisioner/shell: escape single quotes [GH-2067] --- provisioner/shell/provisioner.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index 212ec2abf..baedd645a 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -145,6 +145,9 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { errs = packer.MultiErrorAppend(errs, fmt.Errorf("Environment variable not in format 'key=value': %s", kv)) } else { + // Replace single quotes so they parse + vs[1] = strings.Replace(vs[1], "'", `'"'"'`, -1) + // Single quote env var values p.config.Vars[idx] = fmt.Sprintf("%s='%s'", vs[0], vs[1]) } From facbb6577d060cd841c9c761d5e7320539640743 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 16:19:25 -0400 Subject: [PATCH 141/208] template: allow _ prefix to root level keys for comments [GH-2066] --- template/parse.go | 9 +++++++-- template/parse_test.go | 13 +++++++++++++ template/test-fixtures/parse-comment.json | 4 ++++ .../docs/templates/introduction.html.markdown | 16 ++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 template/test-fixtures/parse-comment.json diff --git a/template/parse.go b/template/parse.go index dbb29569d..f64057db2 100644 --- a/template/parse.go +++ b/template/parse.go @@ -291,11 +291,16 @@ func Parse(r io.Reader) (*Template, error) { if len(md.Unused) > 0 { sort.Strings(md.Unused) for _, unused := range md.Unused { + // Ignore keys starting with '_' as comments + if unused[0] == '_' { + continue + } + err = multierror.Append(err, fmt.Errorf( "Unknown root level key in template: '%s'", unused)) } - - // Return early for these errors + } + if err != nil { return nil, err } diff --git a/template/parse_test.go b/template/parse_test.go index 9abca2f77..5285e27d3 100644 --- a/template/parse_test.go +++ b/template/parse_test.go @@ -303,6 +303,19 @@ func TestParse(t *testing.T) { }, false, }, + + { + "parse-comment.json", + &Template{ + Builders: map[string]*Builder{ + "something": &Builder{ + Name: "something", + Type: "something", + }, + }, + }, + false, + }, } for _, tc := range cases { diff --git a/template/test-fixtures/parse-comment.json b/template/test-fixtures/parse-comment.json new file mode 100644 index 000000000..d3bb95a3b --- /dev/null +++ b/template/test-fixtures/parse-comment.json @@ -0,0 +1,4 @@ +{ + "_info": "foo", + "builders": [{"type": "something"}] +} diff --git a/website/source/docs/templates/introduction.html.markdown b/website/source/docs/templates/introduction.html.markdown index a8be4f592..3dc363916 100644 --- a/website/source/docs/templates/introduction.html.markdown +++ b/website/source/docs/templates/introduction.html.markdown @@ -58,6 +58,22 @@ Along with each key, it is noted whether it is required or not. For more information on how to define and use user variables, read the sub-section on [user variables in templates](/docs/templates/user-variables.html). +## Comments + +JSON doesn't support comments and Packer reports unknown keys as validation +errors. If you'd like to comment your template, you can prefix a _root level_ +key with an underscore. Example: + +```javascript +{ + "_comment": "This is a comment", + "builders": [{}] +} +``` + +**Important:** Only _root level_ keys can be underscore prefixed. Keys within +builders, provisioners, etc. will still result in validation errors. + ## Example Template Below is an example of a basic template that is nearly fully functional. It is just From 7eff6b117da1047004dd33fb209f7cf54422509b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 16:24:47 -0400 Subject: [PATCH 142/208] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b96154c2b..c933040c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ BACKWARDS INCOMPATIBILITIES: + * core: SSH connection will no longer request a PTY by default. This + can be enabled per builder. * builder/digitalocean: no longer supports the v1 API which has been deprecated for some time. Most configurations should continue to work as long as you use the `api_token` field for auth. From c3f54ba5a9650a39f89e1471154b1292db34a9c0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 16:39:12 -0400 Subject: [PATCH 143/208] fix: virtualbox rename fixes overrides [GH-1828] --- fix/fixer_virtualbox_rename.go | 40 ++++++++++++++++++++++++--- fix/fixer_virtualbox_rename_test.go | 42 +++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/fix/fixer_virtualbox_rename.go b/fix/fixer_virtualbox_rename.go index 292bf0bf1..34187ac2c 100644 --- a/fix/fixer_virtualbox_rename.go +++ b/fix/fixer_virtualbox_rename.go @@ -8,14 +8,14 @@ import ( type FixerVirtualBoxRename struct{} func (FixerVirtualBoxRename) Fix(input map[string]interface{}) (map[string]interface{}, error) { - // The type we'll decode into; we only care about builders type template struct { - Builders []map[string]interface{} + Builders []map[string]interface{} + Provisioners []interface{} } // Decode the input into our structure, if we can var tpl template - if err := mapstructure.Decode(input, &tpl); err != nil { + if err := mapstructure.WeakDecode(input, &tpl); err != nil { return nil, err } @@ -37,7 +37,39 @@ func (FixerVirtualBoxRename) Fix(input map[string]interface{}) (map[string]inter builder["type"] = "virtualbox-iso" } - input["builders"] = tpl.Builders + for i, raw := range tpl.Provisioners { + var m map[string]interface{} + if err := mapstructure.WeakDecode(raw, &m); err != nil { + // Ignore errors, could be a non-map + continue + } + + raw, ok := m["override"] + if !ok { + continue + } + + var override map[string]interface{} + if err := mapstructure.WeakDecode(raw, &override); err != nil { + return nil, err + } + + if raw, ok := override["virtualbox"]; ok { + override["virtualbox-iso"] = raw + delete(override, "virtualbox") + + // Set the change + m["override"] = override + tpl.Provisioners[i] = m + } + } + + if len(tpl.Builders) > 0 { + input["builders"] = tpl.Builders + } + if len(tpl.Provisioners) > 0 { + input["provisioners"] = tpl.Provisioners + } return input, nil } diff --git a/fix/fixer_virtualbox_rename_test.go b/fix/fixer_virtualbox_rename_test.go index 78b7bccf2..355e5276a 100644 --- a/fix/fixer_virtualbox_rename_test.go +++ b/fix/fixer_virtualbox_rename_test.go @@ -46,3 +46,45 @@ func TestFixerVirtualBoxRename_Fix(t *testing.T) { } } } + +func TestFixerVirtualBoxRenameFix_provisionerOverride(t *testing.T) { + cases := []struct { + Input map[string]interface{} + Expected map[string]interface{} + }{ + { + Input: map[string]interface{}{ + "provisioners": []interface{}{ + map[string]interface{}{ + "override": map[string]interface{}{ + "virtualbox": map[string]interface{}{}, + }, + }, + }, + }, + + Expected: map[string]interface{}{ + "provisioners": []interface{}{ + map[string]interface{}{ + "override": map[string]interface{}{ + "virtualbox-iso": map[string]interface{}{}, + }, + }, + }, + }, + }, + } + + for _, tc := range cases { + var f FixerVirtualBoxRename + + output, err := f.Fix(tc.Input) + if err != nil { + t.Fatalf("err: %s", err) + } + + if !reflect.DeepEqual(output, tc.Expected) { + t.Fatalf("unexpected:\n\n%#v\nexpected:\n\n%#v\n", output, tc.Expected) + } + } +} From f1b3c8a7ae49c4acdbb74c862956eb22538cfd17 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 16:48:35 -0400 Subject: [PATCH 144/208] template/interpolate: build_name and build_type functions --- template/interpolate/funcs.go | 22 +++++++++++++ template/interpolate/funcs_test.go | 50 ++++++++++++++++++++++++++++++ template/interpolate/i.go | 15 ++++++--- 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/template/interpolate/funcs.go b/template/interpolate/funcs.go index 6092707b8..e5d01b455 100644 --- a/template/interpolate/funcs.go +++ b/template/interpolate/funcs.go @@ -24,6 +24,8 @@ func init() { // Funcs are the interpolation funcs that are available within interpolations. var FuncGens = map[string]FuncGenerator{ + "build_name": funcGenBuildName, + "build_type": funcGenBuildType, "env": funcGenEnv, "isotime": funcGenIsotime, "pwd": funcGenPwd, @@ -56,6 +58,26 @@ func Funcs(ctx *Context) template.FuncMap { return template.FuncMap(result) } +func funcGenBuildName(ctx *Context) interface{} { + return func() (string, error) { + if ctx == nil || ctx.BuildName == "" { + return "", errors.New("build_name not available") + } + + return ctx.BuildName, nil + } +} + +func funcGenBuildType(ctx *Context) interface{} { + return func() (string, error) { + if ctx == nil || ctx.BuildType == "" { + return "", errors.New("build_name not available") + } + + return ctx.BuildType, nil + } +} + func funcGenEnv(ctx *Context) interface{} { return func(k string) (string, error) { if !ctx.EnableEnv { diff --git a/template/interpolate/funcs_test.go b/template/interpolate/funcs_test.go index ff877f13e..065942c93 100644 --- a/template/interpolate/funcs_test.go +++ b/template/interpolate/funcs_test.go @@ -8,6 +8,56 @@ import ( "time" ) +func TestFuncBuildName(t *testing.T) { + cases := []struct { + Input string + Output string + }{ + { + `{{build_name}}`, + "foo", + }, + } + + ctx := &Context{BuildName: "foo"} + for _, tc := range cases { + i := &I{Value: tc.Input} + result, err := i.Render(ctx) + if err != nil { + t.Fatalf("Input: %s\n\nerr: %s", tc.Input, err) + } + + if result != tc.Output { + t.Fatalf("Input: %s\n\nGot: %s", tc.Input, result) + } + } +} + +func TestFuncBuildType(t *testing.T) { + cases := []struct { + Input string + Output string + }{ + { + `{{build_type}}`, + "foo", + }, + } + + ctx := &Context{BuildType: "foo"} + for _, tc := range cases { + i := &I{Value: tc.Input} + result, err := i.Render(ctx) + if err != nil { + t.Fatalf("Input: %s\n\nerr: %s", tc.Input, err) + } + + if result != tc.Output { + t.Fatalf("Input: %s\n\nGot: %s", tc.Input, result) + } + } +} + func TestFuncEnv(t *testing.T) { cases := []struct { Input string diff --git a/template/interpolate/i.go b/template/interpolate/i.go index d5f7c8413..02f56197a 100644 --- a/template/interpolate/i.go +++ b/template/interpolate/i.go @@ -14,16 +14,23 @@ type Context struct { // Funcs are extra functions available in the template Funcs map[string]interface{} - // TemplatePath is the path to the template that this is being - // rendered within. - TemplatePath string - // UserVariables is the mapping of user variables that the // "user" function reads from. UserVariables map[string]string // EnableEnv enables the env function EnableEnv bool + + // All the fields below are used for built-in functions. + // + // BuildName and BuildType are the name and type, respectively, + // of the builder being used. + // + // TemplatePath is the path to the template that this is being + // rendered within. + BuildName string + BuildType string + TemplatePath string } // Render is shorthand for constructing an I and calling Render. From 472b060394622a1b9c25c0570718a0cba3d691a7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 16:51:29 -0400 Subject: [PATCH 145/208] packer: build_name and build_type work + tests --- helper/config/decode.go | 4 ++ packer/core_test.go | 58 +++++++++++++++++++ .../test-fixtures/build-var-build-name.json | 6 ++ .../test-fixtures/build-var-build-type.json | 6 ++ 4 files changed, 74 insertions(+) create mode 100644 packer/test-fixtures/build-var-build-name.json create mode 100644 packer/test-fixtures/build-var-build-type.json diff --git a/helper/config/decode.go b/helper/config/decode.go index 0148ad27c..ccb71ac7f 100644 --- a/helper/config/decode.go +++ b/helper/config/decode.go @@ -104,6 +104,8 @@ func Decode(target interface{}, config *DecodeOpts, raws ...interface{}) error { // detecting things like user variables from the raw configuration params. func DetectContext(raws ...interface{}) (*interpolate.Context, error) { var s struct { + BuildName string `mapstructure:"packer_build_name"` + BuildType string `mapstructure:"packer_builder_type"` TemplatePath string `mapstructure:"packer_template_path"` Vars map[string]string `mapstructure:"packer_user_variables"` } @@ -115,6 +117,8 @@ func DetectContext(raws ...interface{}) (*interpolate.Context, error) { } return &interpolate.Context{ + BuildName: s.BuildName, + BuildType: s.BuildType, TemplatePath: s.TemplatePath, UserVariables: s.Vars, }, nil diff --git a/packer/core_test.go b/packer/core_test.go index f11242d0c..cc958356e 100644 --- a/packer/core_test.go +++ b/packer/core_test.go @@ -142,6 +142,64 @@ func TestCoreBuild_env(t *testing.T) { } } +func TestCoreBuild_buildNameVar(t *testing.T) { + config := TestCoreConfig(t) + testCoreTemplate(t, config, fixtureDir("build-var-build-name.json")) + b := TestBuilder(t, config, "test") + core := TestCore(t, config) + + b.ArtifactId = "hello" + + build, err := core.Build("test") + if err != nil { + t.Fatalf("err: %s", err) + } + + if _, err := build.Prepare(); err != nil { + t.Fatalf("err: %s", err) + } + + // Interpolate the config + var result map[string]interface{} + err = configHelper.Decode(&result, nil, b.PrepareConfig...) + if err != nil { + t.Fatalf("err: %s", err) + } + + if result["value"] != "test" { + t.Fatalf("bad: %#v", result) + } +} + +func TestCoreBuild_buildTypeVar(t *testing.T) { + config := TestCoreConfig(t) + testCoreTemplate(t, config, fixtureDir("build-var-build-type.json")) + b := TestBuilder(t, config, "test") + core := TestCore(t, config) + + b.ArtifactId = "hello" + + build, err := core.Build("test") + if err != nil { + t.Fatalf("err: %s", err) + } + + if _, err := build.Prepare(); err != nil { + t.Fatalf("err: %s", err) + } + + // Interpolate the config + var result map[string]interface{} + err = configHelper.Decode(&result, nil, b.PrepareConfig...) + if err != nil { + t.Fatalf("err: %s", err) + } + + if result["value"] != "test" { + t.Fatalf("bad: %#v", result) + } +} + func TestCoreBuild_nonExist(t *testing.T) { config := TestCoreConfig(t) testCoreTemplate(t, config, fixtureDir("build-basic.json")) diff --git a/packer/test-fixtures/build-var-build-name.json b/packer/test-fixtures/build-var-build-name.json new file mode 100644 index 000000000..da43d9ad9 --- /dev/null +++ b/packer/test-fixtures/build-var-build-name.json @@ -0,0 +1,6 @@ +{ + "builders": [{ + "type": "test", + "value": "{{build_name}}" + }] +} diff --git a/packer/test-fixtures/build-var-build-type.json b/packer/test-fixtures/build-var-build-type.json new file mode 100644 index 000000000..3d925e406 --- /dev/null +++ b/packer/test-fixtures/build-var-build-type.json @@ -0,0 +1,6 @@ +{ + "builders": [{ + "type": "test", + "value": "{{build_type}}" + }] +} From 3976a34d29c125f4c92c8a9d31d469446dcce75f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 16:58:37 -0400 Subject: [PATCH 146/208] builder/virtualbox: validate output dir in step, no in config --- builder/virtualbox/common/output_config.go | 11 +-------- .../virtualbox/common/output_config_test.go | 22 +---------------- builder/virtualbox/common/step_output_dir.go | 11 ++++++++- .../virtualbox/common/step_output_dir_test.go | 24 +++++++++++++++++++ 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/builder/virtualbox/common/output_config.go b/builder/virtualbox/common/output_config.go index f3427183c..7b5ddcd45 100644 --- a/builder/virtualbox/common/output_config.go +++ b/builder/virtualbox/common/output_config.go @@ -2,7 +2,6 @@ package common import ( "fmt" - "os" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/template/interpolate" @@ -17,13 +16,5 @@ func (c *OutputConfig) Prepare(ctx *interpolate.Context, pc *common.PackerConfig c.OutputDir = fmt.Sprintf("output-%s", pc.PackerBuildName) } - var errs []error - if !pc.PackerForce { - if _, err := os.Stat(c.OutputDir); err == nil { - errs = append(errs, fmt.Errorf( - "Output directory '%s' already exists. It must not exist.", c.OutputDir)) - } - } - - return errs + return nil } diff --git a/builder/virtualbox/common/output_config_test.go b/builder/virtualbox/common/output_config_test.go index 7fa039a16..a4d8e7999 100644 --- a/builder/virtualbox/common/output_config_test.go +++ b/builder/virtualbox/common/output_config_test.go @@ -39,27 +39,7 @@ func TestOutputConfigPrepare_exists(t *testing.T) { PackerForce: false, } errs := c.Prepare(testConfigTemplate(t), pc) - if len(errs) == 0 { - t.Fatal("should have errors") - } -} - -func TestOutputConfigPrepare_forceExists(t *testing.T) { - td, err := ioutil.TempDir("", "packer") - if err != nil { - t.Fatalf("err: %s", err) - } - defer os.RemoveAll(td) - - c := new(OutputConfig) - c.OutputDir = td - - pc := &common.PackerConfig{ - PackerBuildName: "foo", - PackerForce: true, - } - errs := c.Prepare(testConfigTemplate(t), pc) - if len(errs) > 0 { + if len(errs) != 0 { t.Fatal("should not have errors") } } diff --git a/builder/virtualbox/common/step_output_dir.go b/builder/virtualbox/common/step_output_dir.go index 209bbabe2..e01928b7a 100644 --- a/builder/virtualbox/common/step_output_dir.go +++ b/builder/virtualbox/common/step_output_dir.go @@ -22,7 +22,16 @@ type StepOutputDir struct { func (s *StepOutputDir) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) - if _, err := os.Stat(s.Path); err == nil && s.Force { + if _, err := os.Stat(s.Path); err == nil { + if !s.Force { + err := fmt.Errorf( + "Output directory exists: %s\n\n"+ + "Use the force flag to delete it prior to building.", + s.Path) + state.Put("error", err) + return multistep.ActionHalt + } + ui.Say("Deleting previous output directory...") os.RemoveAll(s.Path) } diff --git a/builder/virtualbox/common/step_output_dir_test.go b/builder/virtualbox/common/step_output_dir_test.go index be485c278..77d1f855f 100644 --- a/builder/virtualbox/common/step_output_dir_test.go +++ b/builder/virtualbox/common/step_output_dir_test.go @@ -45,6 +45,30 @@ func TestStepOutputDir(t *testing.T) { } } +func TestStepOutputDir_exists(t *testing.T) { + state := testState(t) + step := testStepOutputDir(t) + + // Make the dir + if err := os.MkdirAll(step.Path, 0755); err != nil { + t.Fatalf("bad: %s", err) + } + + // Test the run + if action := step.Run(state); action != multistep.ActionHalt { + t.Fatalf("bad action: %#v", action) + } + if _, ok := state.GetOk("error"); !ok { + t.Fatal("should have error") + } + + // Test the cleanup + step.Cleanup(state) + if _, err := os.Stat(step.Path); err != nil { + t.Fatalf("err: %s", err) + } +} + func TestStepOutputDir_cancelled(t *testing.T) { state := testState(t) step := testStepOutputDir(t) From d8518981317b1afea2885948f56606bb8687f990 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 17:00:14 -0400 Subject: [PATCH 147/208] builder/vmware: mirror virtualbox output dir changes --- builder/vmware/common/output_config.go | 11 +---- builder/vmware/common/output_config_test.go | 45 +-------------------- builder/vmware/iso/builder_test.go | 4 +- 3 files changed, 5 insertions(+), 55 deletions(-) diff --git a/builder/vmware/common/output_config.go b/builder/vmware/common/output_config.go index f3427183c..7b5ddcd45 100644 --- a/builder/vmware/common/output_config.go +++ b/builder/vmware/common/output_config.go @@ -2,7 +2,6 @@ package common import ( "fmt" - "os" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/template/interpolate" @@ -17,13 +16,5 @@ func (c *OutputConfig) Prepare(ctx *interpolate.Context, pc *common.PackerConfig c.OutputDir = fmt.Sprintf("output-%s", pc.PackerBuildName) } - var errs []error - if !pc.PackerForce { - if _, err := os.Stat(c.OutputDir); err == nil { - errs = append(errs, fmt.Errorf( - "Output directory '%s' already exists. It must not exist.", c.OutputDir)) - } - } - - return errs + return nil } diff --git a/builder/vmware/common/output_config_test.go b/builder/vmware/common/output_config_test.go index 7fa039a16..7de378d89 100644 --- a/builder/vmware/common/output_config_test.go +++ b/builder/vmware/common/output_config_test.go @@ -1,10 +1,9 @@ package common import ( - "github.com/mitchellh/packer/common" - "io/ioutil" - "os" "testing" + + "github.com/mitchellh/packer/common" ) func TestOutputConfigPrepare(t *testing.T) { @@ -23,43 +22,3 @@ func TestOutputConfigPrepare(t *testing.T) { t.Fatal("should have output dir") } } - -func TestOutputConfigPrepare_exists(t *testing.T) { - td, err := ioutil.TempDir("", "packer") - if err != nil { - t.Fatalf("err: %s", err) - } - defer os.RemoveAll(td) - - c := new(OutputConfig) - c.OutputDir = td - - pc := &common.PackerConfig{ - PackerBuildName: "foo", - PackerForce: false, - } - errs := c.Prepare(testConfigTemplate(t), pc) - if len(errs) == 0 { - t.Fatal("should have errors") - } -} - -func TestOutputConfigPrepare_forceExists(t *testing.T) { - td, err := ioutil.TempDir("", "packer") - if err != nil { - t.Fatalf("err: %s", err) - } - defer os.RemoveAll(td) - - c := new(OutputConfig) - c.OutputDir = td - - pc := &common.PackerConfig{ - PackerBuildName: "foo", - PackerForce: true, - } - errs := c.Prepare(testConfigTemplate(t), pc) - if len(errs) > 0 { - t.Fatal("should not have errors") - } -} diff --git a/builder/vmware/iso/builder_test.go b/builder/vmware/iso/builder_test.go index 1749b396e..bf658938b 100644 --- a/builder/vmware/iso/builder_test.go +++ b/builder/vmware/iso/builder_test.go @@ -353,8 +353,8 @@ func TestBuilderPrepare_OutputDir(t *testing.T) { if len(warns) > 0 { t.Fatalf("bad: %#v", warns) } - if err == nil { - t.Fatal("should have error") + if err != nil { + t.Fatalf("err: %s", err) } // Test with a good one From 511013dbe4b845dcb2898f37852a2e260162c9f9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 17:05:38 -0400 Subject: [PATCH 148/208] fix go vet warning --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index aab28c141..7f5cb7bef 100644 --- a/main.go +++ b/main.go @@ -238,7 +238,7 @@ func loadConfig() (*config, error) { return nil, err } - log.Println("[WARN] Config file doesn't exist: %s", configFilePath) + log.Printf("[WARN] Config file doesn't exist: %s", configFilePath) return &config, nil } defer f.Close() From 4b3ed5d7e2f61aaa1c02951aad97d5ec33c46005 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 17:42:38 -0400 Subject: [PATCH 149/208] helper/communicator --- helper/communicator/config.go | 38 ++++++ helper/communicator/step_connect.go | 44 +++++++ helper/communicator/step_connect_ssh.go | 153 ++++++++++++++++++++++++ 3 files changed, 235 insertions(+) create mode 100644 helper/communicator/config.go create mode 100644 helper/communicator/step_connect.go create mode 100644 helper/communicator/step_connect_ssh.go diff --git a/helper/communicator/config.go b/helper/communicator/config.go new file mode 100644 index 000000000..2500a7d2d --- /dev/null +++ b/helper/communicator/config.go @@ -0,0 +1,38 @@ +package communicator + +import ( + "errors" + "time" + + "github.com/mitchellh/packer/template/interpolate" +) + +// Config is the common configuration that communicators allow within +// a builder. +type Config struct { + SSHHost string `mapstructure:"ssh_host"` + SSHPort int `mapstructure:"ssh_port"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPassword string `mapstructure:"ssh_password"` + SSHPrivateKey string `mapstructure:"ssh_private_key_file"` + SSHPty bool `mapstructure:"ssh_pty"` + SSHTimeout time.Duration `mapstructure:"ssh_timeout"` +} + +func (c *Config) Prepare(ctx *interpolate.Context) []error { + if c.SSHPort == 0 { + c.SSHPort = 22 + } + + if c.SSHTimeout == 0 { + c.SSHTimeout = 5 * time.Minute + } + + // Validation + var errs []error + if c.SSHUsername == "" { + errs = append(errs, errors.New("An ssh_username must be specified")) + } + + return errs +} diff --git a/helper/communicator/step_connect.go b/helper/communicator/step_connect.go new file mode 100644 index 000000000..77feebfb9 --- /dev/null +++ b/helper/communicator/step_connect.go @@ -0,0 +1,44 @@ +package communicator + +import ( + "github.com/mitchellh/multistep" + gossh "golang.org/x/crypto/ssh" +) + +// StepConnect is a multistep Step implementation that connects to +// the proper communicator and stores it in the "communicator" key in the +// state bag. +type StepConnect struct { + // Config is the communicator config struct + Config *Config + + // The fields below are callbacks to assist with connecting to SSH. + // + // SSHAddress should return the default host to connect to for SSH. + // This is only called if ssh_host isn't specified in the config. + // + // SSHConfig should return the default configuration for + // connecting via SSH. + SSHAddress func(multistep.StateBag) (string, error) + SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error) + + substep multistep.Step +} + +func (s *StepConnect) Run(state multistep.StateBag) multistep.StepAction { + // Eventually we might switch between multiple of these depending + // on the communicator type. + s.substep = &StepConnectSSH{ + Config: s.Config, + SSHAddress: s.SSHAddress, + SSHConfig: s.SSHConfig, + } + + return s.substep.Run(state) +} + +func (s *StepConnect) Cleanup(state multistep.StateBag) { + if s.substep != nil { + s.substep.Cleanup(state) + } +} diff --git a/helper/communicator/step_connect_ssh.go b/helper/communicator/step_connect_ssh.go new file mode 100644 index 000000000..9be653c01 --- /dev/null +++ b/helper/communicator/step_connect_ssh.go @@ -0,0 +1,153 @@ +package communicator + +import ( + "errors" + "fmt" + "log" + "strings" + "time" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/communicator/ssh" + "github.com/mitchellh/packer/packer" + gossh "golang.org/x/crypto/ssh" +) + +// StepConnectSSH is a step that only connects to SSH. +// +// In general, you should use StepConnect. +type StepConnectSSH struct { + // All the fields below are documented on StepConnect + Config *Config + SSHAddress func(multistep.StateBag) (string, error) + SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error) +} + +func (s *StepConnectSSH) Run(state multistep.StateBag) multistep.StepAction { + ui := state.Get("ui").(packer.Ui) + + var comm packer.Communicator + var err error + + cancel := make(chan struct{}) + waitDone := make(chan bool, 1) + go func() { + ui.Say("Waiting for SSH to become available...") + comm, err = s.waitForSSH(state, cancel) + waitDone <- true + }() + + log.Printf("[INFO] Waiting for SSH, up to timeout: %s", s.Config.SSHTimeout) + timeout := time.After(s.Config.SSHTimeout) +WaitLoop: + for { + // Wait for either SSH to become available, a timeout to occur, + // or an interrupt to come through. + select { + case <-waitDone: + if err != nil { + ui.Error(fmt.Sprintf("Error waiting for SSH: %s", err)) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Say("Connected to SSH!") + state.Put("communicator", comm) + break WaitLoop + case <-timeout: + err := fmt.Errorf("Timeout waiting for SSH.") + state.Put("error", err) + ui.Error(err.Error()) + close(cancel) + return multistep.ActionHalt + case <-time.After(1 * time.Second): + if _, ok := state.GetOk(multistep.StateCancelled); ok { + // The step sequence was cancelled, so cancel waiting for SSH + // and just start the halting process. + close(cancel) + log.Println("[WARN] Interrupt detected, quitting waiting for SSH.") + return multistep.ActionHalt + } + } + } + + return multistep.ActionContinue +} + +func (s *StepConnectSSH) Cleanup(multistep.StateBag) { +} + +func (s *StepConnectSSH) waitForSSH(state multistep.StateBag, cancel <-chan struct{}) (packer.Communicator, error) { + handshakeAttempts := 0 + + var comm packer.Communicator + first := true + for { + // Don't check for cancel or wait on first iteration + if !first { + select { + case <-cancel: + log.Println("[DEBUG] SSH wait cancelled. Exiting loop.") + return nil, errors.New("SSH wait cancelled") + case <-time.After(5 * time.Second): + } + } + first = false + + // First we request the TCP connection information + address, err := s.SSHAddress(state) + if err != nil { + log.Printf("[DEBUG] Error getting SSH address: %s", err) + continue + } + + // Retrieve the SSH configuration + sshConfig, err := s.SSHConfig(state) + if err != nil { + log.Printf("[DEBUG] Error getting SSH config: %s", err) + continue + } + + // Attempt to connect to SSH port + connFunc := ssh.ConnectFunc("tcp", address) + nc, err := connFunc() + if err != nil { + log.Printf("[DEBUG] TCP connection to SSH ip/port failed: %s", err) + continue + } + nc.Close() + + // Then we attempt to connect via SSH + config := &ssh.Config{ + Connection: connFunc, + SSHConfig: sshConfig, + Pty: s.Config.SSHPty, + } + + log.Println("[INFO] Attempting SSH connection...") + comm, err = ssh.New(address, config) + if err != nil { + log.Printf("[DEBUG] SSH handshake err: %s", err) + + // Only count this as an attempt if we were able to attempt + // to authenticate. Note this is very brittle since it depends + // on the string of the error... but I don't see any other way. + if strings.Contains(err.Error(), "authenticate") { + log.Printf( + "[DEBUG] Detected authentication error. Increasing handshake attempts.") + handshakeAttempts += 1 + } + + if handshakeAttempts < 10 { + // Try to connect via SSH a handful of times + continue + } + + return nil, err + } + + break + } + + return comm, nil +} From d545431f9bc735f5200489917512da5230ec7418 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 17:42:43 -0400 Subject: [PATCH 150/208] builder/null: adopt helper/communicator --- builder/null/builder.go | 17 +++++++++++------ builder/null/config.go | 30 +++++++++++------------------- builder/null/config_test.go | 10 +++++----- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/builder/null/builder.go b/builder/null/builder.go index 7ca1b57fd..925075ee0 100644 --- a/builder/null/builder.go +++ b/builder/null/builder.go @@ -1,11 +1,12 @@ package null import ( + "log" + "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/packer" - "log" - "time" ) const BuilderId = "fnoeding.null" @@ -27,10 +28,14 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { steps := []multistep.Step{ - &common.StepConnectSSH{ - SSHAddress: SSHAddress(b.config.Host, b.config.Port), - SSHConfig: SSHConfig(b.config.SSHUsername, b.config.SSHPassword, b.config.SSHPrivateKeyFile), - SSHWaitTimeout: 1 * time.Minute, + &communicator.StepConnect{ + Config: &b.config.CommConfig, + SSHAddress: SSHAddress( + b.config.CommConfig.SSHHost, b.config.CommConfig.SSHPort), + SSHConfig: SSHConfig( + b.config.CommConfig.SSHUsername, + b.config.CommConfig.SSHPassword, + b.config.CommConfig.SSHPrivateKey), }, &common.StepProvision{}, } diff --git a/builder/null/config.go b/builder/null/config.go index aa3f15120..a6a12332e 100644 --- a/builder/null/config.go +++ b/builder/null/config.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -12,49 +13,40 @@ import ( type Config struct { common.PackerConfig `mapstructure:",squash"` - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPassword string `mapstructure:"ssh_password"` - SSHPrivateKeyFile string `mapstructure:"ssh_private_key_file"` + CommConfig communicator.Config `mapstructure:",squash"` } func NewConfig(raws ...interface{}) (*Config, []string, error) { var c Config err := config.Decode(&c, &config.DecodeOpts{ - Interpolate: true, - InterpolateFilter: &interpolate.RenderFilter{ - Exclude: []string{ - "run_command", - }, - }, + Interpolate: true, + InterpolateFilter: &interpolate.RenderFilter{}, }, raws...) if err != nil { return nil, nil, err } - if c.Port == 0 { - c.Port = 22 - } - var errs *packer.MultiError - if c.Host == "" { + if es := c.CommConfig.Prepare(nil); len(es) > 0 { + errs = packer.MultiErrorAppend(errs, es...) + } + if c.CommConfig.SSHHost == "" { errs = packer.MultiErrorAppend(errs, fmt.Errorf("host must be specified")) } - if c.SSHUsername == "" { + if c.CommConfig.SSHUsername == "" { errs = packer.MultiErrorAppend(errs, fmt.Errorf("ssh_username must be specified")) } - if c.SSHPassword == "" && c.SSHPrivateKeyFile == "" { + if c.CommConfig.SSHPassword == "" && c.CommConfig.SSHPrivateKey == "" { errs = packer.MultiErrorAppend(errs, fmt.Errorf("one of ssh_password and ssh_private_key_file must be specified")) } - if c.SSHPassword != "" && c.SSHPrivateKeyFile != "" { + if c.CommConfig.SSHPassword != "" && c.CommConfig.SSHPrivateKey != "" { errs = packer.MultiErrorAppend(errs, fmt.Errorf("only one of ssh_password and ssh_private_key_file must be specified")) } diff --git a/builder/null/config_test.go b/builder/null/config_test.go index dd574de35..12123378d 100644 --- a/builder/null/config_test.go +++ b/builder/null/config_test.go @@ -6,7 +6,7 @@ import ( func testConfig() map[string]interface{} { return map[string]interface{}{ - "host": "foo", + "ssh_host": "foo", "ssh_username": "bar", "ssh_password": "baz", } @@ -48,8 +48,8 @@ func TestConfigPrepare_port(t *testing.T) { // default port should be 22 delete(raw, "port") c, warns, errs := NewConfig(raw) - if c.Port != 22 { - t.Fatalf("bad: port should default to 22, not %d", c.Port) + if c.CommConfig.SSHPort != 22 { + t.Fatalf("bad: port should default to 22, not %d", c.CommConfig.SSHPort) } testConfigOk(t, warns, errs) } @@ -58,12 +58,12 @@ func TestConfigPrepare_host(t *testing.T) { raw := testConfig() // No host - delete(raw, "host") + delete(raw, "ssh_host") _, warns, errs := NewConfig(raw) testConfigErr(t, warns, errs) // Good host - raw["host"] = "good" + raw["ssh_host"] = "good" _, warns, errs = NewConfig(raw) testConfigOk(t, warns, errs) } From 4b4fe2280d291a19c356bbc793ecf6760983748b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 17:50:45 -0400 Subject: [PATCH 151/208] helper/communicator: can be disabled --- helper/communicator/config.go | 5 +++ helper/communicator/config_test.go | 28 +++++++++++++++++ helper/communicator/step_connect.go | 28 +++++++++++++---- helper/communicator/step_connect_test.go | 39 ++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 helper/communicator/config_test.go create mode 100644 helper/communicator/step_connect_test.go diff --git a/helper/communicator/config.go b/helper/communicator/config.go index 2500a7d2d..28d1d3a43 100644 --- a/helper/communicator/config.go +++ b/helper/communicator/config.go @@ -10,6 +10,7 @@ import ( // Config is the common configuration that communicators allow within // a builder. type Config struct { + Type string `mapstructure:"communicator"` SSHHost string `mapstructure:"ssh_host"` SSHPort int `mapstructure:"ssh_port"` SSHUsername string `mapstructure:"ssh_username"` @@ -20,6 +21,10 @@ type Config struct { } func (c *Config) Prepare(ctx *interpolate.Context) []error { + if c.Type == "" { + c.Type = "ssh" + } + if c.SSHPort == 0 { c.SSHPort = 22 } diff --git a/helper/communicator/config_test.go b/helper/communicator/config_test.go new file mode 100644 index 000000000..f57fb68ca --- /dev/null +++ b/helper/communicator/config_test.go @@ -0,0 +1,28 @@ +package communicator + +import ( + "testing" + + "github.com/mitchellh/packer/template/interpolate" +) + +func testConfig() *Config { + return &Config{ + SSHUsername: "root", + } +} + +func TestConfigType(t *testing.T) { + c := testConfig() + if err := c.Prepare(testContext(t)); len(err) > 0 { + t.Fatalf("bad: %#v", err) + } + + if c.Type != "ssh" { + t.Fatal("bad: %#v", c) + } +} + +func testContext(t *testing.T) *interpolate.Context { + return nil +} diff --git a/helper/communicator/step_connect.go b/helper/communicator/step_connect.go index 77feebfb9..e6338027e 100644 --- a/helper/communicator/step_connect.go +++ b/helper/communicator/step_connect.go @@ -1,6 +1,9 @@ package communicator import ( + "fmt" + "log" + "github.com/mitchellh/multistep" gossh "golang.org/x/crypto/ssh" ) @@ -26,14 +29,27 @@ type StepConnect struct { } func (s *StepConnect) Run(state multistep.StateBag) multistep.StepAction { - // Eventually we might switch between multiple of these depending - // on the communicator type. - s.substep = &StepConnectSSH{ - Config: s.Config, - SSHAddress: s.SSHAddress, - SSHConfig: s.SSHConfig, + typeMap := map[string]multistep.Step{ + "none": nil, + "ssh": &StepConnectSSH{ + Config: s.Config, + SSHAddress: s.SSHAddress, + SSHConfig: s.SSHConfig, + }, } + step, ok := typeMap[s.Config.Type] + if !ok { + state.Put("error", fmt.Errorf("unknown communicator type: %s", s.Config.Type)) + return multistep.ActionHalt + } + + if step == nil { + log.Printf("[INFO] communicator disabled, will not connect") + return multistep.ActionContinue + } + + s.substep = step return s.substep.Run(state) } diff --git a/helper/communicator/step_connect_test.go b/helper/communicator/step_connect_test.go new file mode 100644 index 000000000..bf908f8fb --- /dev/null +++ b/helper/communicator/step_connect_test.go @@ -0,0 +1,39 @@ +package communicator + +import ( + "bytes" + "testing" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" +) + +func TestStepConnect_impl(t *testing.T) { + var _ multistep.Step = new(StepConnect) +} + +func TestStepConnect_none(t *testing.T) { + state := testState(t) + + step := &StepConnect{ + Config: &Config{ + Type: "none", + }, + } + defer step.Cleanup(state) + + // run the step + if action := step.Run(state); action != multistep.ActionContinue { + t.Fatalf("bad action: %#v", action) + } +} + +func testState(t *testing.T) multistep.StateBag { + state := new(multistep.BasicStateBag) + state.Put("hook", &packer.MockHook{}) + state.Put("ui", &packer.BasicUi{ + Reader: new(bytes.Buffer), + Writer: new(bytes.Buffer), + }) + return state +} From 60081c323a2e546717199ca12c83209eb375a6f7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 17:51:27 -0400 Subject: [PATCH 152/208] helper/communicator: ssh settings aren't required if type is none --- helper/communicator/config.go | 6 ++++-- helper/communicator/config_test.go | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/helper/communicator/config.go b/helper/communicator/config.go index 28d1d3a43..a2719db13 100644 --- a/helper/communicator/config.go +++ b/helper/communicator/config.go @@ -35,8 +35,10 @@ func (c *Config) Prepare(ctx *interpolate.Context) []error { // Validation var errs []error - if c.SSHUsername == "" { - errs = append(errs, errors.New("An ssh_username must be specified")) + if c.Type == "ssh" { + if c.SSHUsername == "" { + errs = append(errs, errors.New("An ssh_username must be specified")) + } } return errs diff --git a/helper/communicator/config_test.go b/helper/communicator/config_test.go index f57fb68ca..029c9fe35 100644 --- a/helper/communicator/config_test.go +++ b/helper/communicator/config_test.go @@ -23,6 +23,13 @@ func TestConfigType(t *testing.T) { } } +func TestConfig_none(t *testing.T) { + c := &Config{Type: "none"} + if err := c.Prepare(testContext(t)); len(err) > 0 { + t.Fatalf("bad: %#v", err) + } +} + func testContext(t *testing.T) *interpolate.Context { return nil } From 90581899a4f0013e70a66b2e0999405dd0023835 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 17:53:45 -0400 Subject: [PATCH 153/208] helper/config: decode time durations --- helper/config/decode.go | 1 + helper/config/decode_test.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/helper/config/decode.go b/helper/config/decode.go index 0148ad27c..20554da61 100644 --- a/helper/config/decode.go +++ b/helper/config/decode.go @@ -66,6 +66,7 @@ func Decode(target interface{}, config *DecodeOpts, raws ...interface{}) error { DecodeHook: mapstructure.ComposeDecodeHookFunc( uint8ToStringHook, mapstructure.StringToSliceHookFunc(","), + mapstructure.StringToTimeDurationHookFunc(), ), }) if err != nil { diff --git a/helper/config/decode_test.go b/helper/config/decode_test.go index 43aa615a7..f9fa590c9 100644 --- a/helper/config/decode_test.go +++ b/helper/config/decode_test.go @@ -3,6 +3,7 @@ package config import ( "reflect" "testing" + "time" "github.com/mitchellh/packer/template/interpolate" ) @@ -11,6 +12,7 @@ func TestDecode(t *testing.T) { type Target struct { Name string Address string + Time time.Duration } cases := map[string]struct { @@ -22,10 +24,12 @@ func TestDecode(t *testing.T) { []interface{}{ map[string]interface{}{ "name": "bar", + "time": "5s", }, }, &Target{ Name: "bar", + Time: 5 * time.Second, }, nil, }, From 5d630bf5fb555f4ea496a13a8b811779827c5be6 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:05:10 -0400 Subject: [PATCH 154/208] helper/communicator: validate ssh keys --- helper/communicator/config.go | 12 ++++++++++ helper/communicator/ssh.go | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 helper/communicator/ssh.go diff --git a/helper/communicator/config.go b/helper/communicator/config.go index a2719db13..a2d93a480 100644 --- a/helper/communicator/config.go +++ b/helper/communicator/config.go @@ -2,6 +2,8 @@ package communicator import ( "errors" + "fmt" + "os" "time" "github.com/mitchellh/packer/template/interpolate" @@ -39,6 +41,16 @@ func (c *Config) Prepare(ctx *interpolate.Context) []error { if c.SSHUsername == "" { errs = append(errs, errors.New("An ssh_username must be specified")) } + + if c.SSHPrivateKey != "" { + if _, err := os.Stat(c.SSHPrivateKey); err != nil { + errs = append(errs, fmt.Errorf( + "ssh_private_key_file is invalid: %s", err)) + } else if _, err := SSHFileSigner(c.SSHPrivateKey); err != nil { + errs = append(errs, fmt.Errorf( + "ssh_private_key_file is invalid: %s", err)) + } + } } return errs diff --git a/helper/communicator/ssh.go b/helper/communicator/ssh.go new file mode 100644 index 000000000..831d620bc --- /dev/null +++ b/helper/communicator/ssh.go @@ -0,0 +1,44 @@ +package communicator + +import ( + "encoding/pem" + "fmt" + "io/ioutil" + "os" + + "golang.org/x/crypto/ssh" +) + +// SSHFileSigner returns an ssh.Signer for a key file. +func SSHFileSigner(path string) (ssh.Signer, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + keyBytes, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + // We parse the private key on our own first so that we can + // show a nicer error if the private key has a password. + block, _ := pem.Decode(keyBytes) + if block == nil { + return nil, fmt.Errorf( + "Failed to read key '%s': no key found", path) + } + if block.Headers["Proc-Type"] == "4,ENCRYPTED" { + return nil, fmt.Errorf( + "Failed to read key '%s': password protected keys are\n"+ + "not supported. Please decrypt the key prior to use.", path) + } + + signer, err := ssh.ParsePrivateKey(keyBytes) + if err != nil { + return nil, fmt.Errorf("Error setting up SSH config: %s", err) + } + + return signer, nil +} From d5166a8e6c411a2bca48507a3a0a691540fabf6a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:08:12 -0400 Subject: [PATCH 155/208] builder/virtualbox: use new communicator abstraction --- builder/virtualbox/common/ssh.go | 8 +-- builder/virtualbox/common/ssh_config.go | 52 ++++++-------------- builder/virtualbox/common/ssh_config_test.go | 46 ++++------------- builder/virtualbox/iso/builder.go | 11 +++-- builder/virtualbox/ovf/builder.go | 11 +++-- 5 files changed, 41 insertions(+), 87 deletions(-) diff --git a/builder/virtualbox/common/ssh.go b/builder/virtualbox/common/ssh.go index 9ca2529b8..c20ac2836 100644 --- a/builder/virtualbox/common/ssh.go +++ b/builder/virtualbox/common/ssh.go @@ -17,13 +17,13 @@ func SSHAddress(state multistep.StateBag) (string, error) { func SSHConfigFunc(config SSHConfig) func(multistep.StateBag) (*gossh.ClientConfig, error) { return func(state multistep.StateBag) (*gossh.ClientConfig, error) { auth := []gossh.AuthMethod{ - gossh.Password(config.SSHPassword), + gossh.Password(config.Comm.SSHPassword), gossh.KeyboardInteractive( - ssh.PasswordKeyboardInteractive(config.SSHPassword)), + ssh.PasswordKeyboardInteractive(config.Comm.SSHPassword)), } if config.SSHKeyPath != "" { - signer, err := commonssh.FileSigner(config.SSHKeyPath) + signer, err := commonssh.FileSigner(config.Comm.SSHPrivateKey) if err != nil { return nil, err } @@ -32,7 +32,7 @@ func SSHConfigFunc(config SSHConfig) func(multistep.StateBag) (*gossh.ClientConf } return &gossh.ClientConfig{ - User: config.SSHUser, + User: config.Comm.SSHUsername, Auth: auth, }, nil } diff --git a/builder/virtualbox/common/ssh_config.go b/builder/virtualbox/common/ssh_config.go index 366b86201..8997c58f7 100644 --- a/builder/virtualbox/common/ssh_config.go +++ b/builder/virtualbox/common/ssh_config.go @@ -2,25 +2,23 @@ package common import ( "errors" - "fmt" - "os" "time" - commonssh "github.com/mitchellh/packer/common/ssh" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/template/interpolate" ) type SSHConfig struct { - SSHHostPortMin uint `mapstructure:"ssh_host_port_min"` - SSHHostPortMax uint `mapstructure:"ssh_host_port_max"` - SSHKeyPath string `mapstructure:"ssh_key_path"` - SSHPassword string `mapstructure:"ssh_password"` - SSHPort uint `mapstructure:"ssh_port"` - SSHUser string `mapstructure:"ssh_username"` - RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` - SSHSkipNatMapping bool `mapstructure:"ssh_skip_nat_mapping"` + Comm communicator.Config `mapstructure:",squash"` - SSHWaitTimeout time.Duration + SSHHostPortMin uint `mapstructure:"ssh_host_port_min"` + SSHHostPortMax uint `mapstructure:"ssh_host_port_max"` + SSHSkipNatMapping bool `mapstructure:"ssh_skip_nat_mapping"` + + // These are deprecated, but we keep them around for BC + // TODO(@mitchellh): remove + SSHKeyPath string `mapstructure:"ssh_key_path"` + SSHWaitTimeout time.Duration `mapstructure:"ssh_wait_timeout"` } func (c *SSHConfig) Prepare(ctx *interpolate.Context) []error { @@ -32,37 +30,19 @@ func (c *SSHConfig) Prepare(ctx *interpolate.Context) []error { c.SSHHostPortMax = 4444 } - if c.SSHPort == 0 { - c.SSHPort = 22 - } - - if c.RawSSHWaitTimeout == "" { - c.RawSSHWaitTimeout = "20m" - } - - var errs []error + // TODO: backwards compatibility, write fixer instead if c.SSHKeyPath != "" { - if _, err := os.Stat(c.SSHKeyPath); err != nil { - errs = append(errs, fmt.Errorf("ssh_key_path is invalid: %s", err)) - } else if _, err := commonssh.FileSigner(c.SSHKeyPath); err != nil { - errs = append(errs, fmt.Errorf("ssh_key_path is invalid: %s", err)) - } + c.Comm.SSHPrivateKey = c.SSHKeyPath + } + if c.SSHWaitTimeout != 0 { + c.Comm.SSHTimeout = c.SSHWaitTimeout } + errs := c.Comm.Prepare(ctx) if c.SSHHostPortMin > c.SSHHostPortMax { errs = append(errs, errors.New("ssh_host_port_min must be less than ssh_host_port_max")) } - if c.SSHUser == "" { - errs = append(errs, errors.New("An ssh_username must be specified.")) - } - - var err error - c.SSHWaitTimeout, err = time.ParseDuration(c.RawSSHWaitTimeout) - if err != nil { - errs = append(errs, fmt.Errorf("Failed parsing ssh_wait_timeout: %s", err)) - } - return errs } diff --git a/builder/virtualbox/common/ssh_config_test.go b/builder/virtualbox/common/ssh_config_test.go index 489b7eae8..e5b918fe9 100644 --- a/builder/virtualbox/common/ssh_config_test.go +++ b/builder/virtualbox/common/ssh_config_test.go @@ -4,11 +4,15 @@ import ( "io/ioutil" "os" "testing" + + "github.com/mitchellh/packer/helper/communicator" ) func testSSHConfig() *SSHConfig { return &SSHConfig{ - SSHUser: "foo", + Comm: communicator.Config{ + SSHUsername: "foo", + }, } } @@ -27,8 +31,8 @@ func TestSSHConfigPrepare(t *testing.T) { t.Errorf("bad max ssh host port: %d", c.SSHHostPortMax) } - if c.SSHPort != 22 { - t.Errorf("bad ssh port: %d", c.SSHPort) + if c.Comm.SSHPort != 22 { + t.Errorf("bad ssh port: %d", c.Comm.SSHPort) } } @@ -109,46 +113,14 @@ func TestSSHConfigPrepare_SSHUser(t *testing.T) { var errs []error c = testSSHConfig() - c.SSHUser = "" + c.Comm.SSHUsername = "" errs = c.Prepare(testConfigTemplate(t)) if len(errs) == 0 { t.Fatalf("should have error") } c = testSSHConfig() - c.SSHUser = "exists" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) > 0 { - t.Fatalf("should not have error: %#v", errs) - } -} - -func TestSSHConfigPrepare_SSHWaitTimeout(t *testing.T) { - var c *SSHConfig - var errs []error - - // Defaults - c = testSSHConfig() - c.RawSSHWaitTimeout = "" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) > 0 { - t.Fatalf("should not have error: %#v", errs) - } - if c.RawSSHWaitTimeout != "20m" { - t.Fatalf("bad value: %s", c.RawSSHWaitTimeout) - } - - // Test with a bad value - c = testSSHConfig() - c.RawSSHWaitTimeout = "this is not good" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) == 0 { - t.Fatal("should have error") - } - - // Test with a good one - c = testSSHConfig() - c.RawSSHWaitTimeout = "5s" + c.Comm.SSHUsername = "exists" errs = c.Prepare(testConfigTemplate(t)) if len(errs) > 0 { t.Fatalf("should not have error: %#v", errs) diff --git a/builder/virtualbox/iso/builder.go b/builder/virtualbox/iso/builder.go index 908acf2c1..226de0527 100644 --- a/builder/virtualbox/iso/builder.go +++ b/builder/virtualbox/iso/builder.go @@ -11,6 +11,7 @@ import ( "github.com/mitchellh/multistep" vboxcommon "github.com/mitchellh/packer/builder/virtualbox/common" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -253,7 +254,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, new(vboxcommon.StepAttachFloppy), &vboxcommon.StepForwardSSH{ - GuestPort: b.config.SSHPort, + GuestPort: uint(b.config.SSHConfig.Comm.SSHPort), HostPortMin: b.config.SSHHostPortMin, HostPortMax: b.config.SSHHostPortMax, SkipNatMapping: b.config.SSHSkipNatMapping, @@ -271,10 +272,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe VMName: b.config.VMName, Ctx: b.config.ctx, }, - &common.StepConnectSSH{ - SSHAddress: vboxcommon.SSHAddress, - SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig), - SSHWaitTimeout: b.config.SSHWaitTimeout, + &communicator.StepConnect{ + Config: &b.config.SSHConfig.Comm, + SSHAddress: vboxcommon.SSHAddress, + SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig), }, &vboxcommon.StepUploadVersion{ Path: b.config.VBoxVersionFile, diff --git a/builder/virtualbox/ovf/builder.go b/builder/virtualbox/ovf/builder.go index 82ba4d636..05ec6159f 100644 --- a/builder/virtualbox/ovf/builder.go +++ b/builder/virtualbox/ovf/builder.go @@ -10,6 +10,7 @@ import ( "github.com/mitchellh/multistep" vboxcommon "github.com/mitchellh/packer/builder/virtualbox/common" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/packer" ) @@ -82,7 +83,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, new(vboxcommon.StepAttachFloppy), &vboxcommon.StepForwardSSH{ - GuestPort: b.config.SSHPort, + GuestPort: uint(b.config.SSHConfig.Comm.SSHPort), HostPortMin: b.config.SSHHostPortMin, HostPortMax: b.config.SSHHostPortMax, SkipNatMapping: b.config.SSHSkipNatMapping, @@ -100,10 +101,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe VMName: b.config.VMName, Ctx: b.config.ctx, }, - &common.StepConnectSSH{ - SSHAddress: vboxcommon.SSHAddress, - SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig), - SSHWaitTimeout: b.config.SSHWaitTimeout, + &communicator.StepConnect{ + Config: &b.config.SSHConfig.Comm, + SSHAddress: vboxcommon.SSHAddress, + SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig), }, &vboxcommon.StepUploadVersion{ Path: b.config.VBoxVersionFile, From a1ceb5a7ef8a39d78c88783c58f330e52d2cb852 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:10:37 -0400 Subject: [PATCH 156/208] common: remove StepConnectSSH --- common/step_connect_ssh.go | 171 -------------------------------- common/step_connect_ssh_test.go | 14 --- 2 files changed, 185 deletions(-) delete mode 100644 common/step_connect_ssh.go delete mode 100644 common/step_connect_ssh_test.go diff --git a/common/step_connect_ssh.go b/common/step_connect_ssh.go deleted file mode 100644 index 0c1c624bb..000000000 --- a/common/step_connect_ssh.go +++ /dev/null @@ -1,171 +0,0 @@ -package common - -import ( - "errors" - "fmt" - "log" - "strings" - "time" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/communicator/ssh" - "github.com/mitchellh/packer/packer" - gossh "golang.org/x/crypto/ssh" -) - -// StepConnectSSH is a multistep Step implementation that waits for SSH -// to become available. It gets the connection information from a single -// configuration when creating the step. -// -// Uses: -// ui packer.Ui -// -// Produces: -// communicator packer.Communicator -type StepConnectSSH struct { - // SSHAddress is a function that returns the TCP address to connect to - // for SSH. This is a function so that you can query information - // if necessary for this address. - SSHAddress func(multistep.StateBag) (string, error) - - // SSHConfig is a function that returns the proper client configuration - // for SSH access. - SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error) - - // SSHWaitTimeout is the total timeout to wait for SSH to become available. - SSHWaitTimeout time.Duration - - // Pty, if true, will request a Pty from the remote end. - Pty bool - - comm packer.Communicator -} - -func (s *StepConnectSSH) Run(state multistep.StateBag) multistep.StepAction { - ui := state.Get("ui").(packer.Ui) - - var comm packer.Communicator - var err error - - cancel := make(chan struct{}) - waitDone := make(chan bool, 1) - go func() { - ui.Say("Waiting for SSH to become available...") - comm, err = s.waitForSSH(state, cancel) - waitDone <- true - }() - - log.Printf("Waiting for SSH, up to timeout: %s", s.SSHWaitTimeout) - timeout := time.After(s.SSHWaitTimeout) -WaitLoop: - for { - // Wait for either SSH to become available, a timeout to occur, - // or an interrupt to come through. - select { - case <-waitDone: - if err != nil { - ui.Error(fmt.Sprintf("Error waiting for SSH: %s", err)) - state.Put("error", err) - return multistep.ActionHalt - } - - ui.Say("Connected to SSH!") - s.comm = comm - state.Put("communicator", comm) - break WaitLoop - case <-timeout: - err := fmt.Errorf("Timeout waiting for SSH.") - state.Put("error", err) - ui.Error(err.Error()) - close(cancel) - return multistep.ActionHalt - case <-time.After(1 * time.Second): - if _, ok := state.GetOk(multistep.StateCancelled); ok { - // The step sequence was cancelled, so cancel waiting for SSH - // and just start the halting process. - close(cancel) - log.Println("Interrupt detected, quitting waiting for SSH.") - return multistep.ActionHalt - } - } - } - - return multistep.ActionContinue -} - -func (s *StepConnectSSH) Cleanup(multistep.StateBag) { -} - -func (s *StepConnectSSH) waitForSSH(state multistep.StateBag, cancel <-chan struct{}) (packer.Communicator, error) { - handshakeAttempts := 0 - - var comm packer.Communicator - first := true - for { - // Don't check for cancel or wait on first iteration - if !first { - select { - case <-cancel: - log.Println("SSH wait cancelled. Exiting loop.") - return nil, errors.New("SSH wait cancelled") - case <-time.After(5 * time.Second): - } - } - first = false - - // First we request the TCP connection information - address, err := s.SSHAddress(state) - if err != nil { - log.Printf("Error getting SSH address: %s", err) - continue - } - - // Retrieve the SSH configuration - sshConfig, err := s.SSHConfig(state) - if err != nil { - log.Printf("Error getting SSH config: %s", err) - continue - } - - // Attempt to connect to SSH port - connFunc := ssh.ConnectFunc("tcp", address) - nc, err := connFunc() - if err != nil { - log.Printf("TCP connection to SSH ip/port failed: %s", err) - continue - } - nc.Close() - - // Then we attempt to connect via SSH - config := &ssh.Config{ - Connection: connFunc, - SSHConfig: sshConfig, - Pty: s.Pty, - } - - log.Println("Attempting SSH connection...") - comm, err = ssh.New(address, config) - if err != nil { - log.Printf("SSH handshake err: %s", err) - - // Only count this as an attempt if we were able to attempt - // to authenticate. Note this is very brittle since it depends - // on the string of the error... but I don't see any other way. - if strings.Contains(err.Error(), "authenticate") { - log.Printf("Detected authentication error. Increasing handshake attempts.") - handshakeAttempts += 1 - } - - if handshakeAttempts < 10 { - // Try to connect via SSH a handful of times - continue - } - - return nil, err - } - - break - } - - return comm, nil -} diff --git a/common/step_connect_ssh_test.go b/common/step_connect_ssh_test.go deleted file mode 100644 index 49b0e52b4..000000000 --- a/common/step_connect_ssh_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package common - -import ( - "github.com/mitchellh/multistep" - "testing" -) - -func TestStepConnectSSH_Impl(t *testing.T) { - var raw interface{} - raw = new(StepConnectSSH) - if _, ok := raw.(multistep.Step); !ok { - t.Fatalf("connect ssh should be a step") - } -} From e55792811967d9c90367a8f92d9b60b79497055c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:16:12 -0400 Subject: [PATCH 157/208] builder/amazon: use helper/communicator --- builder/amazon/common/run_config.go | 37 ++++-------------------- builder/amazon/common/run_config_test.go | 34 +++++++++------------- builder/amazon/ebs/builder.go | 16 ++++++---- builder/amazon/instance/builder.go | 16 ++++++---- 4 files changed, 38 insertions(+), 65 deletions(-) diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index 0afdeb07c..6dec07b39 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -4,9 +4,9 @@ import ( "errors" "fmt" "os" - "time" "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/template/interpolate" ) @@ -21,11 +21,6 @@ type RunConfig struct { SourceAmi string `mapstructure:"source_ami"` SpotPrice string `mapstructure:"spot_price"` SpotPriceAutoProduct string `mapstructure:"spot_price_auto_product"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPrivateKeyFile string `mapstructure:"ssh_private_key_file"` - SSHPrivateIp bool `mapstructure:"ssh_private_ip"` - SSHPort int `mapstructure:"ssh_port"` SecurityGroupId string `mapstructure:"security_group_id"` SecurityGroupIds []string `mapstructure:"security_group_ids"` SubnetId string `mapstructure:"subnet_id"` @@ -34,27 +29,19 @@ type RunConfig struct { UserDataFile string `mapstructure:"user_data_file"` VpcId string `mapstructure:"vpc_id"` - // Unexported fields that are calculated from others - sshTimeout time.Duration + // Communicator settings + Comm communicator.Config `mapstructure:",squash"` + SSHPrivateIp bool `mapstructure:"ssh_private_ip"` } func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { - // Defaults - if c.SSHPort == 0 { - c.SSHPort = 22 - } - - if c.RawSSHTimeout == "" { - c.RawSSHTimeout = "5m" - } - if c.TemporaryKeyPairName == "" { c.TemporaryKeyPairName = fmt.Sprintf( "packer %s", uuid.TimeOrderedUUID()) } // Validation - var errs []error + errs := c.Comm.Prepare(ctx) if c.SourceAmi == "" { errs = append(errs, errors.New("A source_ami must be specified")) } @@ -70,10 +57,6 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { } } - if c.SSHUsername == "" { - errs = append(errs, errors.New("An ssh_username must be specified")) - } - if c.UserData != "" && c.UserDataFile != "" { errs = append(errs, fmt.Errorf("Only one of user_data or user_data_file can be specified.")) } else if c.UserDataFile != "" { @@ -91,15 +74,5 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { } } - var err error - c.sshTimeout, err = time.ParseDuration(c.RawSSHTimeout) - if err != nil { - errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - return errs } - -func (c *RunConfig) SSHTimeout() time.Duration { - return c.sshTimeout -} diff --git a/builder/amazon/common/run_config_test.go b/builder/amazon/common/run_config_test.go index 8e9c4b6b9..0b029169b 100644 --- a/builder/amazon/common/run_config_test.go +++ b/builder/amazon/common/run_config_test.go @@ -4,6 +4,8 @@ import ( "io/ioutil" "os" "testing" + + "github.com/mitchellh/packer/helper/communicator" ) func init() { @@ -19,7 +21,10 @@ func testConfig() *RunConfig { return &RunConfig{ SourceAmi: "abcd", InstanceType: "m1.small", - SSHUsername: "root", + + Comm: communicator.Config{ + SSHUsername: "foo", + }, } } @@ -62,41 +67,28 @@ func TestRunConfigPrepare_SpotAuto(t *testing.T) { func TestRunConfigPrepare_SSHPort(t *testing.T) { c := testConfig() - c.SSHPort = 0 + c.Comm.SSHPort = 0 if err := c.Prepare(nil); len(err) != 0 { t.Fatalf("err: %s", err) } - if c.SSHPort != 22 { - t.Fatalf("invalid value: %d", c.SSHPort) + if c.Comm.SSHPort != 22 { + t.Fatalf("invalid value: %d", c.Comm.SSHPort) } - c.SSHPort = 44 + c.Comm.SSHPort = 44 if err := c.Prepare(nil); len(err) != 0 { t.Fatalf("err: %s", err) } - if c.SSHPort != 44 { - t.Fatalf("invalid value: %d", c.SSHPort) - } -} - -func TestRunConfigPrepare_SSHTimeout(t *testing.T) { - c := testConfig() - c.RawSSHTimeout = "" - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } - - c.RawSSHTimeout = "bad" - if err := c.Prepare(nil); len(err) != 1 { - t.Fatalf("err: %s", err) + if c.Comm.SSHPort != 44 { + t.Fatalf("invalid value: %d", c.Comm.SSHPort) } } func TestRunConfigPrepare_SSHUsername(t *testing.T) { c := testConfig() - c.SSHUsername = "" + c.Comm.SSHUsername = "" if err := c.Prepare(nil); len(err) != 1 { t.Fatalf("err: %s", err) } diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index c689accee..a9adcf208 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -13,6 +13,7 @@ import ( "github.com/mitchellh/multistep" awscommon "github.com/mitchellh/packer/builder/amazon/common" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -89,11 +90,11 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Debug: b.config.PackerDebug, DebugKeyPath: fmt.Sprintf("ec2_%s.pem", b.config.PackerBuildName), KeyPairName: b.config.TemporaryKeyPairName, - PrivateKeyFile: b.config.SSHPrivateKeyFile, + PrivateKeyFile: b.config.RunConfig.Comm.SSHPrivateKey, }, &awscommon.StepSecurityGroup{ SecurityGroupIds: b.config.SecurityGroupIds, - SSHPort: b.config.SSHPort, + SSHPort: b.config.RunConfig.Comm.SSHPort, VpcId: b.config.VpcId, }, &awscommon.StepRunSourceInstance{ @@ -112,11 +113,14 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe BlockDevices: b.config.BlockDevices, Tags: b.config.RunTags, }, - &common.StepConnectSSH{ + &communicator.StepConnect{ + Config: &b.config.RunConfig.Comm, SSHAddress: awscommon.SSHAddress( - ec2conn, b.config.SSHPort, b.config.SSHPrivateIp), - SSHConfig: awscommon.SSHConfig(b.config.SSHUsername), - SSHWaitTimeout: b.config.SSHTimeout(), + ec2conn, + b.config.RunConfig.Comm.SSHPort, + b.config.SSHPrivateIp), + SSHConfig: awscommon.SSHConfig( + b.config.RunConfig.Comm.SSHUsername), }, &common.StepProvision{}, &stepStopInstance{SpotPrice: b.config.SpotPrice}, diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 91355b913..09bda686a 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -13,6 +13,7 @@ import ( "github.com/mitchellh/multistep" awscommon "github.com/mitchellh/packer/builder/amazon/common" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -175,11 +176,11 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Debug: b.config.PackerDebug, DebugKeyPath: fmt.Sprintf("ec2_%s.pem", b.config.PackerBuildName), KeyPairName: b.config.TemporaryKeyPairName, - PrivateKeyFile: b.config.SSHPrivateKeyFile, + PrivateKeyFile: b.config.RunConfig.Comm.SSHPrivateKey, }, &awscommon.StepSecurityGroup{ SecurityGroupIds: b.config.SecurityGroupIds, - SSHPort: b.config.SSHPort, + SSHPort: b.config.RunConfig.Comm.SSHPort, VpcId: b.config.VpcId, }, &awscommon.StepRunSourceInstance{ @@ -197,11 +198,14 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe BlockDevices: b.config.BlockDevices, Tags: b.config.RunTags, }, - &common.StepConnectSSH{ + &communicator.StepConnect{ + Config: &b.config.RunConfig.Comm, SSHAddress: awscommon.SSHAddress( - ec2conn, b.config.SSHPort, b.config.SSHPrivateIp), - SSHConfig: awscommon.SSHConfig(b.config.SSHUsername), - SSHWaitTimeout: b.config.SSHTimeout(), + ec2conn, + b.config.RunConfig.Comm.SSHPort, + b.config.SSHPrivateIp), + SSHConfig: awscommon.SSHConfig( + b.config.RunConfig.Comm.SSHUsername), }, &common.StepProvision{}, &StepUploadX509Cert{}, From 669f3018813886cdc64cfdbd9a2431d58c7555f6 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:26:13 -0400 Subject: [PATCH 158/208] builder/digitalocean: use helper/comm --- builder/digitalocean/builder.go | 10 ++-- builder/digitalocean/builder_test.go | 54 +++------------------- builder/digitalocean/config.go | 56 ++++++----------------- builder/digitalocean/ssh.go | 4 +- builder/digitalocean/step_droplet_info.go | 2 +- builder/digitalocean/step_power_off.go | 2 +- builder/digitalocean/step_snapshot.go | 2 +- 7 files changed, 31 insertions(+), 99 deletions(-) diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index 97569d0fe..d5f1b7a83 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -6,11 +6,11 @@ package digitalocean import ( "fmt" "log" - "time" "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/packer" "golang.org/x/oauth2" ) @@ -53,10 +53,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, new(stepCreateDroplet), new(stepDropletInfo), - &common.StepConnectSSH{ - SSHAddress: sshAddress, - SSHConfig: sshConfig, - SSHWaitTimeout: 5 * time.Minute, + &communicator.StepConnect{ + Config: &b.config.Comm, + SSHAddress: sshAddress, + SSHConfig: sshConfig, }, new(common.StepProvision), new(stepShutdown), diff --git a/builder/digitalocean/builder_test.go b/builder/digitalocean/builder_test.go index 22c9e3b50..5e3014937 100644 --- a/builder/digitalocean/builder_test.go +++ b/builder/digitalocean/builder_test.go @@ -3,6 +3,7 @@ package digitalocean import ( "strconv" "testing" + "time" "github.com/mitchellh/packer/packer" ) @@ -163,8 +164,8 @@ func TestBuilderPrepare_SSHUsername(t *testing.T) { t.Fatalf("should not have error: %s", err) } - if b.config.SSHUsername != "root" { - t.Errorf("invalid: %s", b.config.SSHUsername) + if b.config.Comm.SSHUsername != "root" { + t.Errorf("invalid: %s", b.config.Comm.SSHUsername) } // Test set @@ -178,52 +179,11 @@ func TestBuilderPrepare_SSHUsername(t *testing.T) { t.Fatalf("should not have error: %s", err) } - if b.config.SSHUsername != "foo" { - t.Errorf("invalid: %s", b.config.SSHUsername) + if b.config.Comm.SSHUsername != "foo" { + t.Errorf("invalid: %s", b.config.Comm.SSHUsername) } } -func TestBuilderPrepare_SSHTimeout(t *testing.T) { - var b Builder - config := testConfig() - - // Test default - warnings, err := b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.RawSSHTimeout != "1m" { - t.Errorf("invalid: %s", b.config.RawSSHTimeout) - } - - // Test set - config["ssh_timeout"] = "30s" - b = Builder{} - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - // Test bad - config["ssh_timeout"] = "tubes" - b = Builder{} - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err == nil { - t.Fatal("should have error") - } - -} - func TestBuilderPrepare_StateTimeout(t *testing.T) { var b Builder config := testConfig() @@ -237,8 +197,8 @@ func TestBuilderPrepare_StateTimeout(t *testing.T) { t.Fatalf("should not have error: %s", err) } - if b.config.RawStateTimeout != "6m" { - t.Errorf("invalid: %s", b.config.RawStateTimeout) + if b.config.StateTimeout != 6*time.Minute { + t.Errorf("invalid: %s", b.config.StateTimeout) } // Test set diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 178b54049..3bddf2f7e 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -9,6 +9,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -16,6 +17,7 @@ import ( type Config struct { common.PackerConfig `mapstructure:",squash"` + Comm communicator.Config `mapstructure:",squash"` APIToken string `mapstructure:"api_token"` @@ -23,20 +25,11 @@ type Config struct { Size string `mapstructure:"size"` Image string `mapstructure:"image"` - PrivateNetworking bool `mapstructure:"private_networking"` - SnapshotName string `mapstructure:"snapshot_name"` - DropletName string `mapstructure:"droplet_name"` - UserData string `mapstructure:"user_data"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort uint `mapstructure:"ssh_port"` - - RawSSHTimeout string `mapstructure:"ssh_timeout"` - RawStateTimeout string `mapstructure:"state_timeout"` - - // These are unexported since they're set by other fields - // being set. - sshTimeout time.Duration - stateTimeout time.Duration + PrivateNetworking bool `mapstructure:"private_networking"` + SnapshotName string `mapstructure:"snapshot_name"` + StateTimeout time.Duration `mapstructure:"state_timeout"` + DropletName string `mapstructure:"droplet_name"` + UserData string `mapstructure:"user_data"` ctx *interpolate.Context } @@ -79,29 +72,22 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { c.DropletName = fmt.Sprintf("packer-%s", uuid.TimeOrderedUUID()) } - if c.SSHUsername == "" { + if c.Comm.SSHUsername == "" { // Default to "root". You can override this if your // SourceImage has a different user account then the DO default - c.SSHUsername = "root" + c.Comm.SSHUsername = "root" } - if c.SSHPort == 0 { - // Default to port 22 per DO default - c.SSHPort = 22 - } - - if c.RawSSHTimeout == "" { - // Default to 1 minute timeouts - c.RawSSHTimeout = "1m" - } - - if c.RawStateTimeout == "" { + if c.StateTimeout == 0 { // Default to 6 minute timeouts waiting for // desired state. i.e waiting for droplet to become active - c.RawStateTimeout = "6m" + c.StateTimeout = 6 * time.Minute } var errs *packer.MultiError + if es := c.Comm.Prepare(c.ctx); len(es) > 0 { + errs = packer.MultiErrorAppend(errs, es...) + } if c.APIToken == "" { // Required configurations that will display errors if not set errs = packer.MultiErrorAppend( @@ -123,20 +109,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { errs, errors.New("image is required")) } - sshTimeout, err := time.ParseDuration(c.RawSSHTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - c.sshTimeout = sshTimeout - - stateTimeout, err := time.ParseDuration(c.RawStateTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing state_timeout: %s", err)) - } - c.stateTimeout = stateTimeout - if errs != nil && len(errs.Errors) > 0 { return nil, nil, errs } diff --git a/builder/digitalocean/ssh.go b/builder/digitalocean/ssh.go index 12046b04b..4d8d1f08c 100644 --- a/builder/digitalocean/ssh.go +++ b/builder/digitalocean/ssh.go @@ -9,7 +9,7 @@ import ( func sshAddress(state multistep.StateBag) (string, error) { config := state.Get("config").(Config) ipAddress := state.Get("droplet_ip").(string) - return fmt.Sprintf("%s:%d", ipAddress, config.SSHPort), nil + return fmt.Sprintf("%s:%d", ipAddress, config.Comm.SSHPort), nil } func sshConfig(state multistep.StateBag) (*ssh.ClientConfig, error) { @@ -22,7 +22,7 @@ func sshConfig(state multistep.StateBag) (*ssh.ClientConfig, error) { } return &ssh.ClientConfig{ - User: config.SSHUsername, + User: config.Comm.SSHUsername, Auth: []ssh.AuthMethod{ ssh.PublicKeys(signer), }, diff --git a/builder/digitalocean/step_droplet_info.go b/builder/digitalocean/step_droplet_info.go index 5fbcb7141..81d84dc8d 100644 --- a/builder/digitalocean/step_droplet_info.go +++ b/builder/digitalocean/step_droplet_info.go @@ -18,7 +18,7 @@ func (s *stepDropletInfo) Run(state multistep.StateBag) multistep.StepAction { ui.Say("Waiting for droplet to become active...") - err := waitForDropletState("active", dropletId, client, c.stateTimeout) + err := waitForDropletState("active", dropletId, client, c.StateTimeout) if err != nil { err := fmt.Errorf("Error waiting for droplet to become active: %s", err) state.Put("error", err) diff --git a/builder/digitalocean/step_power_off.go b/builder/digitalocean/step_power_off.go index 94891e227..8341083aa 100644 --- a/builder/digitalocean/step_power_off.go +++ b/builder/digitalocean/step_power_off.go @@ -42,7 +42,7 @@ func (s *stepPowerOff) Run(state multistep.StateBag) multistep.StepAction { } log.Println("Waiting for poweroff event to complete...") - err = waitForDropletState("off", dropletId, client, c.stateTimeout) + err = waitForDropletState("off", dropletId, client, c.StateTimeout) if err != nil { state.Put("error", err) ui.Error(err.Error()) diff --git a/builder/digitalocean/step_snapshot.go b/builder/digitalocean/step_snapshot.go index f6902b8c5..fdc303105 100644 --- a/builder/digitalocean/step_snapshot.go +++ b/builder/digitalocean/step_snapshot.go @@ -41,7 +41,7 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { // With the pending state over, verify that we're in the active state ui.Say("Waiting for snapshot to complete...") - err = waitForDropletState("active", dropletId, client, c.stateTimeout) + err = waitForDropletState("active", dropletId, client, c.StateTimeout) if err != nil { err := fmt.Errorf("Error waiting for snapshot to complete: %s", err) state.Put("error", err) From 502076c92e6f9abb15e77d6170cfa86105b0b495 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:30:16 -0400 Subject: [PATCH 159/208] builder/googlecompute: use helper/comm --- builder/googlecompute/builder.go | 9 ++++--- builder/googlecompute/config.go | 26 +++---------------- builder/googlecompute/ssh.go | 4 +-- builder/googlecompute/step_create_instance.go | 2 +- 4 files changed, 12 insertions(+), 29 deletions(-) diff --git a/builder/googlecompute/builder.go b/builder/googlecompute/builder.go index 8e35598d2..3ed576b50 100644 --- a/builder/googlecompute/builder.go +++ b/builder/googlecompute/builder.go @@ -8,6 +8,7 @@ import ( "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/packer" ) @@ -60,10 +61,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &StepInstanceInfo{ Debug: b.config.PackerDebug, }, - &common.StepConnectSSH{ - SSHAddress: sshAddress, - SSHConfig: sshConfig, - SSHWaitTimeout: b.config.sshTimeout, + &communicator.StepConnect{ + Config: &b.config.Comm, + SSHAddress: sshAddress, + SSHConfig: sshConfig, }, new(common.StepProvision), new(StepTeardownInstance), diff --git a/builder/googlecompute/config.go b/builder/googlecompute/config.go index 69d223a75..7f59aa183 100644 --- a/builder/googlecompute/config.go +++ b/builder/googlecompute/config.go @@ -7,6 +7,7 @@ import ( "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -17,6 +18,7 @@ import ( // state of the config object. type Config struct { common.PackerConfig `mapstructure:",squash"` + Comm communicator.Config `mapstructure:",squash"` AccountFile string `mapstructure:"account_file"` ProjectId string `mapstructure:"project_id"` @@ -31,16 +33,12 @@ type Config struct { Network string `mapstructure:"network"` SourceImage string `mapstructure:"source_image"` SourceImageProjectId string `mapstructure:"source_image_project_id"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort uint `mapstructure:"ssh_port"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` RawStateTimeout string `mapstructure:"state_timeout"` Tags []string `mapstructure:"tags"` Zone string `mapstructure:"zone"` account accountFile privateKeyBytes []byte - sshTimeout time.Duration stateTimeout time.Duration ctx *interpolate.Context } @@ -88,20 +86,12 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { c.MachineType = "n1-standard-1" } - if c.RawSSHTimeout == "" { - c.RawSSHTimeout = "5m" - } - if c.RawStateTimeout == "" { c.RawStateTimeout = "5m" } - if c.SSHUsername == "" { - c.SSHUsername = "root" - } - - if c.SSHPort == 0 { - c.SSHPort = 22 + if c.Comm.SSHUsername == "" { + c.Comm.SSHUsername = "root" } var errs *packer.MultiError @@ -122,14 +112,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { errs, errors.New("a zone must be specified")) } - // Process timeout settings. - sshTimeout, err := time.ParseDuration(c.RawSSHTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - c.sshTimeout = sshTimeout - stateTimeout, err := time.ParseDuration(c.RawStateTimeout) if err != nil { errs = packer.MultiErrorAppend( diff --git a/builder/googlecompute/ssh.go b/builder/googlecompute/ssh.go index e04029e44..446648884 100644 --- a/builder/googlecompute/ssh.go +++ b/builder/googlecompute/ssh.go @@ -10,7 +10,7 @@ import ( func sshAddress(state multistep.StateBag) (string, error) { config := state.Get("config").(*Config) ipAddress := state.Get("instance_ip").(string) - return fmt.Sprintf("%s:%d", ipAddress, config.SSHPort), nil + return fmt.Sprintf("%s:%d", ipAddress, config.Comm.SSHPort), nil } // sshConfig returns the ssh configuration. @@ -24,7 +24,7 @@ func sshConfig(state multistep.StateBag) (*ssh.ClientConfig, error) { } return &ssh.ClientConfig{ - User: config.SSHUsername, + User: config.Comm.SSHUsername, Auth: []ssh.AuthMethod{ ssh.PublicKeys(signer), }, diff --git a/builder/googlecompute/step_create_instance.go b/builder/googlecompute/step_create_instance.go index 6bfee5460..939925c58 100644 --- a/builder/googlecompute/step_create_instance.go +++ b/builder/googlecompute/step_create_instance.go @@ -32,7 +32,7 @@ func (config *Config) getInstanceMetadata(sshPublicKey string) map[string]string // Merge any existing ssh keys with our public key sshMetaKey := "sshKeys" - sshKeys := fmt.Sprintf("%s:%s", config.SSHUsername, sshPublicKey) + sshKeys := fmt.Sprintf("%s:%s", config.Comm.SSHUsername, sshPublicKey) if confSshKeys, exists := instanceMetadata[sshMetaKey]; exists { sshKeys = fmt.Sprintf("%s\n%s", sshKeys, confSshKeys) } From f55e2d2c4b7d005191975c0e7b0256cd504d705e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:34:37 -0400 Subject: [PATCH 160/208] builder/openstack: convert to helper/comm --- builder/openstack/builder.go | 23 ++++++++++------ builder/openstack/run_config.go | 41 +++++----------------------- builder/openstack/run_config_test.go | 34 +++++++++-------------- 3 files changed, 34 insertions(+), 64 deletions(-) diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index ab60afc0e..6d178c6ef 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -5,10 +5,11 @@ package openstack import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/common" "log" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -19,9 +20,10 @@ const BuilderId = "mitchellh.openstack" type Config struct { common.PackerConfig `mapstructure:",squash"` - AccessConfig `mapstructure:",squash"` - ImageConfig `mapstructure:",squash"` - RunConfig `mapstructure:",squash"` + + AccessConfig `mapstructure:",squash"` + ImageConfig `mapstructure:",squash"` + RunConfig `mapstructure:",squash"` ctx interpolate.Context } @@ -88,10 +90,13 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe FloatingIpPool: b.config.FloatingIpPool, FloatingIp: b.config.FloatingIp, }, - &common.StepConnectSSH{ - SSHAddress: SSHAddress(computeClient, b.config.SSHInterface, b.config.SSHPort), - SSHConfig: SSHConfig(b.config.SSHUsername), - SSHWaitTimeout: b.config.SSHTimeout(), + &communicator.StepConnect{ + Config: &b.config.RunConfig.Comm, + SSHAddress: SSHAddress( + computeClient, + b.config.SSHInterface, + b.config.RunConfig.Comm.SSHPort), + SSHConfig: SSHConfig(b.config.RunConfig.Comm.SSHUsername), }, &common.StepProvision{}, &stepCreateImage{}, diff --git a/builder/openstack/run_config.go b/builder/openstack/run_config.go index 4a6a1b81f..a758b0dd1 100644 --- a/builder/openstack/run_config.go +++ b/builder/openstack/run_config.go @@ -2,21 +2,19 @@ package openstack import ( "errors" - "fmt" - "time" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/template/interpolate" ) // RunConfig contains configuration for running an instance from a source // image and details on how to access that launched image. type RunConfig struct { + Comm communicator.Config `mapstructure:",squash"` + SSHInterface string `mapstructure:"ssh_interface"` + SourceImage string `mapstructure:"source_image"` Flavor string `mapstructure:"flavor"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort int `mapstructure:"ssh_port"` - SSHInterface string `mapstructure:"ssh_interface"` AvailabilityZone string `mapstructure:"availability_zone"` RackconnectWait bool `mapstructure:"rackconnect_wait"` FloatingIpPool string `mapstructure:"floating_ip_pool"` @@ -27,23 +25,12 @@ type RunConfig struct { // Not really used, but here for BC OpenstackProvider string `mapstructure:"openstack_provider"` UseFloatingIp bool `mapstructure:"use_floating_ip"` - - // Unexported fields that are calculated from others - sshTimeout time.Duration } func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { // Defaults - if c.SSHUsername == "" { - c.SSHUsername = "root" - } - - if c.SSHPort == 0 { - c.SSHPort = 22 - } - - if c.RawSSHTimeout == "" { - c.RawSSHTimeout = "5m" + if c.Comm.SSHUsername == "" { + c.Comm.SSHUsername = "root" } if c.UseFloatingIp && c.FloatingIpPool == "" { @@ -51,8 +38,7 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { } // Validation - var err error - errs := make([]error, 0) + errs := c.Comm.Prepare(ctx) if c.SourceImage == "" { errs = append(errs, errors.New("A source_image must be specified")) } @@ -61,18 +47,5 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { errs = append(errs, errors.New("A flavor must be specified")) } - if c.SSHUsername == "" { - errs = append(errs, errors.New("An ssh_username must be specified")) - } - - c.sshTimeout, err = time.ParseDuration(c.RawSSHTimeout) - if err != nil { - errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - return errs } - -func (c *RunConfig) SSHTimeout() time.Duration { - return c.sshTimeout -} diff --git a/builder/openstack/run_config_test.go b/builder/openstack/run_config_test.go index 16b89b352..113934a29 100644 --- a/builder/openstack/run_config_test.go +++ b/builder/openstack/run_config_test.go @@ -3,6 +3,8 @@ package openstack import ( "os" "testing" + + "github.com/mitchellh/packer/helper/communicator" ) func init() { @@ -17,7 +19,10 @@ func testRunConfig() *RunConfig { return &RunConfig{ SourceImage: "abcd", Flavor: "m1.small", - SSHUsername: "root", + + Comm: communicator.Config{ + SSHUsername: "foo", + }, } } @@ -47,41 +52,28 @@ func TestRunConfigPrepare_SourceImage(t *testing.T) { func TestRunConfigPrepare_SSHPort(t *testing.T) { c := testRunConfig() - c.SSHPort = 0 + c.Comm.SSHPort = 0 if err := c.Prepare(nil); len(err) != 0 { t.Fatalf("err: %s", err) } - if c.SSHPort != 22 { - t.Fatalf("invalid value: %d", c.SSHPort) + if c.Comm.SSHPort != 22 { + t.Fatalf("invalid value: %d", c.Comm.SSHPort) } - c.SSHPort = 44 + c.Comm.SSHPort = 44 if err := c.Prepare(nil); len(err) != 0 { t.Fatalf("err: %s", err) } - if c.SSHPort != 44 { - t.Fatalf("invalid value: %d", c.SSHPort) - } -} - -func TestRunConfigPrepare_SSHTimeout(t *testing.T) { - c := testRunConfig() - c.RawSSHTimeout = "" - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } - - c.RawSSHTimeout = "bad" - if err := c.Prepare(nil); len(err) != 1 { - t.Fatalf("err: %s", err) + if c.Comm.SSHPort != 44 { + t.Fatalf("invalid value: %d", c.Comm.SSHPort) } } func TestRunConfigPrepare_SSHUsername(t *testing.T) { c := testRunConfig() - c.SSHUsername = "" + c.Comm.SSHUsername = "" if err := c.Prepare(nil); len(err) != 0 { t.Fatalf("err: %s", err) } From 820bad69390687185970d9b32f6005320da62b09 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:43:27 -0400 Subject: [PATCH 161/208] builder/parallels: convert to new comm type --- builder/parallels/common/ssh.go | 8 ++-- builder/parallels/common/ssh_config.go | 47 ++++++--------------- builder/parallels/common/ssh_config_test.go | 46 ++++---------------- builder/parallels/iso/builder.go | 9 ++-- builder/parallels/pvm/builder.go | 12 +++--- 5 files changed, 37 insertions(+), 85 deletions(-) diff --git a/builder/parallels/common/ssh.go b/builder/parallels/common/ssh.go index becf68e42..827124677 100644 --- a/builder/parallels/common/ssh.go +++ b/builder/parallels/common/ssh.go @@ -29,13 +29,13 @@ func SSHAddress(state multistep.StateBag) (string, error) { func SSHConfigFunc(config SSHConfig) func(multistep.StateBag) (*ssh.ClientConfig, error) { return func(state multistep.StateBag) (*ssh.ClientConfig, error) { auth := []ssh.AuthMethod{ - ssh.Password(config.SSHPassword), + ssh.Password(config.Comm.SSHPassword), ssh.KeyboardInteractive( - packerssh.PasswordKeyboardInteractive(config.SSHPassword)), + packerssh.PasswordKeyboardInteractive(config.Comm.SSHPassword)), } if config.SSHKeyPath != "" { - signer, err := commonssh.FileSigner(config.SSHKeyPath) + signer, err := commonssh.FileSigner(config.Comm.SSHPrivateKey) if err != nil { return nil, err } @@ -44,7 +44,7 @@ func SSHConfigFunc(config SSHConfig) func(multistep.StateBag) (*ssh.ClientConfig } return &ssh.ClientConfig{ - User: config.SSHUser, + User: config.Comm.SSHUsername, Auth: auth, }, nil } diff --git a/builder/parallels/common/ssh_config.go b/builder/parallels/common/ssh_config.go index 9f1a9506f..bea164b06 100644 --- a/builder/parallels/common/ssh_config.go +++ b/builder/parallels/common/ssh_config.go @@ -1,52 +1,29 @@ package common import ( - "errors" - "fmt" - "os" "time" - commonssh "github.com/mitchellh/packer/common/ssh" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/template/interpolate" ) type SSHConfig struct { - SSHKeyPath string `mapstructure:"ssh_key_path"` - SSHPassword string `mapstructure:"ssh_password"` - SSHPort uint `mapstructure:"ssh_port"` - SSHUser string `mapstructure:"ssh_username"` - RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` + Comm communicator.Config `mapstructure:",squash"` - SSHWaitTimeout time.Duration + // These are deprecated, but we keep them around for BC + // TODO(@mitchellh): remove + SSHKeyPath string `mapstructure:"ssh_key_path"` + SSHWaitTimeout time.Duration `mapstructure:"ssh_wait_timeout"` } func (c *SSHConfig) Prepare(ctx *interpolate.Context) []error { - if c.SSHPort == 0 { - c.SSHPort = 22 - } - - if c.RawSSHWaitTimeout == "" { - c.RawSSHWaitTimeout = "20m" - } - - var errs []error + // TODO: backwards compatibility, write fixer instead if c.SSHKeyPath != "" { - if _, err := os.Stat(c.SSHKeyPath); err != nil { - errs = append(errs, fmt.Errorf("ssh_key_path is invalid: %s", err)) - } else if _, err := commonssh.FileSigner(c.SSHKeyPath); err != nil { - errs = append(errs, fmt.Errorf("ssh_key_path is invalid: %s", err)) - } + c.Comm.SSHPrivateKey = c.SSHKeyPath + } + if c.SSHWaitTimeout != 0 { + c.Comm.SSHTimeout = c.SSHWaitTimeout } - if c.SSHUser == "" { - errs = append(errs, errors.New("An ssh_username must be specified.")) - } - - var err error - c.SSHWaitTimeout, err = time.ParseDuration(c.RawSSHWaitTimeout) - if err != nil { - errs = append(errs, fmt.Errorf("Failed parsing ssh_wait_timeout: %s", err)) - } - - return errs + return c.Comm.Prepare(ctx) } diff --git a/builder/parallels/common/ssh_config_test.go b/builder/parallels/common/ssh_config_test.go index a6c9e8ef5..01dd1ff62 100644 --- a/builder/parallels/common/ssh_config_test.go +++ b/builder/parallels/common/ssh_config_test.go @@ -4,11 +4,15 @@ import ( "io/ioutil" "os" "testing" + + "github.com/mitchellh/packer/helper/communicator" ) func testSSHConfig() *SSHConfig { return &SSHConfig{ - SSHUser: "foo", + Comm: communicator.Config{ + SSHUsername: "foo", + }, } } @@ -19,8 +23,8 @@ func TestSSHConfigPrepare(t *testing.T) { t.Fatalf("err: %#v", errs) } - if c.SSHPort != 22 { - t.Errorf("bad ssh port: %d", c.SSHPort) + if c.Comm.SSHPort != 22 { + t.Errorf("bad ssh port: %d", c.Comm.SSHPort) } } @@ -78,46 +82,14 @@ func TestSSHConfigPrepare_SSHUser(t *testing.T) { var errs []error c = testSSHConfig() - c.SSHUser = "" + c.Comm.SSHUsername = "" errs = c.Prepare(testConfigTemplate(t)) if len(errs) == 0 { t.Fatalf("should have error") } c = testSSHConfig() - c.SSHUser = "exists" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) > 0 { - t.Fatalf("should not have error: %#v", errs) - } -} - -func TestSSHConfigPrepare_SSHWaitTimeout(t *testing.T) { - var c *SSHConfig - var errs []error - - // Defaults - c = testSSHConfig() - c.RawSSHWaitTimeout = "" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) > 0 { - t.Fatalf("should not have error: %#v", errs) - } - if c.RawSSHWaitTimeout != "20m" { - t.Fatalf("bad value: %s", c.RawSSHWaitTimeout) - } - - // Test with a bad value - c = testSSHConfig() - c.RawSSHWaitTimeout = "this is not good" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) == 0 { - t.Fatal("should have error") - } - - // Test with a good one - c = testSSHConfig() - c.RawSSHWaitTimeout = "5s" + c.Comm.SSHUsername = "exists" errs = c.Prepare(testConfigTemplate(t)) if len(errs) > 0 { t.Fatalf("should not have error: %#v", errs) diff --git a/builder/parallels/iso/builder.go b/builder/parallels/iso/builder.go index 24ef3c726..9758e6bec 100644 --- a/builder/parallels/iso/builder.go +++ b/builder/parallels/iso/builder.go @@ -9,6 +9,7 @@ import ( "github.com/mitchellh/multistep" parallelscommon "github.com/mitchellh/packer/builder/parallels/common" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -245,10 +246,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe VMName: b.config.VMName, Ctx: b.config.ctx, }, - &common.StepConnectSSH{ - SSHAddress: parallelscommon.SSHAddress, - SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig), - SSHWaitTimeout: b.config.SSHWaitTimeout, + &communicator.StepConnect{ + Config: &b.config.SSHConfig.Comm, + SSHAddress: parallelscommon.SSHAddress, + SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig), }, ¶llelscommon.StepUploadVersion{ Path: b.config.PrlctlVersionFile, diff --git a/builder/parallels/pvm/builder.go b/builder/parallels/pvm/builder.go index a0c7ca9cd..1ddd64202 100644 --- a/builder/parallels/pvm/builder.go +++ b/builder/parallels/pvm/builder.go @@ -3,11 +3,13 @@ package pvm import ( "errors" "fmt" + "log" + "github.com/mitchellh/multistep" parallelscommon "github.com/mitchellh/packer/builder/parallels/common" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/packer" - "log" ) // Builder implements packer.Builder and builds the actual Parallels @@ -80,10 +82,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe VMName: b.config.VMName, Ctx: b.config.ctx, }, - &common.StepConnectSSH{ - SSHAddress: parallelscommon.SSHAddress, - SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig), - SSHWaitTimeout: b.config.SSHWaitTimeout, + &communicator.StepConnect{ + Config: &b.config.SSHConfig.Comm, + SSHAddress: parallelscommon.SSHAddress, + SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig), }, ¶llelscommon.StepUploadVersion{ Path: b.config.PrlctlVersionFile, From 89af447c8cfd34dcabaf760bf35ac54239bf1731 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:47:59 -0400 Subject: [PATCH 162/208] builder/qemu: convert to helper/comm --- builder/qemu/builder.go | 69 ++++++++++++++---------------------- builder/qemu/builder_test.go | 8 ++--- builder/qemu/ssh.go | 10 +++--- 3 files changed, 33 insertions(+), 54 deletions(-) diff --git a/builder/qemu/builder.go b/builder/qemu/builder.go index cb0ac6c88..2a1a2fe1c 100644 --- a/builder/qemu/builder.go +++ b/builder/qemu/builder.go @@ -12,7 +12,7 @@ import ( "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common" - commonssh "github.com/mitchellh/packer/common/ssh" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -78,6 +78,7 @@ type Builder struct { type Config struct { common.PackerConfig `mapstructure:",squash"` + Comm communicator.Config `mapstructure:",squash"` Accelerator string `mapstructure:"accelerator"` BootCommand []string `mapstructure:"boot_command"` @@ -103,25 +104,24 @@ type Config struct { ShutdownCommand string `mapstructure:"shutdown_command"` SSHHostPortMin uint `mapstructure:"ssh_host_port_min"` SSHHostPortMax uint `mapstructure:"ssh_host_port_max"` - SSHPassword string `mapstructure:"ssh_password"` - SSHPort uint `mapstructure:"ssh_port"` - SSHUser string `mapstructure:"ssh_username"` - SSHKeyPath string `mapstructure:"ssh_key_path"` VNCPortMin uint `mapstructure:"vnc_port_min"` VNCPortMax uint `mapstructure:"vnc_port_max"` VMName string `mapstructure:"vm_name"` + // These are deprecated, but we keep them around for BC + // TODO(@mitchellh): remove + SSHKeyPath string `mapstructure:"ssh_key_path"` + SSHWaitTimeout time.Duration `mapstructure:"ssh_wait_timeout"` + // TODO(mitchellh): deprecate RunOnce bool `mapstructure:"run_once"` RawBootWait string `mapstructure:"boot_wait"` RawSingleISOUrl string `mapstructure:"iso_url"` RawShutdownTimeout string `mapstructure:"shutdown_timeout"` - RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` bootWait time.Duration `` shutdownTimeout time.Duration `` - sshWaitTimeout time.Duration `` ctx interpolate.Context } @@ -139,9 +139,6 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { return nil, err } - var errs *packer.MultiError - warnings := make([]string, 0) - if b.config.DiskSize == 0 { b.config.DiskSize = 40000 } @@ -190,10 +187,6 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { b.config.SSHHostPortMax = 4444 } - if b.config.SSHPort == 0 { - b.config.SSHPort = 22 - } - if b.config.VNCPortMin == 0 { b.config.VNCPortMin = 5900 } @@ -222,6 +215,21 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { b.config.DiskInterface = "virtio" } + // TODO: backwards compatibility, write fixer instead + if b.config.SSHKeyPath != "" { + b.config.Comm.SSHPrivateKey = b.config.SSHKeyPath + } + if b.config.SSHWaitTimeout != 0 { + b.config.Comm.SSHTimeout = b.config.SSHWaitTimeout + } + + var errs *packer.MultiError + warnings := make([]string, 0) + + if es := b.config.Comm.Prepare(&b.config.ctx); len(es) > 0 { + errs = packer.MultiErrorAppend(errs, es...) + } + if !(b.config.Format == "qcow2" || b.config.Format == "raw") { errs = packer.MultiErrorAppend( errs, errors.New("invalid format, only 'qcow2' or 'raw' are allowed")) @@ -314,42 +322,17 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { b.config.RawShutdownTimeout = "5m" } - if b.config.RawSSHWaitTimeout == "" { - b.config.RawSSHWaitTimeout = "20m" - } - b.config.shutdownTimeout, err = time.ParseDuration(b.config.RawShutdownTimeout) if err != nil { errs = packer.MultiErrorAppend( errs, fmt.Errorf("Failed parsing shutdown_timeout: %s", err)) } - if b.config.SSHKeyPath != "" { - if _, err := os.Stat(b.config.SSHKeyPath); err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("ssh_key_path is invalid: %s", err)) - } else if _, err := commonssh.FileSigner(b.config.SSHKeyPath); err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("ssh_key_path is invalid: %s", err)) - } - } - if b.config.SSHHostPortMin > b.config.SSHHostPortMax { errs = packer.MultiErrorAppend( errs, errors.New("ssh_host_port_min must be less than ssh_host_port_max")) } - if b.config.SSHUser == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("An ssh_username must be specified.")) - } - - b.config.sshWaitTimeout, err = time.ParseDuration(b.config.RawSSHWaitTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing ssh_wait_timeout: %s", err)) - } - if b.config.VNCPortMin > b.config.VNCPortMax { errs = packer.MultiErrorAppend( errs, fmt.Errorf("vnc_port_min must be less than vnc_port_max")) @@ -409,10 +392,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe steprun, &stepBootWait{}, &stepTypeBootCommand{}, - &common.StepConnectSSH{ - SSHAddress: sshAddress, - SSHConfig: sshConfig, - SSHWaitTimeout: b.config.sshWaitTimeout, + &communicator.StepConnect{ + Config: &b.config.Comm, + SSHAddress: sshAddress, + SSHConfig: sshConfig, }, new(common.StepProvision), new(stepShutdown), diff --git a/builder/qemu/builder_test.go b/builder/qemu/builder_test.go index e3415d514..84d1d40c3 100644 --- a/builder/qemu/builder_test.go +++ b/builder/qemu/builder_test.go @@ -79,8 +79,8 @@ func TestBuilderPrepare_Defaults(t *testing.T) { t.Errorf("bad max ssh host port: %d", b.config.SSHHostPortMax) } - if b.config.SSHPort != 22 { - t.Errorf("bad ssh port: %d", b.config.SSHPort) + if b.config.Comm.SSHPort != 22 { + t.Errorf("bad ssh port: %d", b.config.Comm.SSHPort) } if b.config.VMName != "packer-foo" { @@ -595,10 +595,6 @@ func TestBuilderPrepare_SSHWaitTimeout(t *testing.T) { t.Fatalf("err: %s", err) } - if b.config.RawSSHWaitTimeout != "20m" { - t.Fatalf("bad value: %s", b.config.RawSSHWaitTimeout) - } - // Test with a bad value config["ssh_wait_timeout"] = "this is not good" b = Builder{} diff --git a/builder/qemu/ssh.go b/builder/qemu/ssh.go index 9724d7483..17dc36de9 100644 --- a/builder/qemu/ssh.go +++ b/builder/qemu/ssh.go @@ -18,13 +18,13 @@ func sshConfig(state multistep.StateBag) (*gossh.ClientConfig, error) { config := state.Get("config").(*Config) auth := []gossh.AuthMethod{ - gossh.Password(config.SSHPassword), + gossh.Password(config.Comm.SSHPassword), gossh.KeyboardInteractive( - ssh.PasswordKeyboardInteractive(config.SSHPassword)), + ssh.PasswordKeyboardInteractive(config.Comm.SSHPassword)), } - if config.SSHKeyPath != "" { - signer, err := commonssh.FileSigner(config.SSHKeyPath) + if config.Comm.SSHPrivateKey != "" { + signer, err := commonssh.FileSigner(config.Comm.SSHPrivateKey) if err != nil { return nil, err } @@ -33,7 +33,7 @@ func sshConfig(state multistep.StateBag) (*gossh.ClientConfig, error) { } return &gossh.ClientConfig{ - User: config.SSHUser, + User: config.Comm.SSHUsername, Auth: auth, }, nil } From b61ed3adfcdcc34de3c08aea087e2b3acbb0c9c9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:52:44 -0400 Subject: [PATCH 163/208] builder/vmware: convert to helper/comm --- builder/vmware/common/ssh.go | 16 +++--- builder/vmware/common/ssh_config.go | 62 ++++++------------------ builder/vmware/common/ssh_config_test.go | 61 +++-------------------- builder/vmware/iso/builder.go | 10 ++-- builder/vmware/iso/builder_test.go | 8 +-- builder/vmware/iso/driver_esx5.go | 2 +- builder/vmware/vmx/builder.go | 10 ++-- 7 files changed, 44 insertions(+), 125 deletions(-) diff --git a/builder/vmware/common/ssh.go b/builder/vmware/common/ssh.go index 167bd6792..7181e90bf 100644 --- a/builder/vmware/common/ssh.go +++ b/builder/vmware/common/ssh.go @@ -18,8 +18,8 @@ func SSHAddressFunc(config *SSHConfig) func(multistep.StateBag) (string, error) driver := state.Get("driver").(Driver) vmxPath := state.Get("vmx_path").(string) - if config.SSHHost != "" { - return fmt.Sprintf("%s:%d", config.SSHHost, config.SSHPort), nil + if config.Comm.SSHHost != "" { + return fmt.Sprintf("%s:%d", config.Comm.SSHHost, config.Comm.SSHPort), nil } log.Println("Lookup up IP information...") @@ -62,20 +62,20 @@ func SSHAddressFunc(config *SSHConfig) func(multistep.StateBag) (string, error) } log.Printf("Detected IP: %s", ipAddress) - return fmt.Sprintf("%s:%d", ipAddress, config.SSHPort), nil + return fmt.Sprintf("%s:%d", ipAddress, config.Comm.SSHPort), nil } } func SSHConfigFunc(config *SSHConfig) func(multistep.StateBag) (*gossh.ClientConfig, error) { return func(state multistep.StateBag) (*gossh.ClientConfig, error) { auth := []gossh.AuthMethod{ - gossh.Password(config.SSHPassword), + gossh.Password(config.Comm.SSHPassword), gossh.KeyboardInteractive( - ssh.PasswordKeyboardInteractive(config.SSHPassword)), + ssh.PasswordKeyboardInteractive(config.Comm.SSHPassword)), } - if config.SSHKeyPath != "" { - signer, err := commonssh.FileSigner(config.SSHKeyPath) + if config.Comm.SSHPrivateKey != "" { + signer, err := commonssh.FileSigner(config.Comm.SSHPrivateKey) if err != nil { return nil, err } @@ -84,7 +84,7 @@ func SSHConfigFunc(config *SSHConfig) func(multistep.StateBag) (*gossh.ClientCon } return &gossh.ClientConfig{ - User: config.SSHUser, + User: config.Comm.SSHUsername, Auth: auth, }, nil } diff --git a/builder/vmware/common/ssh_config.go b/builder/vmware/common/ssh_config.go index 1bd481d92..86754f6e4 100644 --- a/builder/vmware/common/ssh_config.go +++ b/builder/vmware/common/ssh_config.go @@ -1,63 +1,33 @@ package common import ( - "errors" - "fmt" - "net" - "os" "time" - commonssh "github.com/mitchellh/packer/common/ssh" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/template/interpolate" ) type SSHConfig struct { - SSHUser string `mapstructure:"ssh_username"` - SSHKeyPath string `mapstructure:"ssh_key_path"` - SSHPassword string `mapstructure:"ssh_password"` - SSHHost string `mapstructure:"ssh_host"` - SSHPort uint `mapstructure:"ssh_port"` - SSHSkipRequestPty bool `mapstructure:"ssh_skip_request_pty"` - RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` + Comm communicator.Config `mapstructure:",squash"` - SSHWaitTimeout time.Duration + // These are deprecated, but we keep them around for BC + // TODO(@mitchellh): remove + SSHKeyPath string `mapstructure:"ssh_key_path"` + SSHSkipRequestPty bool `mapstructure:"ssh_skip_request_pty"` + SSHWaitTimeout time.Duration `mapstructure:"ssh_wait_timeout"` } func (c *SSHConfig) Prepare(ctx *interpolate.Context) []error { - if c.SSHPort == 0 { - c.SSHPort = 22 - } - - if c.RawSSHWaitTimeout == "" { - c.RawSSHWaitTimeout = "20m" - } - - var errs []error + // TODO: backwards compatibility, write fixer instead if c.SSHKeyPath != "" { - if _, err := os.Stat(c.SSHKeyPath); err != nil { - errs = append(errs, fmt.Errorf("ssh_key_path is invalid: %s", err)) - } else if _, err := commonssh.FileSigner(c.SSHKeyPath); err != nil { - errs = append(errs, fmt.Errorf("ssh_key_path is invalid: %s", err)) - } + c.Comm.SSHPrivateKey = c.SSHKeyPath + } + if c.SSHWaitTimeout != 0 { + c.Comm.SSHTimeout = c.SSHWaitTimeout + } + if c.SSHSkipRequestPty { + c.Comm.SSHPty = false } - if c.SSHHost != "" { - if ip := net.ParseIP(c.SSHHost); ip == nil { - if _, err := net.LookupHost(c.SSHHost); err != nil { - errs = append(errs, errors.New("ssh_host is an invalid IP or hostname")) - } - } - } - - if c.SSHUser == "" { - errs = append(errs, errors.New("An ssh_username must be specified.")) - } - - var err error - c.SSHWaitTimeout, err = time.ParseDuration(c.RawSSHWaitTimeout) - if err != nil { - errs = append(errs, fmt.Errorf("Failed parsing ssh_wait_timeout: %s", err)) - } - - return errs + return c.Comm.Prepare(ctx) } diff --git a/builder/vmware/common/ssh_config_test.go b/builder/vmware/common/ssh_config_test.go index a6c9e8ef5..1fd84c02f 100644 --- a/builder/vmware/common/ssh_config_test.go +++ b/builder/vmware/common/ssh_config_test.go @@ -4,11 +4,15 @@ import ( "io/ioutil" "os" "testing" + + "github.com/mitchellh/packer/helper/communicator" ) func testSSHConfig() *SSHConfig { return &SSHConfig{ - SSHUser: "foo", + Comm: communicator.Config{ + SSHUsername: "foo", + }, } } @@ -19,8 +23,8 @@ func TestSSHConfigPrepare(t *testing.T) { t.Fatalf("err: %#v", errs) } - if c.SSHPort != 22 { - t.Errorf("bad ssh port: %d", c.SSHPort) + if c.Comm.SSHPort != 22 { + t.Errorf("bad ssh port: %d", c.Comm.SSHPort) } } @@ -73,57 +77,6 @@ func TestSSHConfigPrepare_SSHKeyPath(t *testing.T) { } } -func TestSSHConfigPrepare_SSHUser(t *testing.T) { - var c *SSHConfig - var errs []error - - c = testSSHConfig() - c.SSHUser = "" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) == 0 { - t.Fatalf("should have error") - } - - c = testSSHConfig() - c.SSHUser = "exists" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) > 0 { - t.Fatalf("should not have error: %#v", errs) - } -} - -func TestSSHConfigPrepare_SSHWaitTimeout(t *testing.T) { - var c *SSHConfig - var errs []error - - // Defaults - c = testSSHConfig() - c.RawSSHWaitTimeout = "" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) > 0 { - t.Fatalf("should not have error: %#v", errs) - } - if c.RawSSHWaitTimeout != "20m" { - t.Fatalf("bad value: %s", c.RawSSHWaitTimeout) - } - - // Test with a bad value - c = testSSHConfig() - c.RawSSHWaitTimeout = "this is not good" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) == 0 { - t.Fatal("should have error") - } - - // Test with a good one - c = testSSHConfig() - c.RawSSHWaitTimeout = "5s" - errs = c.Prepare(testConfigTemplate(t)) - if len(errs) > 0 { - t.Fatalf("should not have error: %#v", errs) - } -} - const testPem = ` -----BEGIN RSA PRIVATE KEY----- MIIEpQIBAAKCAQEAxd4iamvrwRJvtNDGQSIbNvvIQN8imXTRWlRY62EvKov60vqu diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go index dce0bbd2a..9599df675 100644 --- a/builder/vmware/iso/builder.go +++ b/builder/vmware/iso/builder.go @@ -13,6 +13,7 @@ import ( "github.com/mitchellh/multistep" vmwcommon "github.com/mitchellh/packer/builder/vmware/common" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -298,11 +299,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe VMName: b.config.VMName, Ctx: b.config.ctx, }, - &common.StepConnectSSH{ - SSHAddress: driver.SSHAddress, - SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), - SSHWaitTimeout: b.config.SSHWaitTimeout, - Pty: !b.config.SSHSkipRequestPty, + &communicator.StepConnect{ + Config: &b.config.SSHConfig.Comm, + SSHAddress: driver.SSHAddress, + SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), }, &vmwcommon.StepUploadTools{ RemoteType: b.config.RemoteType, diff --git a/builder/vmware/iso/builder_test.go b/builder/vmware/iso/builder_test.go index 1749b396e..60c6aeca0 100644 --- a/builder/vmware/iso/builder_test.go +++ b/builder/vmware/iso/builder_test.go @@ -1,12 +1,12 @@ package iso import ( - "github.com/mitchellh/packer/packer" "io/ioutil" "os" "reflect" "testing" - "time" + + "github.com/mitchellh/packer/packer" ) func testConfig() map[string]interface{} { @@ -138,10 +138,6 @@ func TestBuilderPrepare_Defaults(t *testing.T) { t.Errorf("bad Version: %s", b.config.Version) } - if b.config.SSHWaitTimeout != (20 * time.Minute) { - t.Errorf("bad wait timeout: %s", b.config.SSHWaitTimeout) - } - if b.config.VMName != "packer-foo" { t.Errorf("bad vm name: %s", b.config.VMName) } diff --git a/builder/vmware/iso/driver_esx5.go b/builder/vmware/iso/driver_esx5.go index ed56db067..26642f3d7 100644 --- a/builder/vmware/iso/driver_esx5.go +++ b/builder/vmware/iso/driver_esx5.go @@ -253,7 +253,7 @@ func (d *ESX5Driver) SSHAddress(state multistep.StateBag) (string, error) { return "", errors.New("VM network port found, but no IP address") } - address := fmt.Sprintf("%s:%d", record["IPAddress"], config.SSHPort) + address := fmt.Sprintf("%s:%d", record["IPAddress"], config.Comm.SSHPort) state.Put("vm_address", address) return address, nil } diff --git a/builder/vmware/vmx/builder.go b/builder/vmware/vmx/builder.go index 24597f73c..1390da547 100644 --- a/builder/vmware/vmx/builder.go +++ b/builder/vmware/vmx/builder.go @@ -9,6 +9,7 @@ import ( "github.com/mitchellh/multistep" vmwcommon "github.com/mitchellh/packer/builder/vmware/common" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/packer" ) @@ -90,11 +91,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe VMName: b.config.VMName, Ctx: b.config.ctx, }, - &common.StepConnectSSH{ - SSHAddress: driver.SSHAddress, - SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), - SSHWaitTimeout: b.config.SSHWaitTimeout, - Pty: !b.config.SSHSkipRequestPty, + &communicator.StepConnect{ + Config: &b.config.SSHConfig.Comm, + SSHAddress: driver.SSHAddress, + SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), }, &vmwcommon.StepUploadTools{ RemoteType: b.config.RemoteType, From 68e4734caf5256ad19f9fabf8d7c5b924849dead Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 18:55:37 -0400 Subject: [PATCH 164/208] builder/null: pass unit tests --- builder/null/config_test.go | 7 ++++- helper/communicator/testing.go | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 helper/communicator/testing.go diff --git a/builder/null/config_test.go b/builder/null/config_test.go index 12123378d..a3b96421e 100644 --- a/builder/null/config_test.go +++ b/builder/null/config_test.go @@ -1,7 +1,10 @@ package null import ( + "os" "testing" + + "github.com/mitchellh/packer/helper/communicator" ) func testConfig() map[string]interface{} { @@ -97,7 +100,9 @@ func TestConfigPrepare_sshCredential(t *testing.T) { testConfigOk(t, warns, errs) // only ssh_private_key_file - raw["ssh_private_key_file"] = "good" + testFile := communicator.TestPEM(t) + defer os.Remove(testFile) + raw["ssh_private_key_file"] = testFile delete(raw, "ssh_password") _, warns, errs = NewConfig(raw) testConfigOk(t, warns, errs) diff --git a/helper/communicator/testing.go b/helper/communicator/testing.go new file mode 100644 index 000000000..1c89f15e3 --- /dev/null +++ b/helper/communicator/testing.go @@ -0,0 +1,47 @@ +package communicator + +import ( + "io/ioutil" + "testing" +) + +func TestPEM(t *testing.T) string { + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("err: %s", err) + } + tf.Write([]byte(TestPEMContents)) + tf.Close() + + return tf.Name() +} + +const TestPEMContents = ` +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAxd4iamvrwRJvtNDGQSIbNvvIQN8imXTRWlRY62EvKov60vqu +hh+rDzFYAIIzlmrJopvOe0clqmi3mIP9dtkjPFrYflq52a2CF5q+BdwsJXuRHbJW +LmStZUwW1khSz93DhvhmK50nIaczW63u4EO/jJb3xj+wxR1Nkk9bxi3DDsYFt8SN +AzYx9kjlEYQ/+sI4/ATfmdV9h78SVotjScupd9KFzzi76gWq9gwyCBLRynTUWlyD +2UOfJRkOvhN6/jKzvYfVVwjPSfA9IMuooHdScmC4F6KBKJl/zf/zETM0XyzIDNmH +uOPbCiljq2WoRM+rY6ET84EO0kVXbfx8uxUsqQIDAQABAoIBAQCkPj9TF0IagbM3 +5BSs/CKbAWS4dH/D4bPlxx4IRCNirc8GUg+MRb04Xz0tLuajdQDqeWpr6iLZ0RKV +BvreLF+TOdV7DNQ4XE4gSdJyCtCaTHeort/aordL3l0WgfI7mVk0L/yfN1PEG4YG +E9q1TYcyrB3/8d5JwIkjabxERLglCcP+geOEJp+QijbvFIaZR/n2irlKW4gSy6ko +9B0fgUnhkHysSg49ChHQBPQ+o5BbpuLrPDFMiTPTPhdfsvGGcyCGeqfBA56oHcSF +K02Fg8OM+Bd1lb48LAN9nWWY4WbwV+9bkN3Ym8hO4c3a/Dxf2N7LtAQqWZzFjvM3 +/AaDvAgBAoGBAPLD+Xn1IYQPMB2XXCXfOuJewRY7RzoVWvMffJPDfm16O7wOiW5+ +2FmvxUDayk4PZy6wQMzGeGKnhcMMZTyaq2g/QtGfrvy7q1Lw2fB1VFlVblvqhoJa +nMJojjC4zgjBkXMHsRLeTmgUKyGs+fdFbfI6uejBnnf+eMVUMIdJ+6I9AoGBANCn +kWO9640dttyXURxNJ3lBr2H3dJOkmD6XS+u+LWqCSKQe691Y/fZ/ZL0Oc4Mhy7I6 +hsy3kDQ5k2V0fkaNODQIFJvUqXw2pMewUk8hHc9403f4fe9cPrL12rQ8WlQw4yoC +v2B61vNczCCUDtGxlAaw8jzSRaSI5s6ax3K7enbdAoGBAJB1WYDfA2CoAQO6y9Sl +b07A/7kQ8SN5DbPaqrDrBdJziBQxukoMJQXJeGFNUFD/DXFU5Fp2R7C86vXT7HIR +v6m66zH+CYzOx/YE6EsUJms6UP9VIVF0Rg/RU7teXQwM01ZV32LQ8mswhTH20o/3 +uqMHmxUMEhZpUMhrfq0isyApAoGAe1UxGTXfj9AqkIVYylPIq2HqGww7+jFmVEj1 +9Wi6S6Sq72ffnzzFEPkIQL/UA4TsdHMnzsYKFPSbbXLIWUeMGyVTmTDA5c0e5XIR +lPhMOKCAzv8w4VUzMnEkTzkFY5JqFCD/ojW57KvDdNZPVB+VEcdxyAW6aKELXMAc +eHLc1nkCgYEApm/motCTPN32nINZ+Vvywbv64ZD+gtpeMNP3CLrbe1X9O+H52AXa +1jCoOldWR8i2bs2NVPcKZgdo6fFULqE4dBX7Te/uYEIuuZhYLNzRO1IKU/YaqsXG +3bfQ8hKYcSnTfE0gPtLDnqCIxTocaGLSHeG3TH9fTw+dA8FvWpUztI4= +-----END RSA PRIVATE KEY----- +` From 115d583cff475b870028ded5d8374f4d881613e9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 19:23:33 -0400 Subject: [PATCH 165/208] helper/communicator: make host more generic --- builder/amazon/common/ssh.go | 6 +++--- builder/amazon/ebs/builder.go | 3 +-- builder/amazon/instance/builder.go | 3 +-- builder/digitalocean/builder.go | 6 +++--- builder/digitalocean/ssh.go | 8 ++++---- builder/googlecompute/builder.go | 6 +++--- builder/googlecompute/ssh.go | 7 +++---- builder/null/builder.go | 3 +-- builder/null/ssh.go | 6 ++---- builder/openstack/builder.go | 5 ++--- builder/openstack/ssh.go | 13 ++++++------- builder/parallels/common/ssh.go | 6 ++---- builder/parallels/iso/builder.go | 6 +++--- builder/parallels/pvm/builder.go | 6 +++--- builder/qemu/builder.go | 7 ++++--- builder/qemu/ssh.go | 10 ++++++---- builder/virtualbox/common/ssh.go | 10 ++++++---- builder/virtualbox/iso/builder.go | 7 ++++--- builder/virtualbox/ovf/builder.go | 7 ++++--- builder/vmware/common/driver.go | 4 ++-- builder/vmware/common/driver_fusion5.go | 4 ++-- builder/vmware/common/driver_mock.go | 16 ++++++++-------- builder/vmware/common/driver_player5.go | 4 ++-- builder/vmware/common/driver_workstation9.go | 4 ++-- builder/vmware/common/ssh.go | 6 +++--- builder/vmware/iso/builder.go | 6 +++--- builder/vmware/iso/driver_esx5.go | 4 ++-- builder/vmware/vmx/builder.go | 6 +++--- helper/communicator/step_connect.go | 18 ++++++++++-------- helper/communicator/step_connect_ssh.go | 19 +++++++++++++++---- 30 files changed, 113 insertions(+), 103 deletions(-) diff --git a/builder/amazon/common/ssh.go b/builder/amazon/common/ssh.go index 302a90beb..cf644eb25 100644 --- a/builder/amazon/common/ssh.go +++ b/builder/amazon/common/ssh.go @@ -10,9 +10,9 @@ import ( "golang.org/x/crypto/ssh" ) -// SSHAddress returns a function that can be given to the SSH communicator +// SSHHost returns a function that can be given to the SSH communicator // for determining the SSH address based on the instance DNS name. -func SSHAddress(e *ec2.EC2, port int, private bool) func(multistep.StateBag) (string, error) { +func SSHHost(e *ec2.EC2, private bool) func(multistep.StateBag) (string, error) { return func(state multistep.StateBag) (string, error) { for j := 0; j < 2; j++ { var host string @@ -28,7 +28,7 @@ func SSHAddress(e *ec2.EC2, port int, private bool) func(multistep.StateBag) (st } if host != "" { - return fmt.Sprintf("%s:%d", host, port), nil + return host, nil } r, err := e.DescribeInstances(&ec2.DescribeInstancesInput{ diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index a9adcf208..cd3cd8f05 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -115,9 +115,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, &communicator.StepConnect{ Config: &b.config.RunConfig.Comm, - SSHAddress: awscommon.SSHAddress( + Host: awscommon.SSHHost( ec2conn, - b.config.RunConfig.Comm.SSHPort, b.config.SSHPrivateIp), SSHConfig: awscommon.SSHConfig( b.config.RunConfig.Comm.SSHUsername), diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 09bda686a..d26cc63e3 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -200,9 +200,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, &communicator.StepConnect{ Config: &b.config.RunConfig.Comm, - SSHAddress: awscommon.SSHAddress( + Host: awscommon.SSHHost( ec2conn, - b.config.RunConfig.Comm.SSHPort, b.config.SSHPrivateIp), SSHConfig: awscommon.SSHConfig( b.config.RunConfig.Comm.SSHUsername), diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index d5f1b7a83..ec57ed2b2 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -54,9 +54,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe new(stepCreateDroplet), new(stepDropletInfo), &communicator.StepConnect{ - Config: &b.config.Comm, - SSHAddress: sshAddress, - SSHConfig: sshConfig, + Config: &b.config.Comm, + Host: commHost, + SSHConfig: sshConfig, }, new(common.StepProvision), new(stepShutdown), diff --git a/builder/digitalocean/ssh.go b/builder/digitalocean/ssh.go index 4d8d1f08c..5367dde1f 100644 --- a/builder/digitalocean/ssh.go +++ b/builder/digitalocean/ssh.go @@ -2,14 +2,14 @@ package digitalocean import ( "fmt" - "github.com/mitchellh/multistep" "golang.org/x/crypto/ssh" + + "github.com/mitchellh/multistep" ) -func sshAddress(state multistep.StateBag) (string, error) { - config := state.Get("config").(Config) +func commHost(state multistep.StateBag) (string, error) { ipAddress := state.Get("droplet_ip").(string) - return fmt.Sprintf("%s:%d", ipAddress, config.Comm.SSHPort), nil + return ipAddress, nil } func sshConfig(state multistep.StateBag) (*ssh.ClientConfig, error) { diff --git a/builder/googlecompute/builder.go b/builder/googlecompute/builder.go index 3ed576b50..0f6e4bd2f 100644 --- a/builder/googlecompute/builder.go +++ b/builder/googlecompute/builder.go @@ -62,9 +62,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Debug: b.config.PackerDebug, }, &communicator.StepConnect{ - Config: &b.config.Comm, - SSHAddress: sshAddress, - SSHConfig: sshConfig, + Config: &b.config.Comm, + Host: commHost, + SSHConfig: sshConfig, }, new(common.StepProvision), new(StepTeardownInstance), diff --git a/builder/googlecompute/ssh.go b/builder/googlecompute/ssh.go index 446648884..5b7940591 100644 --- a/builder/googlecompute/ssh.go +++ b/builder/googlecompute/ssh.go @@ -2,15 +2,14 @@ package googlecompute import ( "fmt" + "github.com/mitchellh/multistep" "golang.org/x/crypto/ssh" ) -// sshAddress returns the ssh address. -func sshAddress(state multistep.StateBag) (string, error) { - config := state.Get("config").(*Config) +func commHost(state multistep.StateBag) (string, error) { ipAddress := state.Get("instance_ip").(string) - return fmt.Sprintf("%s:%d", ipAddress, config.Comm.SSHPort), nil + return ipAddress, nil } // sshConfig returns the ssh configuration. diff --git a/builder/null/builder.go b/builder/null/builder.go index 925075ee0..fed303540 100644 --- a/builder/null/builder.go +++ b/builder/null/builder.go @@ -30,8 +30,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe steps := []multistep.Step{ &communicator.StepConnect{ Config: &b.config.CommConfig, - SSHAddress: SSHAddress( - b.config.CommConfig.SSHHost, b.config.CommConfig.SSHPort), + Host: CommHost(b.config.CommConfig.SSHHost), SSHConfig: SSHConfig( b.config.CommConfig.SSHUsername, b.config.CommConfig.SSHPassword, diff --git a/builder/null/ssh.go b/builder/null/ssh.go index e6ac9ab16..483390e86 100644 --- a/builder/null/ssh.go +++ b/builder/null/ssh.go @@ -8,11 +8,9 @@ import ( "io/ioutil" ) -// SSHAddress returns a function that can be given to the SSH communicator -// for determining the SSH address -func SSHAddress(host string, port int) func(multistep.StateBag) (string, error) { +func CommHost(host string) func(multistep.StateBag) (string, error) { return func(state multistep.StateBag) (string, error) { - return fmt.Sprintf("%s:%d", host, port), nil + return host, nil } } diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index 6d178c6ef..69ab6a016 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -92,10 +92,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, &communicator.StepConnect{ Config: &b.config.RunConfig.Comm, - SSHAddress: SSHAddress( + Host: CommHost( computeClient, - b.config.SSHInterface, - b.config.RunConfig.Comm.SSHPort), + b.config.SSHInterface), SSHConfig: SSHConfig(b.config.RunConfig.Comm.SSHUsername), }, &common.StepProvision{}, diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go index 76c2686b1..dc4c91771 100644 --- a/builder/openstack/ssh.go +++ b/builder/openstack/ssh.go @@ -13,22 +13,21 @@ import ( "golang.org/x/crypto/ssh" ) -// SSHAddress returns a function that can be given to the SSH communicator -// for determining the SSH address based on the server AccessIPv4 setting.. -func SSHAddress( +// CommHost looks up the host for the communicator. +func CommHost( client *gophercloud.ServiceClient, - sshinterface string, port int) func(multistep.StateBag) (string, error) { + sshinterface string) func(multistep.StateBag) (string, error) { return func(state multistep.StateBag) (string, error) { s := state.Get("server").(*servers.Server) // If we have a floating IP, use that ip := state.Get("access_ip").(*floatingip.FloatingIP) if ip != nil && ip.IP != "" { - return fmt.Sprintf("%s:%d", ip.IP, port), nil + return ip.IP, nil } if s.AccessIPv4 != "" { - return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil + return s.AccessIPv4, nil } // Get all the addresses associated with this server. This @@ -53,7 +52,7 @@ func SSHAddress( } } if addr != "" { - return fmt.Sprintf("%s:%d", addr, port), nil + return addr, nil } } } diff --git a/builder/parallels/common/ssh.go b/builder/parallels/common/ssh.go index 827124677..9e0b2b907 100644 --- a/builder/parallels/common/ssh.go +++ b/builder/parallels/common/ssh.go @@ -1,15 +1,13 @@ package common import ( - "fmt" - "github.com/mitchellh/multistep" commonssh "github.com/mitchellh/packer/common/ssh" packerssh "github.com/mitchellh/packer/communicator/ssh" "golang.org/x/crypto/ssh" ) -func SSHAddress(state multistep.StateBag) (string, error) { +func CommHost(state multistep.StateBag) (string, error) { vmName := state.Get("vmName").(string) driver := state.Get("driver").(Driver) @@ -23,7 +21,7 @@ func SSHAddress(state multistep.StateBag) (string, error) { return "", err } - return fmt.Sprintf("%s:22", ip), nil + return ip, nil } func SSHConfigFunc(config SSHConfig) func(multistep.StateBag) (*ssh.ClientConfig, error) { diff --git a/builder/parallels/iso/builder.go b/builder/parallels/iso/builder.go index 9758e6bec..46fa73687 100644 --- a/builder/parallels/iso/builder.go +++ b/builder/parallels/iso/builder.go @@ -247,9 +247,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Ctx: b.config.ctx, }, &communicator.StepConnect{ - Config: &b.config.SSHConfig.Comm, - SSHAddress: parallelscommon.SSHAddress, - SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig), + Config: &b.config.SSHConfig.Comm, + Host: parallelscommon.CommHost, + SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig), }, ¶llelscommon.StepUploadVersion{ Path: b.config.PrlctlVersionFile, diff --git a/builder/parallels/pvm/builder.go b/builder/parallels/pvm/builder.go index 1ddd64202..0e71ea0f2 100644 --- a/builder/parallels/pvm/builder.go +++ b/builder/parallels/pvm/builder.go @@ -83,9 +83,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Ctx: b.config.ctx, }, &communicator.StepConnect{ - Config: &b.config.SSHConfig.Comm, - SSHAddress: parallelscommon.SSHAddress, - SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig), + Config: &b.config.SSHConfig.Comm, + Host: parallelscommon.CommHost, + SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig), }, ¶llelscommon.StepUploadVersion{ Path: b.config.PrlctlVersionFile, diff --git a/builder/qemu/builder.go b/builder/qemu/builder.go index 2a1a2fe1c..9df908989 100644 --- a/builder/qemu/builder.go +++ b/builder/qemu/builder.go @@ -393,9 +393,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &stepBootWait{}, &stepTypeBootCommand{}, &communicator.StepConnect{ - Config: &b.config.Comm, - SSHAddress: sshAddress, - SSHConfig: sshConfig, + Config: &b.config.Comm, + Host: commHost, + SSHConfig: sshConfig, + SSHPort: commPort, }, new(common.StepProvision), new(stepShutdown), diff --git a/builder/qemu/ssh.go b/builder/qemu/ssh.go index 17dc36de9..498d3fbe9 100644 --- a/builder/qemu/ssh.go +++ b/builder/qemu/ssh.go @@ -1,17 +1,19 @@ package qemu import ( - "fmt" - "github.com/mitchellh/multistep" commonssh "github.com/mitchellh/packer/common/ssh" "github.com/mitchellh/packer/communicator/ssh" gossh "golang.org/x/crypto/ssh" ) -func sshAddress(state multistep.StateBag) (string, error) { +func commHost(state multistep.StateBag) (string, error) { + return "127.0.0.1", nil +} + +func commPort(state multistep.StateBag) (int, error) { sshHostPort := state.Get("sshHostPort").(uint) - return fmt.Sprintf("127.0.0.1:%d", sshHostPort), nil + return int(sshHostPort), nil } func sshConfig(state multistep.StateBag) (*gossh.ClientConfig, error) { diff --git a/builder/virtualbox/common/ssh.go b/builder/virtualbox/common/ssh.go index c20ac2836..2584528dd 100644 --- a/builder/virtualbox/common/ssh.go +++ b/builder/virtualbox/common/ssh.go @@ -1,17 +1,19 @@ package common import ( - "fmt" - "github.com/mitchellh/multistep" commonssh "github.com/mitchellh/packer/common/ssh" "github.com/mitchellh/packer/communicator/ssh" gossh "golang.org/x/crypto/ssh" ) -func SSHAddress(state multistep.StateBag) (string, error) { +func CommHost(state multistep.StateBag) (string, error) { + return "127.0.0.1", nil +} + +func SSHPort(state multistep.StateBag) (int, error) { sshHostPort := state.Get("sshHostPort").(uint) - return fmt.Sprintf("127.0.0.1:%d", sshHostPort), nil + return int(sshHostPort), nil } func SSHConfigFunc(config SSHConfig) func(multistep.StateBag) (*gossh.ClientConfig, error) { diff --git a/builder/virtualbox/iso/builder.go b/builder/virtualbox/iso/builder.go index 226de0527..f37c65c5a 100644 --- a/builder/virtualbox/iso/builder.go +++ b/builder/virtualbox/iso/builder.go @@ -273,9 +273,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Ctx: b.config.ctx, }, &communicator.StepConnect{ - Config: &b.config.SSHConfig.Comm, - SSHAddress: vboxcommon.SSHAddress, - SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig), + Config: &b.config.SSHConfig.Comm, + Host: vboxcommon.CommHost, + SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig), + SSHPort: vboxcommon.SSHPort, }, &vboxcommon.StepUploadVersion{ Path: b.config.VBoxVersionFile, diff --git a/builder/virtualbox/ovf/builder.go b/builder/virtualbox/ovf/builder.go index 05ec6159f..c35f2a50f 100644 --- a/builder/virtualbox/ovf/builder.go +++ b/builder/virtualbox/ovf/builder.go @@ -102,9 +102,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Ctx: b.config.ctx, }, &communicator.StepConnect{ - Config: &b.config.SSHConfig.Comm, - SSHAddress: vboxcommon.SSHAddress, - SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig), + Config: &b.config.SSHConfig.Comm, + Host: vboxcommon.CommHost, + SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig), + SSHPort: vboxcommon.SSHPort, }, &vboxcommon.StepUploadVersion{ Path: b.config.VBoxVersionFile, diff --git a/builder/vmware/common/driver.go b/builder/vmware/common/driver.go index ee8dbc30e..c06c8fd92 100644 --- a/builder/vmware/common/driver.go +++ b/builder/vmware/common/driver.go @@ -29,9 +29,9 @@ type Driver interface { // Checks if the VMX file at the given path is running. IsRunning(string) (bool, error) - // SSHAddress returns the SSH address for the VM that is being + // CommHost returns the host address for the VM that is being // managed by this driver. - SSHAddress(multistep.StateBag) (string, error) + CommHost(multistep.StateBag) (string, error) // Start starts a VM specified by the path to the VMX given. Start(string, bool) error diff --git a/builder/vmware/common/driver_fusion5.go b/builder/vmware/common/driver_fusion5.go index 3a295e731..a10f11902 100644 --- a/builder/vmware/common/driver_fusion5.go +++ b/builder/vmware/common/driver_fusion5.go @@ -69,8 +69,8 @@ func (d *Fusion5Driver) IsRunning(vmxPath string) (bool, error) { return false, nil } -func (d *Fusion5Driver) SSHAddress(state multistep.StateBag) (string, error) { - return SSHAddressFunc(d.SSHConfig)(state) +func (d *Fusion5Driver) CommHost(state multistep.StateBag) (string, error) { + return CommHost(d.SSHConfig)(state) } func (d *Fusion5Driver) Start(vmxPath string, headless bool) error { diff --git a/builder/vmware/common/driver_mock.go b/builder/vmware/common/driver_mock.go index 6aa0d02c0..fcd80a51b 100644 --- a/builder/vmware/common/driver_mock.go +++ b/builder/vmware/common/driver_mock.go @@ -29,10 +29,10 @@ type DriverMock struct { IsRunningResult bool IsRunningErr error - SSHAddressCalled bool - SSHAddressState multistep.StateBag - SSHAddressResult string - SSHAddressErr error + CommHostCalled bool + CommHostState multistep.StateBag + CommHostResult string + CommHostErr error StartCalled bool StartPath string @@ -92,10 +92,10 @@ func (d *DriverMock) IsRunning(path string) (bool, error) { return d.IsRunningResult, d.IsRunningErr } -func (d *DriverMock) SSHAddress(state multistep.StateBag) (string, error) { - d.SSHAddressCalled = true - d.SSHAddressState = state - return d.SSHAddressResult, d.SSHAddressErr +func (d *DriverMock) CommHost(state multistep.StateBag) (string, error) { + d.CommHostCalled = true + d.CommHostState = state + return d.CommHostResult, d.CommHostErr } func (d *DriverMock) Start(path string, headless bool) error { diff --git a/builder/vmware/common/driver_player5.go b/builder/vmware/common/driver_player5.go index 5bb80a0f2..1552e92ea 100644 --- a/builder/vmware/common/driver_player5.go +++ b/builder/vmware/common/driver_player5.go @@ -97,8 +97,8 @@ func (d *Player5Driver) IsRunning(vmxPath string) (bool, error) { return false, nil } -func (d *Player5Driver) SSHAddress(state multistep.StateBag) (string, error) { - return SSHAddressFunc(d.SSHConfig)(state) +func (d *Player5Driver) CommHost(state multistep.StateBag) (string, error) { + return CommHost(d.SSHConfig)(state) } func (d *Player5Driver) Start(vmxPath string, headless bool) error { diff --git a/builder/vmware/common/driver_workstation9.go b/builder/vmware/common/driver_workstation9.go index 4c72c72b3..debcefcbc 100644 --- a/builder/vmware/common/driver_workstation9.go +++ b/builder/vmware/common/driver_workstation9.go @@ -70,8 +70,8 @@ func (d *Workstation9Driver) IsRunning(vmxPath string) (bool, error) { return false, nil } -func (d *Workstation9Driver) SSHAddress(state multistep.StateBag) (string, error) { - return SSHAddressFunc(d.SSHConfig)(state) +func (d *Workstation9Driver) CommHost(state multistep.StateBag) (string, error) { + return CommHost(d.SSHConfig)(state) } func (d *Workstation9Driver) Start(vmxPath string, headless bool) error { diff --git a/builder/vmware/common/ssh.go b/builder/vmware/common/ssh.go index 7181e90bf..86e184bb5 100644 --- a/builder/vmware/common/ssh.go +++ b/builder/vmware/common/ssh.go @@ -13,13 +13,13 @@ import ( gossh "golang.org/x/crypto/ssh" ) -func SSHAddressFunc(config *SSHConfig) func(multistep.StateBag) (string, error) { +func CommHost(config *SSHConfig) func(multistep.StateBag) (string, error) { return func(state multistep.StateBag) (string, error) { driver := state.Get("driver").(Driver) vmxPath := state.Get("vmx_path").(string) if config.Comm.SSHHost != "" { - return fmt.Sprintf("%s:%d", config.Comm.SSHHost, config.Comm.SSHPort), nil + return config.Comm.SSHHost, nil } log.Println("Lookup up IP information...") @@ -62,7 +62,7 @@ func SSHAddressFunc(config *SSHConfig) func(multistep.StateBag) (string, error) } log.Printf("Detected IP: %s", ipAddress) - return fmt.Sprintf("%s:%d", ipAddress, config.Comm.SSHPort), nil + return ipAddress, nil } } diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go index 9599df675..38ba3a4a1 100644 --- a/builder/vmware/iso/builder.go +++ b/builder/vmware/iso/builder.go @@ -300,9 +300,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Ctx: b.config.ctx, }, &communicator.StepConnect{ - Config: &b.config.SSHConfig.Comm, - SSHAddress: driver.SSHAddress, - SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), + Config: &b.config.SSHConfig.Comm, + Host: driver.CommHost, + SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), }, &vmwcommon.StepUploadTools{ RemoteType: b.config.RemoteType, diff --git a/builder/vmware/iso/driver_esx5.go b/builder/vmware/iso/driver_esx5.go index 26642f3d7..8162db468 100644 --- a/builder/vmware/iso/driver_esx5.go +++ b/builder/vmware/iso/driver_esx5.go @@ -218,7 +218,7 @@ func (d *ESX5Driver) VNCAddress(portMin, portMax uint) (string, uint, error) { return d.Host, vncPort, nil } -func (d *ESX5Driver) SSHAddress(state multistep.StateBag) (string, error) { +func (d *ESX5Driver) CommHost(state multistep.StateBag) (string, error) { config := state.Get("config").(*Config) if address, ok := state.GetOk("vm_address"); ok { @@ -253,7 +253,7 @@ func (d *ESX5Driver) SSHAddress(state multistep.StateBag) (string, error) { return "", errors.New("VM network port found, but no IP address") } - address := fmt.Sprintf("%s:%d", record["IPAddress"], config.Comm.SSHPort) + address := record["IPAddress"] state.Put("vm_address", address) return address, nil } diff --git a/builder/vmware/vmx/builder.go b/builder/vmware/vmx/builder.go index 1390da547..aa86d3669 100644 --- a/builder/vmware/vmx/builder.go +++ b/builder/vmware/vmx/builder.go @@ -92,9 +92,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Ctx: b.config.ctx, }, &communicator.StepConnect{ - Config: &b.config.SSHConfig.Comm, - SSHAddress: driver.SSHAddress, - SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), + Config: &b.config.SSHConfig.Comm, + Host: driver.CommHost, + SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig), }, &vmwcommon.StepUploadTools{ RemoteType: b.config.RemoteType, diff --git a/helper/communicator/step_connect.go b/helper/communicator/step_connect.go index e6338027e..a31dd4eb8 100644 --- a/helper/communicator/step_connect.go +++ b/helper/communicator/step_connect.go @@ -15,15 +15,16 @@ type StepConnect struct { // Config is the communicator config struct Config *Config + // Host should return a host that can be connected to for communicator + // connections. + Host func(multistep.StateBag) (string, error) + // The fields below are callbacks to assist with connecting to SSH. // - // SSHAddress should return the default host to connect to for SSH. - // This is only called if ssh_host isn't specified in the config. - // // SSHConfig should return the default configuration for // connecting via SSH. - SSHAddress func(multistep.StateBag) (string, error) - SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error) + SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error) + SSHPort func(multistep.StateBag) (int, error) substep multistep.Step } @@ -32,9 +33,10 @@ func (s *StepConnect) Run(state multistep.StateBag) multistep.StepAction { typeMap := map[string]multistep.Step{ "none": nil, "ssh": &StepConnectSSH{ - Config: s.Config, - SSHAddress: s.SSHAddress, - SSHConfig: s.SSHConfig, + Config: s.Config, + Host: s.Host, + SSHConfig: s.SSHConfig, + SSHPort: s.SSHPort, }, } diff --git a/helper/communicator/step_connect_ssh.go b/helper/communicator/step_connect_ssh.go index 9be653c01..0b54bae9d 100644 --- a/helper/communicator/step_connect_ssh.go +++ b/helper/communicator/step_connect_ssh.go @@ -18,9 +18,10 @@ import ( // In general, you should use StepConnect. type StepConnectSSH struct { // All the fields below are documented on StepConnect - Config *Config - SSHAddress func(multistep.StateBag) (string, error) - SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error) + Config *Config + Host func(multistep.StateBag) (string, error) + SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error) + SSHPort func(multistep.StateBag) (int, error) } func (s *StepConnectSSH) Run(state multistep.StateBag) multistep.StepAction { @@ -95,11 +96,19 @@ func (s *StepConnectSSH) waitForSSH(state multistep.StateBag, cancel <-chan stru first = false // First we request the TCP connection information - address, err := s.SSHAddress(state) + host, err := s.Host(state) if err != nil { log.Printf("[DEBUG] Error getting SSH address: %s", err) continue } + port := s.Config.SSHPort + if s.SSHPort != nil { + port, err = s.SSHPort(state) + if err != nil { + log.Printf("[DEBUG] Error getting SSH port: %s", err) + continue + } + } // Retrieve the SSH configuration sshConfig, err := s.SSHConfig(state) @@ -108,6 +117,8 @@ func (s *StepConnectSSH) waitForSSH(state multistep.StateBag, cancel <-chan stru continue } + address := fmt.Sprintf("%s:%d", host, port) + // Attempt to connect to SSH port connFunc := ssh.ConnectFunc("tcp", address) nc, err := connFunc() From c3cc9e844e7a036f7939c792353539e441d3244c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 19:24:57 -0400 Subject: [PATCH 166/208] helper/communicator: fix vet --- helper/communicator/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper/communicator/config_test.go b/helper/communicator/config_test.go index 029c9fe35..dc1bd965d 100644 --- a/helper/communicator/config_test.go +++ b/helper/communicator/config_test.go @@ -19,7 +19,7 @@ func TestConfigType(t *testing.T) { } if c.Type != "ssh" { - t.Fatal("bad: %#v", c) + t.Fatalf("bad: %#v", c) } } From ea86cb4c7e6c3ca3e8f894d8277ce91bc478b8bf Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 19:30:16 -0400 Subject: [PATCH 167/208] builder/qemu: use proper ssh port [GH-2074] --- builder/qemu/step_run.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builder/qemu/step_run.go b/builder/qemu/step_run.go index 4f64c4d8b..816a3d3d3 100644 --- a/builder/qemu/step_run.go +++ b/builder/qemu/step_run.go @@ -80,7 +80,8 @@ func getCommandArgs(bootDrive string, state multistep.StateBag) ([]string, error defaultArgs["-name"] = vmName defaultArgs["-machine"] = fmt.Sprintf("type=%s", config.MachineType) - defaultArgs["-netdev"] = fmt.Sprintf("user,id=user.0,hostfwd=tcp::%v-:22", sshHostPort) + defaultArgs["-netdev"] = fmt.Sprintf( + "user,id=user.0,hostfwd=tcp::%v-:%d", sshHostPort, config.SSHPort) defaultArgs["-device"] = fmt.Sprintf("%s,netdev=user.0", config.NetDevice) defaultArgs["-drive"] = fmt.Sprintf("file=%s,if=%s,cache=%s,discard=%s", imgPath, config.DiskInterface, config.DiskCache, config.DiskDiscard) if !config.DiskImage { From 8d0904e296065a2951ef0856d2ce9b5e9c4b6c7e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 19:39:42 -0400 Subject: [PATCH 168/208] helper/communicator: configurable handshake attempts [GH-1988] --- helper/communicator/config.go | 21 +++++++++++++-------- helper/communicator/step_connect_ssh.go | 6 ++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/helper/communicator/config.go b/helper/communicator/config.go index a2d93a480..6a5e7b97b 100644 --- a/helper/communicator/config.go +++ b/helper/communicator/config.go @@ -12,14 +12,15 @@ import ( // Config is the common configuration that communicators allow within // a builder. type Config struct { - Type string `mapstructure:"communicator"` - SSHHost string `mapstructure:"ssh_host"` - SSHPort int `mapstructure:"ssh_port"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPassword string `mapstructure:"ssh_password"` - SSHPrivateKey string `mapstructure:"ssh_private_key_file"` - SSHPty bool `mapstructure:"ssh_pty"` - SSHTimeout time.Duration `mapstructure:"ssh_timeout"` + Type string `mapstructure:"communicator"` + SSHHost string `mapstructure:"ssh_host"` + SSHPort int `mapstructure:"ssh_port"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPassword string `mapstructure:"ssh_password"` + SSHPrivateKey string `mapstructure:"ssh_private_key_file"` + SSHPty bool `mapstructure:"ssh_pty"` + SSHTimeout time.Duration `mapstructure:"ssh_timeout"` + SSHHandshakeAttempts int `mapstructure:"ssh_handshake_attempts"` } func (c *Config) Prepare(ctx *interpolate.Context) []error { @@ -35,6 +36,10 @@ func (c *Config) Prepare(ctx *interpolate.Context) []error { c.SSHTimeout = 5 * time.Minute } + if c.SSHHandshakeAttempts == 0 { + c.SSHHandshakeAttempts = 10 + } + // Validation var errs []error if c.Type == "ssh" { diff --git a/helper/communicator/step_connect_ssh.go b/helper/communicator/step_connect_ssh.go index 0b54bae9d..4b664fe4c 100644 --- a/helper/communicator/step_connect_ssh.go +++ b/helper/communicator/step_connect_ssh.go @@ -149,8 +149,10 @@ func (s *StepConnectSSH) waitForSSH(state multistep.StateBag, cancel <-chan stru handshakeAttempts += 1 } - if handshakeAttempts < 10 { - // Try to connect via SSH a handful of times + if handshakeAttempts < s.Config.SSHHandshakeAttempts { + // Try to connect via SSH a handful of times. We sleep here + // so we don't get a ton of authentication errors back to back. + time.Sleep(2 * time.Second) continue } From 74b9da5b23bef88dae9ca5918aabc764da630766 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 19:48:25 -0400 Subject: [PATCH 169/208] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c933040c1..4391393d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ FEATURES: * **New config function: `template_dir`**: The directory to the template being built. This should be used for template-relative paths. [GH-54] + * **Disable SSH:** Set `communicator` to "none" in any builder to disable SSH + connections. Note that provisioners won't work if this is done. [GH-1591] IMPROVEMENTS: From b2f8eb68e8f431f803531a87821a3238faf821dc Mon Sep 17 00:00:00 2001 From: Mark Peek Date: Sat, 13 Jun 2015 17:15:49 -0700 Subject: [PATCH 170/208] Enable ssh agent forwarding #1066 --- CHANGELOG.md | 2 + communicator/ssh/communicator.go | 50 +++++++++++++++++++ .../docs/provisioners/shell.html.markdown | 27 ++++++++++ 3 files changed, 79 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4391393d0..9e26c7818 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ FEATURES: being built. This should be used for template-relative paths. [GH-54] * **Disable SSH:** Set `communicator` to "none" in any builder to disable SSH connections. Note that provisioners won't work if this is done. [GH-1591] + * **SSH Agent Forwarding:** SSH Agent Forwarding will now be enabled + to allow access to remote servers such as private git repos. [GH-1066] IMPROVEMENTS: diff --git a/communicator/ssh/communicator.go b/communicator/ssh/communicator.go index 217afc940..193ecae42 100644 --- a/communicator/ssh/communicator.go +++ b/communicator/ssh/communicator.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/mitchellh/packer/packer" "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" "io" "io/ioutil" "log" @@ -226,10 +227,59 @@ func (c *comm) reconnect() (err error) { if sshConn != nil { c.client = ssh.NewClient(sshConn, sshChan, req) } + c.connectToAgent() return } +func (c *comm) connectToAgent() { + if c.client == nil { + return + } + + // open connection to the local agent + socketLocation := os.Getenv("SSH_AUTH_SOCK") + if socketLocation == "" { + log.Printf("no local agent socket") + return + } + agentConn, err := net.Dial("unix", socketLocation) + if err != nil { + log.Printf("could not connect to local agent socket: %s", socketLocation) + return + } + + // create agent and add in auth + forwardingAgent := agent.NewClient(agentConn) + if forwardingAgent == nil { + log.Printf("could not create agent client") + agentConn.Close() + return + } + + // add callback for forwarding agent to SSH config + // XXX - might want to handle reconnects appending multiple callbacks + auth := ssh.PublicKeysCallback(forwardingAgent.Signers) + c.config.SSHConfig.Auth = append(c.config.SSHConfig.Auth, auth) + agent.ForwardToAgent(c.client, forwardingAgent) + + // Setup a session to request agent forwarding + session, err := c.newSession() + if err != nil { + return + } + defer session.Close() + + err = agent.RequestAgentForwarding(session) + if err != nil { + log.Printf("RequestAgentForwarding:", err) + return + } + + log.Printf("agent forwarding enabled") + return +} + func (c *comm) scpSession(scpCommand string, f func(io.Writer, *bufio.Reader) error) error { session, err := c.newSession() if err != nil { diff --git a/website/source/docs/provisioners/shell.html.markdown b/website/source/docs/provisioners/shell.html.markdown index e57910cb0..89a442a83 100644 --- a/website/source/docs/provisioners/shell.html.markdown +++ b/website/source/docs/provisioners/shell.html.markdown @@ -146,6 +146,33 @@ on reboot or in your shell script. For example, on Gentoo: /etc/init.d/net.eth0 stop ``` +## SSH Agent Forwarding + +Some provisioning requires connecting to remote SSH servers from within the +packer instance. The below example is for pulling code from a private git +repository utilizing openssh on the client. Make sure you are running +`ssh-agent` and add your git repo ssh keys into it using `ssh-add /path/to/key`. +When the packer instance needs access to the ssh keys the agent will forward +the request back to your `ssh-agent`. + +Note: when provisioning via git you should add the git server keys into +the `~/.ssh/known_hosts` file otherwise the git command could hang awaiting +input. This can be done by copying the file in via the +[file provisioner](/docs/provisioners/file.html) (more secure) +or using `ssh-keyscan` to populate the file (less secure). An example of the +latter accessing github would be: + +``` +{ + "type": "shell", + "inline": [ + "sudo apt-get install -y git", + "ssh-keyscan github.com >> ~/.ssh/known_hosts", + "git clone git@github.com:exampleorg/myprivaterepo.git" + ] +} +``` + ## Troubleshooting *My shell script doesn't work correctly on Ubuntu* From acf31c31a13bff4c35aa15f12f194119ad5c317d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 19:00:28 -0700 Subject: [PATCH 171/208] communicator/ssh: update logging --- communicator/ssh/communicator.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/communicator/ssh/communicator.go b/communicator/ssh/communicator.go index 193ecae42..8fd9ba91e 100644 --- a/communicator/ssh/communicator.go +++ b/communicator/ssh/communicator.go @@ -240,19 +240,19 @@ func (c *comm) connectToAgent() { // open connection to the local agent socketLocation := os.Getenv("SSH_AUTH_SOCK") if socketLocation == "" { - log.Printf("no local agent socket") + log.Printf("[INFO] no local agent socket, will not connect agent") return } agentConn, err := net.Dial("unix", socketLocation) if err != nil { - log.Printf("could not connect to local agent socket: %s", socketLocation) + log.Printf("[ERROR] could not connect to local agent socket: %s", socketLocation) return } // create agent and add in auth forwardingAgent := agent.NewClient(agentConn) if forwardingAgent == nil { - log.Printf("could not create agent client") + log.Printf("[ERROR] Could not create agent client") agentConn.Close() return } @@ -272,11 +272,11 @@ func (c *comm) connectToAgent() { err = agent.RequestAgentForwarding(session) if err != nil { - log.Printf("RequestAgentForwarding:", err) + log.Printf("[ERROR] RequestAgentForwarding: %#v", err) return } - log.Printf("agent forwarding enabled") + log.Printf("[INFO] agent forwarding enabled") return } From 0c0f876654ef15e8157677f24b5028bd51fb3cc4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:00:40 -0700 Subject: [PATCH 172/208] communicator/winrm --- communicator/winrm/communicator.go | 129 ++++++++++++++++++++++++ communicator/winrm/communicator_test.go | 94 +++++++++++++++++ communicator/winrm/config.go | 14 +++ communicator/winrm/time.go | 32 ++++++ communicator/winrm/time_test.go | 36 +++++++ 5 files changed, 305 insertions(+) create mode 100644 communicator/winrm/communicator.go create mode 100644 communicator/winrm/communicator_test.go create mode 100644 communicator/winrm/config.go create mode 100644 communicator/winrm/time.go create mode 100644 communicator/winrm/time_test.go diff --git a/communicator/winrm/communicator.go b/communicator/winrm/communicator.go new file mode 100644 index 000000000..d0b7eb76c --- /dev/null +++ b/communicator/winrm/communicator.go @@ -0,0 +1,129 @@ +package winrm + +import ( + "fmt" + "io" + "log" + + "github.com/masterzen/winrm/winrm" + "github.com/mitchellh/packer/packer" + "github.com/packer-community/winrmcp/winrmcp" + + // This import is a bit strange, but it's needed so `make updatedeps` + // can see and download it + _ "github.com/dylanmei/winrmtest" +) + +// Communicator represents the WinRM communicator +type Communicator struct { + config *Config + client *winrm.Client + endpoint *winrm.Endpoint +} + +// New creates a new communicator implementation over WinRM. +func New(config *Config) (*Communicator, error) { + endpoint := &winrm.Endpoint{ + Host: config.Host, + Port: config.Port, + + /* + TODO + HTTPS: connInfo.HTTPS, + Insecure: connInfo.Insecure, + CACert: connInfo.CACert, + */ + } + + // Create the client + params := winrm.DefaultParameters() + params.Timeout = formatDuration(config.Timeout) + client, err := winrm.NewClientWithParameters( + endpoint, config.Username, config.Password, params) + if err != nil { + return nil, err + } + + // Create the shell to verify the connection + log.Printf("[DEBUG] connecting to remote shell using WinRM") + shell, err := client.CreateShell() + if err != nil { + log.Printf("[ERROR] connection error: %s", err) + return nil, err + } + + if err := shell.Close(); err != nil { + log.Printf("[ERROR] error closing connection: %s", err) + return nil, err + } + + return &Communicator{ + config: config, + client: client, + endpoint: endpoint, + }, nil +} + +// Start implementation of communicator.Communicator interface +func (c *Communicator) Start(rc *packer.RemoteCmd) error { + shell, err := c.client.CreateShell() + if err != nil { + return err + } + + log.Printf("[INFO] starting remote command: %s", rc.Command) + cmd, err := shell.Execute(rc.Command) + if err != nil { + return err + } + + go runCommand(shell, cmd, rc) + return nil +} + +func runCommand(shell *winrm.Shell, cmd *winrm.Command, rc *packer.RemoteCmd) { + defer shell.Close() + + go io.Copy(rc.Stdout, cmd.Stdout) + go io.Copy(rc.Stderr, cmd.Stderr) + + cmd.Wait() + rc.SetExited(cmd.ExitCode()) +} + +// Upload implementation of communicator.Communicator interface +func (c *Communicator) Upload(path string, input io.Reader) error { + wcp, err := c.newCopyClient() + if err != nil { + return err + } + log.Printf("Uploading file to '%s'", path) + return wcp.Write(path, input) +} + +// UploadScript implementation of communicator.Communicator interface +func (c *Communicator) UploadScript(path string, input io.Reader) error { + return c.Upload(path, input) +} + +// UploadDir implementation of communicator.Communicator interface +func (c *Communicator) UploadDir(dst string, src string) error { + log.Printf("Uploading dir '%s' to '%s'", src, dst) + wcp, err := c.newCopyClient() + if err != nil { + return err + } + return wcp.Copy(src, dst) +} + +func (c *Communicator) newCopyClient() (*winrmcp.Winrmcp, error) { + addr := fmt.Sprintf("%s:%d", c.endpoint.Host, c.endpoint.Port) + return winrmcp.New(addr, &winrmcp.Config{ + Auth: winrmcp.Auth{ + User: c.config.Username, + Password: c.config.Password, + }, + OperationTimeout: c.config.Timeout, + MaxOperationsPerShell: 15, // lowest common denominator + }) +} diff --git a/communicator/winrm/communicator_test.go b/communicator/winrm/communicator_test.go new file mode 100644 index 000000000..73ac6d7b2 --- /dev/null +++ b/communicator/winrm/communicator_test.go @@ -0,0 +1,94 @@ +package winrm + +import ( + "bytes" + "io" + "testing" + "time" + + "github.com/dylanmei/winrmtest" + "github.com/mitchellh/packer/packer" +) + +func newMockWinRMServer(t *testing.T) *winrmtest.Remote { + wrm := winrmtest.NewRemote() + + wrm.CommandFunc( + winrmtest.MatchText("echo foo"), + func(out, err io.Writer) int { + out.Write([]byte("foo")) + return 0 + }) + + wrm.CommandFunc( + winrmtest.MatchPattern(`^echo c29tZXRoaW5n >> ".*"$`), + func(out, err io.Writer) int { + return 0 + }) + + wrm.CommandFunc( + winrmtest.MatchPattern(`^powershell.exe -EncodedCommand .*$`), + func(out, err io.Writer) int { + return 0 + }) + + wrm.CommandFunc( + winrmtest.MatchText("powershell"), + func(out, err io.Writer) int { + return 0 + }) + + return wrm +} + +func TestStart(t *testing.T) { + wrm := newMockWinRMServer(t) + defer wrm.Close() + + c, err := New(&Config{ + Host: wrm.Host, + Port: wrm.Port, + Username: "user", + Password: "pass", + Timeout: 30 * time.Second, + }) + if err != nil { + t.Fatalf("error creating communicator: %s", err) + } + + var cmd packer.RemoteCmd + stdout := new(bytes.Buffer) + cmd.Command = "echo foo" + cmd.Stdout = stdout + + err = c.Start(&cmd) + if err != nil { + t.Fatalf("error executing remote command: %s", err) + } + cmd.Wait() + + if stdout.String() != "foo" { + t.Fatalf("bad command response: expected %q, got %q", "foo", stdout.String()) + } +} + +func TestUpload(t *testing.T) { + wrm := newMockWinRMServer(t) + defer wrm.Close() + + c, err := New(&Config{ + Host: wrm.Host, + Port: wrm.Port, + Username: "user", + Password: "pass", + Timeout: 30 * time.Second, + }) + if err != nil { + t.Fatalf("error creating communicator: %s", err) + } + + err = c.Upload("C:/Temp/terraform.cmd", bytes.NewReader([]byte("something"))) + if err != nil { + t.Fatalf("error uploading file: %s", err) + } +} diff --git a/communicator/winrm/config.go b/communicator/winrm/config.go new file mode 100644 index 000000000..32c082987 --- /dev/null +++ b/communicator/winrm/config.go @@ -0,0 +1,14 @@ +package winrm + +import ( + "time" +) + +// Config is used to configure the WinRM connection +type Config struct { + Host string + Port int + Username string + Password string + Timeout time.Duration +} diff --git a/communicator/winrm/time.go b/communicator/winrm/time.go new file mode 100644 index 000000000..f8fb6fe8d --- /dev/null +++ b/communicator/winrm/time.go @@ -0,0 +1,32 @@ +package winrm + +import ( + "fmt" + "time" +) + +// formatDuration formats the given time.Duration into an ISO8601 +// duration string. +func formatDuration(duration time.Duration) string { + // We're not supporting negative durations + if duration.Seconds() <= 0 { + return "PT0S" + } + + h := int(duration.Hours()) + m := int(duration.Minutes()) - (h * 60) + s := int(duration.Seconds()) - (h*3600 + m*60) + + res := "PT" + if h > 0 { + res = fmt.Sprintf("%s%dH", res, h) + } + if m > 0 { + res = fmt.Sprintf("%s%dM", res, m) + } + if s > 0 { + res = fmt.Sprintf("%s%dS", res, s) + } + + return res +} diff --git a/communicator/winrm/time_test.go b/communicator/winrm/time_test.go new file mode 100644 index 000000000..4daf4cedf --- /dev/null +++ b/communicator/winrm/time_test.go @@ -0,0 +1,36 @@ +package winrm + +import ( + "testing" + "time" +) + +func TestFormatDuration(t *testing.T) { + // Test complex duration with hours, minutes, seconds + d := time.Duration(3701) * time.Second + s := formatDuration(d) + if s != "PT1H1M41S" { + t.Fatalf("bad ISO 8601 duration string: %s", s) + } + + // Test only minutes duration + d = time.Duration(20) * time.Minute + s = formatDuration(d) + if s != "PT20M" { + t.Fatalf("bad ISO 8601 duration string for 20M: %s", s) + } + + // Test only seconds + d = time.Duration(1) * time.Second + s = formatDuration(d) + if s != "PT1S" { + t.Fatalf("bad ISO 8601 duration string for 1S: %s", s) + } + + // Test negative duration (unsupported) + d = time.Duration(-1) * time.Second + s = formatDuration(d) + if s != "PT0S" { + t.Fatalf("bad ISO 8601 duration string for negative: %s", s) + } +} From 7a3975805465d1ef31855465e71254dc34be7851 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:05:48 -0700 Subject: [PATCH 173/208] helper/communicator: WinRM stuff --- communicator/winrm/communicator.go | 14 +-- helper/communicator/config.go | 66 ++++++++--- helper/communicator/step_connect_winrm.go | 134 ++++++++++++++++++++++ helper/communicator/winrm.go | 8 ++ 4 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 helper/communicator/step_connect_winrm.go create mode 100644 helper/communicator/winrm.go diff --git a/communicator/winrm/communicator.go b/communicator/winrm/communicator.go index d0b7eb76c..82686e2a7 100644 --- a/communicator/winrm/communicator.go +++ b/communicator/winrm/communicator.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "log" + "os" "github.com/masterzen/winrm/winrm" "github.com/mitchellh/packer/packer" @@ -92,7 +93,7 @@ func runCommand(shell *winrm.Shell, cmd *winrm.Command, rc *packer.RemoteCmd) { } // Upload implementation of communicator.Communicator interface -func (c *Communicator) Upload(path string, input io.Reader) error { +func (c *Communicator) Upload(path string, input io.Reader, _ *os.FileInfo) error { wcp, err := c.newCopyClient() if err != nil { return err @@ -101,13 +102,8 @@ func (c *Communicator) Upload(path string, input io.Reader) error { return wcp.Write(path, input) } -// UploadScript implementation of communicator.Communicator interface -func (c *Communicator) UploadScript(path string, input io.Reader) error { - return c.Upload(path, input) -} - // UploadDir implementation of communicator.Communicator interface -func (c *Communicator) UploadDir(dst string, src string) error { +func (c *Communicator) UploadDir(dst string, src string, exclude []string) error { log.Printf("Uploading dir '%s' to '%s'", src, dst) wcp, err := c.newCopyClient() if err != nil { @@ -116,6 +112,10 @@ func (c *Communicator) UploadDir(dst string, src string) error { return wcp.Copy(src, dst) } +func (c *Communicator) Download(src string, dst io.Writer) error { + panic("download not implemented") +} + func (c *Communicator) newCopyClient() (*winrmcp.Winrmcp, error) { addr := fmt.Sprintf("%s:%d", c.endpoint.Host, c.endpoint.Port) return winrmcp.New(addr, &winrmcp.Config{ diff --git a/helper/communicator/config.go b/helper/communicator/config.go index a2d93a480..f0cb78df7 100644 --- a/helper/communicator/config.go +++ b/helper/communicator/config.go @@ -12,7 +12,9 @@ import ( // Config is the common configuration that communicators allow within // a builder. type Config struct { - Type string `mapstructure:"communicator"` + Type string `mapstructure:"communicator"` + + // SSH SSHHost string `mapstructure:"ssh_host"` SSHPort int `mapstructure:"ssh_port"` SSHUsername string `mapstructure:"ssh_username"` @@ -20,6 +22,13 @@ type Config struct { SSHPrivateKey string `mapstructure:"ssh_private_key_file"` SSHPty bool `mapstructure:"ssh_pty"` SSHTimeout time.Duration `mapstructure:"ssh_timeout"` + + // WinRM + WinRMUser string `mapstructure:"winrm_username"` + WinRMPassword string `mapstructure:"winrm_password"` + WinRMHost string `mapstructure:"winrm_host"` + WinRMPort int `mapstructure:"winrm_port"` + WinRMTimeout time.Duration `mapstructure:"winrm_timeout"` } func (c *Config) Prepare(ctx *interpolate.Context) []error { @@ -27,6 +36,22 @@ func (c *Config) Prepare(ctx *interpolate.Context) []error { c.Type = "ssh" } + var errs []error + switch c.Type { + case "ssh": + if es := c.prepareSSH(ctx); len(es) > 0 { + errs = append(errs, es...) + } + case "winrm": + if es := c.prepareWinRM(ctx); len(es) > 0 { + errs = append(errs, es...) + } + } + + return errs +} + +func (c *Config) prepareSSH(ctx *interpolate.Context) []error { if c.SSHPort == 0 { c.SSHPort = 22 } @@ -37,21 +62,36 @@ func (c *Config) Prepare(ctx *interpolate.Context) []error { // Validation var errs []error - if c.Type == "ssh" { - if c.SSHUsername == "" { - errs = append(errs, errors.New("An ssh_username must be specified")) - } + if c.SSHUsername == "" { + errs = append(errs, errors.New("An ssh_username must be specified")) + } - if c.SSHPrivateKey != "" { - if _, err := os.Stat(c.SSHPrivateKey); err != nil { - errs = append(errs, fmt.Errorf( - "ssh_private_key_file is invalid: %s", err)) - } else if _, err := SSHFileSigner(c.SSHPrivateKey); err != nil { - errs = append(errs, fmt.Errorf( - "ssh_private_key_file is invalid: %s", err)) - } + if c.SSHPrivateKey != "" { + if _, err := os.Stat(c.SSHPrivateKey); err != nil { + errs = append(errs, fmt.Errorf( + "ssh_private_key_file is invalid: %s", err)) + } else if _, err := SSHFileSigner(c.SSHPrivateKey); err != nil { + errs = append(errs, fmt.Errorf( + "ssh_private_key_file is invalid: %s", err)) } } return errs } + +func (c *Config) prepareWinRM(ctx *interpolate.Context) []error { + if c.WinRMPort == 0 { + c.WinRMPort = 5985 + } + + if c.WinRMTimeout == 0 { + c.WinRMTimeout = 30 * time.Minute + } + + var errs []error + if c.WinRMUser == "" { + errs = append(errs, errors.New("winrm_username must be specified.")) + } + + return errs +} diff --git a/helper/communicator/step_connect_winrm.go b/helper/communicator/step_connect_winrm.go new file mode 100644 index 000000000..bdd0c1499 --- /dev/null +++ b/helper/communicator/step_connect_winrm.go @@ -0,0 +1,134 @@ +package communicator + +import ( + "errors" + "fmt" + "log" + "time" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/communicator/winrm" + "github.com/mitchellh/packer/packer" +) + +// StepConnectWinRM is a multistep Step implementation that waits for WinRM +// to become available. It gets the connection information from a single +// configuration when creating the step. +// +// Uses: +// ui packer.Ui +// +// Produces: +// communicator packer.Communicator +type StepConnectWinRM struct { + // All the fields below are documented on StepConnect + Config *Config + Host func(multistep.StateBag) (string, error) + WinRMConfig func(multistep.StateBag) (*WinRMConfig, error) +} + +func (s *StepConnectWinRM) Run(state multistep.StateBag) multistep.StepAction { + ui := state.Get("ui").(packer.Ui) + + var comm packer.Communicator + var err error + + cancel := make(chan struct{}) + waitDone := make(chan bool, 1) + go func() { + ui.Say("Waiting for WinRM to become available...") + comm, err = s.waitForWinRM(state, cancel) + waitDone <- true + }() + + log.Printf("Waiting for WinRM, up to timeout: %s", s.Config.WinRMTimeout) + timeout := time.After(s.Config.WinRMTimeout) +WaitLoop: + for { + // Wait for either WinRM to become available, a timeout to occur, + // or an interrupt to come through. + select { + case <-waitDone: + if err != nil { + ui.Error(fmt.Sprintf("Error waiting for WinRM: %s", err)) + return multistep.ActionHalt + } + + ui.Say("Connected to WinRM!") + state.Put("communicator", comm) + break WaitLoop + case <-timeout: + err := fmt.Errorf("Timeout waiting for WinRM.") + state.Put("error", err) + ui.Error(err.Error()) + close(cancel) + return multistep.ActionHalt + case <-time.After(1 * time.Second): + if _, ok := state.GetOk(multistep.StateCancelled); ok { + // The step sequence was cancelled, so cancel waiting for WinRM + // and just start the halting process. + close(cancel) + log.Println("Interrupt detected, quitting waiting for WinRM.") + return multistep.ActionHalt + } + } + } + + return multistep.ActionContinue +} + +func (s *StepConnectWinRM) Cleanup(multistep.StateBag) { +} + +func (s *StepConnectWinRM) waitForWinRM(state multistep.StateBag, cancel <-chan struct{}) (packer.Communicator, error) { + var comm packer.Communicator + for { + select { + case <-cancel: + log.Println("[INFO] WinRM wait cancelled. Exiting loop.") + return nil, errors.New("WinRM wait cancelled") + case <-time.After(5 * time.Second): + } + + host, err := s.Host(state) + if err != nil { + log.Printf("[DEBUG] Error getting WinRM host: %s", err) + continue + } + port := s.Config.WinRMPort + + user := s.Config.WinRMUser + password := s.Config.WinRMPassword + if s.WinRMConfig != nil { + config, err := s.WinRMConfig(state) + if err != nil { + log.Printf("[DEBUG] Error getting WinRM config: %s", err) + continue + } + + if config.Username != "" { + user = config.Username + } + if config.Password != "" { + password = config.Password + } + } + + log.Println("[INFO] Attempting WinRM connection...") + comm, err = winrm.New(&winrm.Config{ + Host: host, + Port: port, + Username: user, + Password: password, + Timeout: s.Config.WinRMTimeout, + }) + if err != nil { + log.Printf("[ERROR] WinRM connection err: %s", err) + continue + } + + break + } + + return comm, nil +} diff --git a/helper/communicator/winrm.go b/helper/communicator/winrm.go new file mode 100644 index 000000000..afdf2569d --- /dev/null +++ b/helper/communicator/winrm.go @@ -0,0 +1,8 @@ +package communicator + +// WinRMConfig is configuration that can be returned at runtime to +// dynamically configure WinRM. +type WinRMConfig struct { + Username string + Password string +} From 4be10b428a78126b012fe6d18b78f790d19b4c58 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:07:17 -0700 Subject: [PATCH 174/208] helper/communicator: hook up WinRM --- helper/communicator/step_connect.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/helper/communicator/step_connect.go b/helper/communicator/step_connect.go index a31dd4eb8..ce77333e1 100644 --- a/helper/communicator/step_connect.go +++ b/helper/communicator/step_connect.go @@ -26,6 +26,12 @@ type StepConnect struct { SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error) SSHPort func(multistep.StateBag) (int, error) + // The fields below are callbacks to assist with connecting to WinRM. + // + // WinRMConfig should return the default configuration for + // connecting via WinRM. + WinRMConfig func(multistep.StateBag) (*WinRMConfig, error) + substep multistep.Step } @@ -38,6 +44,11 @@ func (s *StepConnect) Run(state multistep.StateBag) multistep.StepAction { SSHConfig: s.SSHConfig, SSHPort: s.SSHPort, }, + "winrm": &StepConnectWinRM{ + Config: s.Config, + Host: s.Host, + WinRMConfig: s.WinRMConfig, + }, } step, ok := typeMap[s.Config.Type] From 6077c796f52ee5d427b17fe6ba5a6cfd95a7353f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:10:50 -0700 Subject: [PATCH 175/208] communicator/winrm: fix failing test --- communicator/winrm/communicator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/communicator/winrm/communicator_test.go b/communicator/winrm/communicator_test.go index 73ac6d7b2..5c29a7403 100644 --- a/communicator/winrm/communicator_test.go +++ b/communicator/winrm/communicator_test.go @@ -87,7 +87,7 @@ func TestUpload(t *testing.T) { t.Fatalf("error creating communicator: %s", err) } - err = c.Upload("C:/Temp/terraform.cmd", bytes.NewReader([]byte("something"))) + err = c.Upload("C:/Temp/terraform.cmd", bytes.NewReader([]byte("something")), nil) if err != nil { t.Fatalf("error uploading file: %s", err) } From 114027095554d6ed2d43ad9441967a2f37b7e2f3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:13:54 -0700 Subject: [PATCH 176/208] update CHANGELOG --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e26c7818..79e703edd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,12 +13,14 @@ BACKWARDS INCOMPATIBILITIES: FEATURES: - * **New config function: `template_dir`**: The directory to the template - being built. This should be used for template-relative paths. [GH-54] + * **WinRM:** You can now connect via WinRM with almost every builder. + See the docs for more info. [GH-2239] * **Disable SSH:** Set `communicator` to "none" in any builder to disable SSH connections. Note that provisioners won't work if this is done. [GH-1591] * **SSH Agent Forwarding:** SSH Agent Forwarding will now be enabled to allow access to remote servers such as private git repos. [GH-1066] + * **New config function: `template_dir`**: The directory to the template + being built. This should be used for template-relative paths. [GH-54] IMPROVEMENTS: From fd4e0e9da47f461e820ba4f87b780a66fbebac5a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:35:45 -0700 Subject: [PATCH 177/208] builder/amazon: StepGetPassword --- builder/amazon/common/run_config.go | 6 + builder/amazon/common/step_get_password.go | 155 +++++++++++++++++++++ builder/amazon/ebs/builder.go | 4 + 3 files changed, 165 insertions(+) create mode 100644 builder/amazon/common/step_get_password.go diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index 6dec07b39..5589a5578 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "time" "github.com/mitchellh/packer/common/uuid" "github.com/mitchellh/packer/helper/communicator" @@ -27,6 +28,7 @@ type RunConfig struct { TemporaryKeyPairName string `mapstructure:"temporary_key_pair_name"` UserData string `mapstructure:"user_data"` UserDataFile string `mapstructure:"user_data_file"` + WindowsPasswordTimeout time.Duration `mapstructure:"windows_password_timeout"` VpcId string `mapstructure:"vpc_id"` // Communicator settings @@ -40,6 +42,10 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { "packer %s", uuid.TimeOrderedUUID()) } + if c.WindowsPasswordTimeout == 0 { + c.WindowsPasswordTimeout = 10 * time.Minute + } + // Validation errs := c.Comm.Prepare(ctx) if c.SourceAmi == "" { diff --git a/builder/amazon/common/step_get_password.go b/builder/amazon/common/step_get_password.go new file mode 100644 index 000000000..9d982b10b --- /dev/null +++ b/builder/amazon/common/step_get_password.go @@ -0,0 +1,155 @@ +package common + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "log" + "time" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/helper/communicator" + "github.com/mitchellh/packer/packer" +) + +// StepGetPassword reads the password from a Windows server and sets it +// on the WinRM config. +type StepGetPassword struct { + Comm *communicator.Config + Timeout time.Duration +} + +func (s *StepGetPassword) Run(state multistep.StateBag) multistep.StepAction { + ui := state.Get("ui").(packer.Ui) + image := state.Get("source_image").(*ec2.Image) + + // Skip if we're not Windows... + if *image.Platform != "windows" { + log.Printf("[INFO] Not Windows, skipping get password...") + return multistep.ActionContinue + } + + // If we already have a password, skip it + if s.Comm.WinRMPassword != "" { + ui.Say("Skipping waiting for password since WinRM password set...") + return multistep.ActionContinue + } + + // Get the password + var password string + var err error + cancel := make(chan struct{}) + waitDone := make(chan bool, 1) + go func() { + ui.Say("Waiting for auto-generated password for instance...") + password, err = s.waitForPassword(state, cancel) + waitDone <- true + }() + + timeout := time.After(s.Timeout) +WaitLoop: + for { + // Wait for either SSH to become available, a timeout to occur, + // or an interrupt to come through. + select { + case <-waitDone: + if err != nil { + ui.Error(fmt.Sprintf("Error waiting for password: %s", err)) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Message("Password retrieved!") + s.Comm.WinRMPassword = password + break WaitLoop + case <-timeout: + err := fmt.Errorf("Timeout waiting for password.") + state.Put("error", err) + ui.Error(err.Error()) + close(cancel) + return multistep.ActionHalt + case <-time.After(1 * time.Second): + if _, ok := state.GetOk(multistep.StateCancelled); ok { + // The step sequence was cancelled, so cancel waiting for password + // and just start the halting process. + close(cancel) + log.Println("[WARN] Interrupt detected, quitting waiting for password.") + return multistep.ActionHalt + } + } + } + return multistep.ActionContinue +} + +func (s *StepGetPassword) Cleanup(multistep.StateBag) {} + +func (s *StepGetPassword) waitForPassword(state multistep.StateBag, cancel <-chan struct{}) (string, error) { + ec2conn := state.Get("ec2").(*ec2.EC2) + instance := state.Get("instance").(*ec2.Instance) + privateKey := state.Get("privateKey").(string) + + for { + select { + case <-cancel: + log.Println("[INFO] Retrieve password wait cancelled. Exiting loop.") + return "", errors.New("Retrieve password wait cancelled") + case <-time.After(5 * time.Second): + } + + resp, err := ec2conn.GetPasswordData(&ec2.GetPasswordDataInput{ + InstanceID: instance.InstanceID, + }) + if err != nil { + err := fmt.Errorf("Error retrieving auto-generated instance password: %s", err) + return "", err + } + + if resp.PasswordData != nil && *resp.PasswordData != "" { + decryptedPassword, err := decryptPasswordDataWithPrivateKey( + *resp.PasswordData, []byte(privateKey)) + if err != nil { + err := fmt.Errorf("Error decrypting auto-generated instance password: %s", err) + return "", err + } + + return decryptedPassword, nil + } + } +} + +func decryptPasswordDataWithPrivateKey(passwordData string, pemBytes []byte) (string, error) { + encryptedPasswd, err := base64.StdEncoding.DecodeString(passwordData) + if err != nil { + return "", err + } + + block, _ := pem.Decode(pemBytes) + var asn1Bytes []byte + if _, ok := block.Headers["DEK-Info"]; ok { + return "", errors.New("encrypted private key isn't yet supported") + /* + asn1Bytes, err = x509.DecryptPEMBlock(block, password) + if err != nil { + return "", err + } + */ + } else { + asn1Bytes = block.Bytes + } + + key, err := x509.ParsePKCS1PrivateKey(asn1Bytes) + if err != nil { + return "", err + } + + out, err := rsa.DecryptPKCS1v15(nil, key, encryptedPasswd) + if err != nil { + return "", err + } + + return string(out), nil +} diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index cd3cd8f05..f61b258f4 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -113,6 +113,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe BlockDevices: b.config.BlockDevices, Tags: b.config.RunTags, }, + &awscommon.StepGetPassword{ + Comm: &b.config.RunConfig.Comm, + Timeout: b.config.WindowsPasswordTimeout, + }, &communicator.StepConnect{ Config: &b.config.RunConfig.Comm, Host: awscommon.SSHHost( From d23f254b7675dd1e49b5a393b27e38e0a6214c32 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:39:19 -0700 Subject: [PATCH 178/208] builder/amazon: don't get password if platform not set on image --- builder/amazon/common/step_get_password.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/common/step_get_password.go b/builder/amazon/common/step_get_password.go index 9d982b10b..a2ef04952 100644 --- a/builder/amazon/common/step_get_password.go +++ b/builder/amazon/common/step_get_password.go @@ -28,7 +28,7 @@ func (s *StepGetPassword) Run(state multistep.StateBag) multistep.StepAction { image := state.Get("source_image").(*ec2.Image) // Skip if we're not Windows... - if *image.Platform != "windows" { + if image.Platform == nil || *image.Platform != "windows" { log.Printf("[INFO] Not Windows, skipping get password...") return multistep.ActionContinue } From 022a115d190a4c6eef332bf3358af75991d26c3e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:42:10 -0700 Subject: [PATCH 179/208] builder/amazon: improve messaging --- builder/amazon/common/step_get_password.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/builder/amazon/common/step_get_password.go b/builder/amazon/common/step_get_password.go index a2ef04952..0fdd467eb 100644 --- a/builder/amazon/common/step_get_password.go +++ b/builder/amazon/common/step_get_password.go @@ -46,6 +46,9 @@ func (s *StepGetPassword) Run(state multistep.StateBag) multistep.StepAction { waitDone := make(chan bool, 1) go func() { ui.Say("Waiting for auto-generated password for instance...") + ui.Message( + "It is normal for this process to take up to 15 minutes,\n" + + "but it usually takes around 5. Please wait.") password, err = s.waitForPassword(state, cancel) waitDone <- true }() From 1d94e0f8e38ec69c31b7a28549f06c26687e7f23 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:50:02 -0700 Subject: [PATCH 180/208] template: abslute path for template path --- template/parse.go | 8 ++++++++ template/parse_test.go | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/template/parse.go b/template/parse.go index dbb29569d..4a7069dea 100644 --- a/template/parse.go +++ b/template/parse.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" "sort" "github.com/hashicorp/go-multierror" @@ -317,6 +318,13 @@ func ParseFile(path string) (*Template, error) { return nil, err } + if !filepath.IsAbs(path) { + path, err = filepath.Abs(path) + if err != nil { + return nil, err + } + } + tpl.Path = path return tpl, nil } diff --git a/template/parse_test.go b/template/parse_test.go index 9abca2f77..fa5477a4f 100644 --- a/template/parse_test.go +++ b/template/parse_test.go @@ -1,6 +1,7 @@ package template import ( + "path/filepath" "reflect" "strings" "testing" @@ -306,7 +307,7 @@ func TestParse(t *testing.T) { } for _, tc := range cases { - path := fixtureDir(tc.File) + path, _ := filepath.Abs(fixtureDir(tc.File)) tpl, err := ParseFile(fixtureDir(tc.File)) if (err != nil) != tc.Err { t.Fatalf("err: %s", err) From dc8c94890a8f926c621f062fc693c20ee93a21be Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 22:56:36 -0700 Subject: [PATCH 181/208] helper/config: copy template path properly --- helper/config/decode.go | 1 + 1 file changed, 1 insertion(+) diff --git a/helper/config/decode.go b/helper/config/decode.go index 20554da61..1088fd19b 100644 --- a/helper/config/decode.go +++ b/helper/config/decode.go @@ -42,6 +42,7 @@ func Decode(target interface{}, config *DecodeOpts, raws ...interface{}) error { if config.InterpolateContext == nil { config.InterpolateContext = ctx } else { + config.InterpolateContext.TemplatePath = ctx.TemplatePath config.InterpolateContext.UserVariables = ctx.UserVariables } ctx = config.InterpolateContext From 8f6ecfd9e31c8d57b228ff656da7144ce304dab3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 23:12:59 -0700 Subject: [PATCH 182/208] builder/amazon: various fixes (minor) to get things going --- builder/amazon/common/step_get_password.go | 8 ++++++-- .../amazon/common/step_run_source_instance.go | 8 ++++++++ builder/amazon/common/step_security_group.go | 16 ++++++++++------ builder/amazon/ebs/builder.go | 2 +- builder/amazon/instance/builder.go | 2 +- helper/communicator/config.go | 12 ++++++++++++ 6 files changed, 38 insertions(+), 10 deletions(-) diff --git a/builder/amazon/common/step_get_password.go b/builder/amazon/common/step_get_password.go index 0fdd467eb..37cfe3af6 100644 --- a/builder/amazon/common/step_get_password.go +++ b/builder/amazon/common/step_get_password.go @@ -48,7 +48,9 @@ func (s *StepGetPassword) Run(state multistep.StateBag) multistep.StepAction { ui.Say("Waiting for auto-generated password for instance...") ui.Message( "It is normal for this process to take up to 15 minutes,\n" + - "but it usually takes around 5. Please wait.") + "but it usually takes around 5. Please wait. After the\n" + + "password is read, it will printed out below. Since it should\n" + + "be a temporary password, this should be a minimal security risk.") password, err = s.waitForPassword(state, cancel) waitDone <- true }() @@ -66,7 +68,7 @@ WaitLoop: return multistep.ActionHalt } - ui.Message("Password retrieved!") + ui.Message(fmt.Sprintf(" \nPassword retrieved: %s", password)) s.Comm.WinRMPassword = password break WaitLoop case <-timeout: @@ -121,6 +123,8 @@ func (s *StepGetPassword) waitForPassword(state multistep.StateBag, cancel <-cha return decryptedPassword, nil } + + log.Printf("[DEBUG] Password is blank, will retry...") } } diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 92dafa564..021432e77 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -1,6 +1,7 @@ package common import ( + "encoding/base64" "fmt" "io/ioutil" "log" @@ -53,7 +54,14 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi return multistep.ActionHalt } + // Test if it is encoded already, and if not, encode it + if _, err := base64.StdEncoding.DecodeString(string(contents)); err != nil { + log.Printf("[DEBUG] base64 encoding user data...") + contents = []byte(base64.StdEncoding.EncodeToString(contents)) + } + userData = string(contents) + } ui.Say("Launching a source AWS instance...") diff --git a/builder/amazon/common/step_security_group.go b/builder/amazon/common/step_security_group.go index d870fd1c3..b65ebb408 100644 --- a/builder/amazon/common/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -9,12 +9,13 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/packer" ) type StepSecurityGroup struct { + CommConfig *communicator.Config SecurityGroupIds []string - SSHPort int VpcId string createdGroupId string @@ -30,8 +31,9 @@ func (s *StepSecurityGroup) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionContinue } - if s.SSHPort == 0 { - panic("SSHPort must be set to a non-zero value.") + port := s.CommConfig.Port() + if port == 0 { + panic("port must be set to a non-zero value.") } // Create the group @@ -57,15 +59,17 @@ func (s *StepSecurityGroup) Run(state multistep.StateBag) multistep.StepAction { req := &ec2.AuthorizeSecurityGroupIngressInput{ GroupID: groupResp.GroupID, IPProtocol: aws.String("tcp"), - FromPort: aws.Long(int64(s.SSHPort)), - ToPort: aws.Long(int64(s.SSHPort)), + FromPort: aws.Long(int64(port)), + ToPort: aws.Long(int64(port)), CIDRIP: aws.String("0.0.0.0/0"), } // We loop and retry this a few times because sometimes the security // group isn't available immediately because AWS resources are eventaully // consistent. - ui.Say("Authorizing SSH access on the temporary security group...") + ui.Say(fmt.Sprintf( + "Authorizing access to port %d the temporary security group...", + port)) for i := 0; i < 5; i++ { _, err = ec2conn.AuthorizeSecurityGroupIngress(req) if err == nil { diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index f61b258f4..162c06e28 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -94,7 +94,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, &awscommon.StepSecurityGroup{ SecurityGroupIds: b.config.SecurityGroupIds, - SSHPort: b.config.RunConfig.Comm.SSHPort, + CommConfig: &b.config.RunConfig.Comm, VpcId: b.config.VpcId, }, &awscommon.StepRunSourceInstance{ diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index d26cc63e3..ffe1c2da6 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -179,8 +179,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe PrivateKeyFile: b.config.RunConfig.Comm.SSHPrivateKey, }, &awscommon.StepSecurityGroup{ + CommConfig: &b.config.RunConfig.Comm, SecurityGroupIds: b.config.SecurityGroupIds, - SSHPort: b.config.RunConfig.Comm.SSHPort, VpcId: b.config.VpcId, }, &awscommon.StepRunSourceInstance{ diff --git a/helper/communicator/config.go b/helper/communicator/config.go index f0cb78df7..72dc69b7e 100644 --- a/helper/communicator/config.go +++ b/helper/communicator/config.go @@ -31,6 +31,18 @@ type Config struct { WinRMTimeout time.Duration `mapstructure:"winrm_timeout"` } +// Port returns the port that will be used for access based on config. +func (c *Config) Port() int { + switch c.Type { + case "ssh": + return c.SSHPort + case "winrm": + return c.WinRMPort + default: + return 0 + } +} + func (c *Config) Prepare(ctx *interpolate.Context) []error { if c.Type == "" { c.Type = "ssh" From e9d916a7bcd3a99ee3161efc1ac69f1c95255634 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Jun 2015 23:14:48 -0700 Subject: [PATCH 183/208] builder/amazon: don't print windows password --- builder/amazon/common/step_get_password.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/builder/amazon/common/step_get_password.go b/builder/amazon/common/step_get_password.go index 37cfe3af6..ab51f4394 100644 --- a/builder/amazon/common/step_get_password.go +++ b/builder/amazon/common/step_get_password.go @@ -48,9 +48,7 @@ func (s *StepGetPassword) Run(state multistep.StateBag) multistep.StepAction { ui.Say("Waiting for auto-generated password for instance...") ui.Message( "It is normal for this process to take up to 15 minutes,\n" + - "but it usually takes around 5. Please wait. After the\n" + - "password is read, it will printed out below. Since it should\n" + - "be a temporary password, this should be a minimal security risk.") + "but it usually takes around 5. Please wait.") password, err = s.waitForPassword(state, cancel) waitDone <- true }() @@ -68,7 +66,7 @@ WaitLoop: return multistep.ActionHalt } - ui.Message(fmt.Sprintf(" \nPassword retrieved: %s", password)) + ui.Message(fmt.Sprintf(" \nPassword retrieved!")) s.Comm.WinRMPassword = password break WaitLoop case <-timeout: From b51761ca36e72e8f772e2e179d50b42a8c362a9f Mon Sep 17 00:00:00 2001 From: Marcin Matlaszek Date: Sun, 14 Jun 2015 17:35:50 +0200 Subject: [PATCH 184/208] Fix network interface spec when requesting spot. --- builder/amazon/common/step_run_source_instance.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 92dafa564..71345ade7 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -174,11 +174,15 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi ImageID: &s.SourceAMI, InstanceType: &s.InstanceType, UserData: &userData, - SecurityGroupIDs: securityGroupIds, IAMInstanceProfile: &ec2.IAMInstanceProfileSpecification{Name: &s.IamInstanceProfile}, - SubnetID: &s.SubnetId, NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{ - &ec2.InstanceNetworkInterfaceSpecification{AssociatePublicIPAddress: &s.AssociatePublicIpAddress}, + &ec2.InstanceNetworkInterfaceSpecification{ + DeviceIndex: aws.Long(0), + AssociatePublicIPAddress: &s.AssociatePublicIpAddress, + SubnetID: &s.SubnetId, + Groups: securityGroupIds, + DeleteOnTermination: aws.Boolean(true), + }, }, Placement: &ec2.SpotPlacement{ AvailabilityZone: &availabilityZone, From 101e5986dcef11c688b5ecfb23511abc90a5d5bd Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Jun 2015 10:50:18 -0700 Subject: [PATCH 185/208] builder/amazon: enable windows for instance type too --- builder/amazon/instance/builder.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index ffe1c2da6..ec62394ee 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -198,6 +198,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe BlockDevices: b.config.BlockDevices, Tags: b.config.RunTags, }, + &awscommon.StepGetPassword{ + Comm: &b.config.RunConfig.Comm, + Timeout: b.config.WindowsPasswordTimeout, + }, &communicator.StepConnect{ Config: &b.config.RunConfig.Comm, Host: awscommon.SSHHost( From b2e9277d3b8cec19e6e935629d8888927bd4c95c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Jun 2015 10:51:34 -0700 Subject: [PATCH 186/208] website: update for Windows AWS instances --- website/source/docs/builders/amazon-ebs.html.markdown | 4 ++++ website/source/docs/builders/amazon-instance.html.markdown | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index 0ff9522df..088e6e974 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -168,6 +168,10 @@ each category, the available configuration keys are alphabetized. * `vpc_id` (string) - If launching into a VPC subnet, Packer needs the VPC ID in order to create a temporary security group within the VPC. +* `windows_password_timeout` (string) - The timeout for waiting for + a Windows password for Windows instances. Defaults to 20 minutes. + Example value: "10m" + ## Basic Example Here is a basic example. It is completely valid except for the access keys: diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown index ae5fbff27..249313160 100644 --- a/website/source/docs/builders/amazon-instance.html.markdown +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -209,6 +209,10 @@ each category, the available configuration keys are alphabetized. it is perfectly okay to create this directory as part of the provisioning process. +* `windows_password_timeout` (string) - The timeout for waiting for + a Windows password for Windows instances. Defaults to 20 minutes. + Example value: "10m" + ## Basic Example Here is a basic example. It is completely valid except for the access keys: From cf570a71dc9e279c4791521ca571336a147b8885 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Jun 2015 10:53:03 -0700 Subject: [PATCH 187/208] update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e703edd..4a11f23d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ FEATURES: * **WinRM:** You can now connect via WinRM with almost every builder. See the docs for more info. [GH-2239] + * **Windows AWS Support:** Windows AMIs can now be built without any + external plugins: Packer will start a Windows instance, get the + admin password, and can use WinRM (above) to connect through. [GH-2240] * **Disable SSH:** Set `communicator` to "none" in any builder to disable SSH connections. Note that provisioners won't work if this is done. [GH-1591] * **SSH Agent Forwarding:** SSH Agent Forwarding will now be enabled From 7fc69828c51f3a825b194ba49627883fc76671fb Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Jun 2015 21:47:53 -0700 Subject: [PATCH 188/208] builder/virtualbox: fix forwarding to work with WinRM --- builder/virtualbox/common/step_forward_ssh.go | 19 +++++++++++-------- builder/virtualbox/iso/builder.go | 2 +- builder/virtualbox/ovf/builder.go | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/builder/virtualbox/common/step_forward_ssh.go b/builder/virtualbox/common/step_forward_ssh.go index fe6004281..11bbcff9f 100644 --- a/builder/virtualbox/common/step_forward_ssh.go +++ b/builder/virtualbox/common/step_forward_ssh.go @@ -2,11 +2,13 @@ package common import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" "log" "math/rand" "net" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/helper/communicator" + "github.com/mitchellh/packer/packer" ) // This step adds a NAT port forwarding definition so that SSH is available @@ -19,7 +21,7 @@ import ( // // Produces: type StepForwardSSH struct { - GuestPort uint + CommConfig *communicator.Config HostPortMin uint HostPortMax uint SkipNatMapping bool @@ -30,20 +32,21 @@ func (s *StepForwardSSH) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) vmName := state.Get("vmName").(string) - sshHostPort := s.GuestPort + guestPort := s.CommConfig.Port() + sshHostPort := guestPort if !s.SkipNatMapping { log.Printf("Looking for available SSH port between %d and %d", s.HostPortMin, s.HostPortMax) - var offset uint = 0 + offset := 0 portRange := int(s.HostPortMax - s.HostPortMin) if portRange > 0 { // Have to check if > 0 to avoid a panic - offset = uint(rand.Intn(portRange)) + offset = rand.Intn(portRange) } for { - sshHostPort = offset + s.HostPortMin + sshHostPort = offset + int(s.HostPortMin) log.Printf("Trying port: %d", sshHostPort) l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", sshHostPort)) if err == nil { @@ -57,7 +60,7 @@ func (s *StepForwardSSH) Run(state multistep.StateBag) multistep.StepAction { command := []string{ "modifyvm", vmName, "--natpf1", - fmt.Sprintf("packerssh,tcp,127.0.0.1,%d,,%d", sshHostPort, s.GuestPort), + fmt.Sprintf("packerssh,tcp,127.0.0.1,%d,,%d", sshHostPort, guestPort), } if err := driver.VBoxManage(command...); err != nil { err := fmt.Errorf("Error creating port forwarding rule: %s", err) diff --git a/builder/virtualbox/iso/builder.go b/builder/virtualbox/iso/builder.go index f37c65c5a..dae107170 100644 --- a/builder/virtualbox/iso/builder.go +++ b/builder/virtualbox/iso/builder.go @@ -254,7 +254,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, new(vboxcommon.StepAttachFloppy), &vboxcommon.StepForwardSSH{ - GuestPort: uint(b.config.SSHConfig.Comm.SSHPort), + CommConfig: &b.config.SSHConfig.Comm, HostPortMin: b.config.SSHHostPortMin, HostPortMax: b.config.SSHHostPortMax, SkipNatMapping: b.config.SSHSkipNatMapping, diff --git a/builder/virtualbox/ovf/builder.go b/builder/virtualbox/ovf/builder.go index c35f2a50f..8b9932d54 100644 --- a/builder/virtualbox/ovf/builder.go +++ b/builder/virtualbox/ovf/builder.go @@ -83,7 +83,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, new(vboxcommon.StepAttachFloppy), &vboxcommon.StepForwardSSH{ - GuestPort: uint(b.config.SSHConfig.Comm.SSHPort), + CommConfig: &b.config.SSHConfig.Comm, HostPortMin: b.config.SSHHostPortMin, HostPortMax: b.config.SSHHostPortMax, SkipNatMapping: b.config.SSHSkipNatMapping, From cab2665119a0253448824e6b0ae4363bf6026cbf Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Jun 2015 22:09:38 -0700 Subject: [PATCH 189/208] builder/docker: support custom communicators --- builder/docker/builder.go | 11 +++- builder/docker/comm.go | 52 +++++++++++++++++++ builder/docker/config.go | 10 ++++ builder/docker/driver.go | 4 ++ builder/docker/driver_docker.go | 17 ++++++ builder/docker/driver_mock.go | 11 ++++ ...ep_provision.go => step_connect_docker.go} | 11 ++-- helper/communicator/step_connect.go | 8 +++ 8 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 builder/docker/comm.go rename builder/docker/{step_provision.go => step_connect_docker.go} (68%) diff --git a/builder/docker/builder.go b/builder/docker/builder.go index 96a79b02d..89880aacc 100644 --- a/builder/docker/builder.go +++ b/builder/docker/builder.go @@ -5,6 +5,7 @@ import ( "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/packer" ) @@ -42,7 +43,15 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &StepTempDir{}, &StepPull{}, &StepRun{}, - &StepProvision{}, + &communicator.StepConnect{ + Config: &b.config.Comm, + Host: commHost, + SSHConfig: sshConfig(&b.config.Comm), + CustomConnect: map[string]multistep.Step{ + "docker": &StepConnectDocker{}, + }, + }, + &common.StepProvision{}, } if b.config.Commit { diff --git a/builder/docker/comm.go b/builder/docker/comm.go new file mode 100644 index 000000000..a42d12525 --- /dev/null +++ b/builder/docker/comm.go @@ -0,0 +1,52 @@ +package docker + +import ( + "fmt" + "io/ioutil" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/communicator/ssh" + "github.com/mitchellh/packer/helper/communicator" + gossh "golang.org/x/crypto/ssh" +) + +func commHost(state multistep.StateBag) (string, error) { + containerId := state.Get("container_id").(string) + driver := state.Get("driver").(Driver) + return driver.IPAddress(containerId) +} + +func sshConfig(comm *communicator.Config) func(state multistep.StateBag) (*gossh.ClientConfig, error) { + return func(state multistep.StateBag) (*gossh.ClientConfig, error) { + if comm.SSHPrivateKey != "" { + // key based auth + bytes, err := ioutil.ReadFile(comm.SSHPrivateKey) + if err != nil { + return nil, fmt.Errorf("Error setting up SSH config: %s", err) + } + privateKey := string(bytes) + + signer, err := gossh.ParsePrivateKey([]byte(privateKey)) + if err != nil { + return nil, fmt.Errorf("Error setting up SSH config: %s", err) + } + + return &gossh.ClientConfig{ + User: comm.SSHUsername, + Auth: []gossh.AuthMethod{ + gossh.PublicKeys(signer), + }, + }, nil + } else { + // password based auth + return &gossh.ClientConfig{ + User: comm.SSHUsername, + Auth: []gossh.AuthMethod{ + gossh.Password(comm.SSHPassword), + gossh.KeyboardInteractive( + ssh.PasswordKeyboardInteractive(comm.SSHPassword)), + }, + }, nil + } + } +} diff --git a/builder/docker/config.go b/builder/docker/config.go index d5801c8ba..34fda4309 100644 --- a/builder/docker/config.go +++ b/builder/docker/config.go @@ -6,6 +6,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/communicator" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -13,6 +14,7 @@ import ( type Config struct { common.PackerConfig `mapstructure:",squash"` + Comm communicator.Config `mapstructure:",squash"` Commit bool ExportPath string `mapstructure:"export_path"` @@ -69,7 +71,15 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { c.Pull = true } + // Default to the normal Docker type + if c.Comm.Type == "" { + c.Comm.Type = "docker" + } + var errs *packer.MultiError + if es := c.Comm.Prepare(&c.ctx); len(es) > 0 { + errs = packer.MultiErrorAppend(errs, es...) + } if c.Image == "" { errs = packer.MultiErrorAppend(errs, fmt.Errorf("image must be specified")) diff --git a/builder/docker/driver.go b/builder/docker/driver.go index 7c9e6b868..d88c71022 100644 --- a/builder/docker/driver.go +++ b/builder/docker/driver.go @@ -22,6 +22,10 @@ type Driver interface { // Import imports a container from a tar file Import(path, repo string) (string, error) + // IPAddress returns the address of the container that can be used + // for external access. + IPAddress(id string) (string, error) + // Login. This will lock the driver from performing another Login // until Logout is called. Therefore, any users MUST call Logout. Login(repo, email, username, password string) error diff --git a/builder/docker/driver_docker.go b/builder/docker/driver_docker.go index ea895d362..0d406b1fa 100644 --- a/builder/docker/driver_docker.go +++ b/builder/docker/driver_docker.go @@ -116,6 +116,23 @@ func (d *DockerDriver) Import(path string, repo string) (string, error) { return strings.TrimSpace(stdout.String()), nil } +func (d *DockerDriver) IPAddress(id string) (string, error) { + var stderr, stdout bytes.Buffer + cmd := exec.Command( + "docker", + "inspect", + "--format", + "{{ .NetworkSettings.IPAddress }}", + id) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("Error: %s\n\nStderr: %s", err, stderr.String()) + } + + return strings.TrimSpace(stdout.String()), nil +} + func (d *DockerDriver) Login(repo, email, user, pass string) error { d.l.Lock() diff --git a/builder/docker/driver_mock.go b/builder/docker/driver_mock.go index 420856742..b0170f85f 100644 --- a/builder/docker/driver_mock.go +++ b/builder/docker/driver_mock.go @@ -23,6 +23,11 @@ type MockDriver struct { ImportId string ImportErr error + IPAddressCalled bool + IPAddressID string + IPAddressResult string + IPAddressErr error + LoginCalled bool LoginEmail string LoginUsername string @@ -104,6 +109,12 @@ func (d *MockDriver) Import(path, repo string) (string, error) { return d.ImportId, d.ImportErr } +func (d *MockDriver) IPAddress(id string) (string, error) { + d.IPAddressCalled = true + d.IPAddressID = id + return d.IPAddressResult, d.IPAddressErr +} + func (d *MockDriver) Login(r, e, u, p string) error { d.LoginCalled = true d.LoginRepo = r diff --git a/builder/docker/step_provision.go b/builder/docker/step_connect_docker.go similarity index 68% rename from builder/docker/step_provision.go rename to builder/docker/step_connect_docker.go index d9852ae2b..31f2ea2e4 100644 --- a/builder/docker/step_provision.go +++ b/builder/docker/step_connect_docker.go @@ -2,12 +2,11 @@ package docker import ( "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/common" ) -type StepProvision struct{} +type StepConnectDocker struct{} -func (s *StepProvision) Run(state multistep.StateBag) multistep.StepAction { +func (s *StepConnectDocker) Run(state multistep.StateBag) multistep.StepAction { containerId := state.Get("container_id").(string) driver := state.Get("driver").(Driver) tempDir := state.Get("temp_dir").(string) @@ -28,8 +27,8 @@ func (s *StepProvision) Run(state multistep.StateBag) multistep.StepAction { Version: version, } - prov := common.StepProvision{Comm: comm} - return prov.Run(state) + state.Put("communicator", comm) + return multistep.ActionContinue } -func (s *StepProvision) Cleanup(state multistep.StateBag) {} +func (s *StepConnectDocker) Cleanup(state multistep.StateBag) {} diff --git a/helper/communicator/step_connect.go b/helper/communicator/step_connect.go index ce77333e1..0c1522330 100644 --- a/helper/communicator/step_connect.go +++ b/helper/communicator/step_connect.go @@ -32,6 +32,11 @@ type StepConnect struct { // connecting via WinRM. WinRMConfig func(multistep.StateBag) (*WinRMConfig, error) + // CustomConnect can be set to have custom connectors for specific + // types. These take highest precedence so you can also override + // existing types. + CustomConnect map[string]multistep.Step + substep multistep.Step } @@ -50,6 +55,9 @@ func (s *StepConnect) Run(state multistep.StateBag) multistep.StepAction { WinRMConfig: s.WinRMConfig, }, } + for k, v := range s.CustomConnect { + typeMap[k] = v + } step, ok := typeMap[s.Config.Type] if !ok { From c3e79c62b9a460c0af0dcc2d9ea7ddadfe76c7c5 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Mon, 15 Jun 2015 10:01:32 -0500 Subject: [PATCH 190/208] document force_deregister --- website/source/docs/builders/amazon-chroot.html.markdown | 3 +++ website/source/docs/builders/amazon-ebs.html.markdown | 3 +++ website/source/docs/builders/amazon-instance.html.markdown | 3 +++ 3 files changed, 9 insertions(+) diff --git a/website/source/docs/builders/amazon-chroot.html.markdown b/website/source/docs/builders/amazon-chroot.html.markdown index e4e8cefa4..637153dab 100644 --- a/website/source/docs/builders/amazon-chroot.html.markdown +++ b/website/source/docs/builders/amazon-chroot.html.markdown @@ -124,6 +124,9 @@ each category, the available configuration keys are alphabetized. * `enhanced_networking` (boolean) - Enable enhanced networking (SriovNetSupport) on HVM-compatible AMIs. If true, add `ec2:ModifyInstanceAttribute` to your AWS IAM policy. +* `force_deregister` (boolean) – Force Packer to first deregister an existing +AMI if one with the same name already exists. Default `false`. + * `mount_path` (string) - The path where the volume will be mounted. This is where the chroot environment will be. This defaults to `packer-amazon-chroot-volumes/{{.Device}}`. This is a configuration diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index 0ff9522df..55747aa3c 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -96,6 +96,9 @@ each category, the available configuration keys are alphabetized. * `enhanced_networking` (boolean) - Enable enhanced networking (SriovNetSupport) on HVM-compatible AMIs. If true, add `ec2:ModifyInstanceAttribute` to your AWS IAM policy. +* `force_deregister` (boolean) – Force Packer to first deregister an existing +AMI if one with the same name already exists. Default `false`. + * `iam_instance_profile` (string) - The name of an [IAM instance profile](http://docs.aws.amazon.com/IAM/latest/UserGuide/instance-profiles.html) to launch the EC2 instance with. diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown index ae5fbff27..73b0caa5d 100644 --- a/website/source/docs/builders/amazon-instance.html.markdown +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -136,6 +136,9 @@ each category, the available configuration keys are alphabetized. * `enhanced_networking` (boolean) - Enable enhanced networking (SriovNetSupport) on HVM-compatible AMIs. If true, add `ec2:ModifyInstanceAttribute` to your AWS IAM policy. +* `force_deregister` (boolean) – Force Packer to first deregister an existing +AMI if one with the same name already exists. Default `false`. + * `iam_instance_profile` (string) - The name of an [IAM instance profile](http://docs.aws.amazon.com/IAM/latest/UserGuide/instance-profiles.html) to launch the EC2 instance with. From e814349511c9a94ad47d2d2c3c74e4b46beb3e19 Mon Sep 17 00:00:00 2001 From: Clint Date: Mon, 15 Jun 2015 10:04:11 -0500 Subject: [PATCH 191/208] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a11f23d5..c7a38adca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] + * builder/amazon: Add `force_deregister` option for automatic AMI + deregistration [GH-1873] * builder/digitalocean: Save SSH key to pwd if debug mode is on. [GH-1829] * builder/digitalocean: User data support [GH-2113] * builder/parallels: Support Parallels Desktop 11 [GH-2199] From 07f9eaf5b382b16f011227b8e63be2552c2598b9 Mon Sep 17 00:00:00 2001 From: Clint Date: Mon, 15 Jun 2015 10:04:46 -0500 Subject: [PATCH 192/208] Update CHANGELOG.md update GH issue for the `force_deregister` option --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7a38adca..d20273f0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] * builder/amazon: Add `force_deregister` option for automatic AMI - deregistration [GH-1873] + deregistration [GH-2221] * builder/digitalocean: Save SSH key to pwd if debug mode is on. [GH-1829] * builder/digitalocean: User data support [GH-2113] * builder/parallels: Support Parallels Desktop 11 [GH-2199] From fc62afaafb4573f0da5514878662d9df2d82dda6 Mon Sep 17 00:00:00 2001 From: Clint Date: Mon, 15 Jun 2015 10:07:42 -0500 Subject: [PATCH 193/208] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d20273f0e..a7988ea2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] * builder/amazon: Add `force_deregister` option for automatic AMI deregistration [GH-2221] + * builder/amazon: Now applies tags to EBS snapshots [GH-2212] * builder/digitalocean: Save SSH key to pwd if debug mode is on. [GH-1829] * builder/digitalocean: User data support [GH-2113] * builder/parallels: Support Parallels Desktop 11 [GH-2199] From d8cc24f86ec902a4e498566b852aebe2eaea8b2e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:34:35 -0700 Subject: [PATCH 194/208] builder/openstack: no more port --- builder/openstack/ssh.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go index 7ec1fcfdd..3e7350d11 100644 --- a/builder/openstack/ssh.go +++ b/builder/openstack/ssh.go @@ -22,7 +22,7 @@ func CommHost( // If we have a specific interface, try that if sshinterface != "" { - if addr := sshAddrFromPool(s, sshinterface, port); addr != "" { + if addr := sshAddrFromPool(s, sshinterface); addr != "" { return addr, nil } } @@ -38,7 +38,7 @@ func CommHost( } // Try to get it from the requested interface - if addr := sshAddrFromPool(s, sshinterface, port); addr != "" { + if addr := sshAddrFromPool(s, sshinterface); addr != "" { return addr, nil } @@ -75,7 +75,7 @@ func SSHConfig(username string) func(multistep.StateBag) (*ssh.ClientConfig, err } } -func sshAddrFromPool(s *servers.Server, desired string, port int) string { +func sshAddrFromPool(s *servers.Server, desired string) string { // Get all the addresses associated with this server. This // was taken directly from Terraform. for pool, networkAddresses := range s.Addresses { @@ -106,7 +106,7 @@ func sshAddrFromPool(s *servers.Server, desired string, port int) string { } } if addr != "" { - return fmt.Sprintf("%s:%d", addr, port) + return addr } } } From 62727c14a02c8a90e059539467c8c11f675cd636 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:35:22 -0700 Subject: [PATCH 195/208] update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7988ea2a..5374ab4f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] - * builder/amazon: Add `force_deregister` option for automatic AMI + * builder/amazon: Add `force_deregister` option for automatic AMI deregistration [GH-2221] * builder/amazon: Now applies tags to EBS snapshots [GH-2212] * builder/digitalocean: Save SSH key to pwd if debug mode is on. [GH-1829] @@ -80,6 +80,8 @@ BUG FIXES: * builder/docker: Fix crash that could occur at certain timed ctrl-c [GH-1838] * builder/docker: validate that `export_path` is not a directory [GH-2105] * builder/google: `ssh_timeout` is respected [GH-1781] + * builder/openstack: `ssh_interface` can be used to specify the interface + to retrieve the SSH IP from. [GH-2220] * builder/qemu: Add `disk_discard` option [GH-2120] * builder/virtualbox: Bind HTTP server to IPv4, which is more compatible with OS installers. [GH-1709] From 1691971e085779c62bd2563de0b1a7cf6f5057be Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:37:01 -0700 Subject: [PATCH 196/208] update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5374ab4f6..667352fe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,9 @@ IMPROVEMENTS: have prohibitive firewalls * builder/openstack: Flavor names can be used as well as refs * builder/openstack: Add `availability_zone` [GH-2016] + * builder/openstack: Machine will be stopped prior to imaging if the + cluster supports the `startstop` extension. [GH-2223] + * builder/openstack: Support for user data [GH-2224] * builder/virtualbox: Added option: `ssh_skip_nat_mapping` to skip the automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support @@ -53,6 +56,7 @@ IMPROVEMENTS: BUG FIXES: * core: Fix potential panic for post-processor plugin exits [GH-2098] + * core: `PACKER_CONFIG` may point to a non-existent file [GH-2226] * builder/amazon: Allow spaces in AMI names when using `clean_ami_name` [GH-2182] * builder/amazon: Remove deprecated ec2-upload-bundle paramger [GH-1931] * builder/amazon: Use IAM Profile to upload bundle if provided [GH-1985] From 3ea324d3f6814a3cb090fb9f80301301e662dcd9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:38:14 -0700 Subject: [PATCH 197/208] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 667352fe2..dc1a0815c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ IMPROVEMENTS: automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support * builder/vmware: Support for additional disks [GH-1382] + * command/fix: After fixing, the template is validated [GH-2228] * command/push: Add `-name` flag for specifying name from CLI [GH-2042] * command/push: Push configuration in templates supports variables [GH-1861] * post-processor/docker-save: Can be chained [GH-2179] @@ -109,6 +110,7 @@ BUG FIXES: * provisioner/salt-masterless: Add `--retcode-passthrough` to salt-call * provisioner/shell: chmod executable script to 0755, not 0777 [GH-1708] * provisioner/shell: inline commands failing will fail the provisioner [GH-2069] + * provisioner/shell: single quotes in env vars are escaped [GH-2229] ## 0.7.5 (December 9, 2014) From ac367c2b19b6023d1264e383a389d375136f6513 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:39:21 -0700 Subject: [PATCH 198/208] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc1a0815c..4fcf55149 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,8 @@ BUG FIXES: * builder/vmware: Case-insensitive match of MAC address to find IP [GH-1989] * builder/vmware: More robust IP parsing from ifconfig output [GH-1999] * builder/vmware: Nested output directories for ESXi work [GH-2174] + * command/fix: For the `virtualbox` to `virtualbox-iso` builder rename, + provisioner overrides are now also fixed [GH-2231] * command/validate: don't crash for invalid builds [GH-2139] * post-processor/atlas: Find common archive prefix for Windows [GH-1874] * post-processor/atlas: Fix index out of range panic [GH-1959] From a978bbf781ec999a93f94431f3d1d7f0a99babf0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:40:11 -0700 Subject: [PATCH 199/208] website: update docs for new functions --- .../source/docs/templates/configuration-templates.html.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/source/docs/templates/configuration-templates.html.markdown b/website/source/docs/templates/configuration-templates.html.markdown index 514bf7820..cef1385d3 100644 --- a/website/source/docs/templates/configuration-templates.html.markdown +++ b/website/source/docs/templates/configuration-templates.html.markdown @@ -55,6 +55,8 @@ While some configuration settings have local variables specific to only that configuration, a set of functions are available globally for use in _any string_ in Packer templates. These are listed below for reference. +* `build_name` - The name of the build being run. +* `build_type` - The type of the builder being used currently. * `isotime [FORMAT]` - UTC time, which can be [formatted](http://golang.org/pkg/time/#example_Time_Format). See more examples below. * `lower` - Lowercases the string. From 23c7e715e26ec22324763bc16ca23eeebbd72fa8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:41:15 -0700 Subject: [PATCH 200/208] update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fcf55149..fd5cb7159 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,8 @@ BUG FIXES: floppy disk. [GH-1879] * builder/virtualbox: Fixed regression where downloading ISO without a ".iso" extension didn't work. [GH-1839] + * builder/virtualbox: Output dir is verified at runtime, not template + validation time. [GH-2233] * builder/vmware: Add 100ms delay between keystrokes to avoid subtle timing issues in most cases. [GH-1663] * builder/vmware: Bind HTTP server to IPv4, which is more compatible with @@ -101,6 +103,8 @@ BUG FIXES: * builder/vmware: Case-insensitive match of MAC address to find IP [GH-1989] * builder/vmware: More robust IP parsing from ifconfig output [GH-1999] * builder/vmware: Nested output directories for ESXi work [GH-2174] + * builder/vmware: Output dir is verified at runtime, not template + validation time. [GH-2233] * command/fix: For the `virtualbox` to `virtualbox-iso` builder rename, provisioner overrides are now also fixed [GH-2231] * command/validate: don't crash for invalid builds [GH-2139] From 0787060c3b64789652be01c81efa6e2589c6591c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:41:51 -0700 Subject: [PATCH 201/208] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd5cb7159..3b1d6935e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ BUG FIXES: * builder/openstack: `ssh_interface` can be used to specify the interface to retrieve the SSH IP from. [GH-2220] * builder/qemu: Add `disk_discard` option [GH-2120] + * builder/qemu: Use proper SSH port, not hardcoded to 22. [GH-2236] * builder/virtualbox: Bind HTTP server to IPv4, which is more compatible with OS installers. [GH-1709] * builder/virtualbox: Remove the floppy controller in addition to the From 8b52fa5226ef0ab37c1e7cd6e7fae5a289e23202 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:43:14 -0700 Subject: [PATCH 202/208] update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b1d6935e..1e80fb53f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ FEATURES: connections. Note that provisioners won't work if this is done. [GH-1591] * **SSH Agent Forwarding:** SSH Agent Forwarding will now be enabled to allow access to remote servers such as private git repos. [GH-1066] + * **New config function: `build_name`**: The name of the currently running + build. [GH-2232] + * **New config function: `build_type`**: The type of the currently running + builder. This is useful for provisioners. [GH-2232] * **New config function: `template_dir`**: The directory to the template being built. This should be used for template-relative paths. [GH-54] From 661cd0243efa2e201662538abd368e15776e419f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:44:44 -0700 Subject: [PATCH 203/208] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e80fb53f..ed5b0d111 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] + * builder/*: Add `ssh_handshake_attempts` to configure the number of + handshake attempts done before failure [GH-2237] * builder/amazon: Add `force_deregister` option for automatic AMI deregistration [GH-2221] * builder/amazon: Now applies tags to EBS snapshots [GH-2212] From 70fdb36524d2213b7f451e12b42dc1b2cc049434 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 09:49:41 -0700 Subject: [PATCH 204/208] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed5b0d111..ac36c6fe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ FEATURES: connections. Note that provisioners won't work if this is done. [GH-1591] * **SSH Agent Forwarding:** SSH Agent Forwarding will now be enabled to allow access to remote servers such as private git repos. [GH-1066] + * **Docker builder supports SSH**: The Docker builder now supports containers + with SSH, just set `communicator` to "ssh" [GH-2244] * **New config function: `build_name`**: The name of the currently running build. [GH-2232] * **New config function: `build_type`**: The type of the currently running From d015d20a2440f8bd92ff881c064439a462b64447 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Mon, 15 Jun 2015 11:53:03 -0500 Subject: [PATCH 205/208] document block device mapping fields --- .../docs/builders/amazon-ebs.html.markdown | 29 +++++++++++++++---- .../builders/amazon-instance.html.markdown | 27 +++++++++++++---- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index af3ece59e..a7153a950 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -61,11 +61,26 @@ each category, the available configuration keys are alphabetized. ### Optional: * `ami_block_device_mappings` (array of block device mappings) - Add the block - device mappings to the AMI. The block device mappings allow for keys: - "device\_name" (string), "virtual\_name" (string), "snapshot\_id" (string), - "volume\_type" (string), "volume\_size" (integer), "delete\_on\_termination" - (boolean), "encrypted" (boolean), "no\_device" (boolean), and "iops" - (integer). + device mappings to the AMI. The block device mappings allow for keys: + + - `device_name` (string) – The device name exposed to the instance (for + example, "/dev/sdh" or "xvdh") + - `virtual_name` (string) – The virtual device name. See the documentation on + [Block Device Mapping][1] for more information + - `snapshot_id` (string) – The ID of the snapshot + - `volume_type` (string) – The volume type. gp2 for General Purpose (SSD) + volumes, io1 for Provisioned IOPS (SSD) volumes, and standard for Magnetic + volumes + - `volume_size` (integer) – The size of the volume, in GiB. Required if not + specifying a `snapshot_id` + - `delete_on_termination` (boolean) – Indicates whether the EBS volume is + deleted on instance termination + - `encrypted` (boolean) – Indicates whether to encrypt the volume or not + - `no_device` (boolean) – Suppresses the specified device included in the + block device mapping of the AMI + - `iops` (integer) – The number of I/O operations per second (IOPS) that the + volume supports. See the documentation on [IOPs][2] for more information + * `ami_description` (string) - The description to set for the resulting AMI(s). By default this description is empty. @@ -255,3 +270,7 @@ Here is an example using the optional AMI tags. This will add the tags } } ``` + + +[1]: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_BlockDeviceMapping.html +[2]: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_EbsBlockDevice.html diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown index 3ca82731b..2c7f6e4a7 100644 --- a/website/source/docs/builders/amazon-instance.html.markdown +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -81,12 +81,26 @@ each category, the available configuration keys are alphabetized. ### Optional: * `ami_block_device_mappings` (array of block device mappings) - Add the block - device mappings to the AMI. The block device mappings allow for keys: - "device\_name" (string), "virtual\_name" (string), "snapshot\_id" (string), - "volume\_type" (string), "volume\_size" (integer), "delete\_on\_termination" - (boolean), "encrypted" (boolean), "no\_device" (boolean), and "iops" (integer). - See [amazon-ebs](/docs/builders/amazon-ebs.html) for an example template. + device mappings to the AMI. The block device mappings allow for keys: + - `device_name` (string) – The device name exposed to the instance (for + example, "/dev/sdh" or "xvdh") + - `virtual_name` (string) – The virtual device name. See the documentation on + [Block Device Mapping][1] for more information + - `snapshot_id` (string) – The ID of the snapshot + - `volume_type` (string) – The volume type. gp2 for General Purpose (SSD) + volumes, io1 for Provisioned IOPS (SSD) volumes, and standard for Magnetic + volumes + - `volume_size` (integer) – The size of the volume, in GiB. Required if not + specifying a `snapshot_id` + - `delete_on_termination` (boolean) – Indicates whether the EBS volume is + deleted on instance termination + - `encrypted` (boolean) – Indicates whether to encrypt the volume or not + - `no_device` (boolean) – Suppresses the specified device included in the + block device mapping of the AMI + - `iops` (integer) – The number of I/O operations per second (IOPS) that the + volume supports. See the documentation on [IOPs][2] for more information + * `ami_description` (string) - The description to set for the resulting AMI(s). By default this description is empty. @@ -318,3 +332,6 @@ sudo -i -n ec2-upload-bundle \ The available template variables should be self-explanatory based on the parameters they're used to satisfy the `ec2-upload-bundle` command. + +[1]: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_BlockDeviceMapping.html +[2]: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_EbsBlockDevice.html From d22c4173d3457684fd2fd1bc290cafd62f43c09a Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Mon, 15 Jun 2015 11:53:21 -0500 Subject: [PATCH 206/208] fix crash when waiting for an instance that has failed --- builder/amazon/common/step_run_source_instance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index ec330ebc4..f88db5efc 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -245,7 +245,7 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi } latestInstance, err := WaitForState(&stateChange) if err != nil { - err := fmt.Errorf("Error waiting for instance (%s) to become ready: %s", *s.instance.InstanceID, err) + err := fmt.Errorf("Error waiting for instance (%s) to become ready: %s", instanceId, err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt From 523a3342b8155ead8a23aeed69344a8eb838298f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Jun 2015 10:15:08 -0700 Subject: [PATCH 207/208] builder/qemu: fix tests --- builder/qemu/step_run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/qemu/step_run.go b/builder/qemu/step_run.go index 816a3d3d3..21472f179 100644 --- a/builder/qemu/step_run.go +++ b/builder/qemu/step_run.go @@ -81,7 +81,7 @@ func getCommandArgs(bootDrive string, state multistep.StateBag) ([]string, error defaultArgs["-name"] = vmName defaultArgs["-machine"] = fmt.Sprintf("type=%s", config.MachineType) defaultArgs["-netdev"] = fmt.Sprintf( - "user,id=user.0,hostfwd=tcp::%v-:%d", sshHostPort, config.SSHPort) + "user,id=user.0,hostfwd=tcp::%v-:%d", sshHostPort, config.Comm.Port()) defaultArgs["-device"] = fmt.Sprintf("%s,netdev=user.0", config.NetDevice) defaultArgs["-drive"] = fmt.Sprintf("file=%s,if=%s,cache=%s,discard=%s", imgPath, config.DiskInterface, config.DiskCache, config.DiskDiscard) if !config.DiskImage { From d393bb5112241c75ecd8da878399ba62038d0d75 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Mon, 15 Jun 2015 10:30:45 -0700 Subject: [PATCH 208/208] make updatedeps will actually update now --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0574cbb5c..9abc16995 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ testrace: go test -race $(TEST) $(TESTARGS) updatedeps: - go get -d -v -p 2 ./... + go get -u -d -v -p 2 ./... vet: @go tool vet 2>/dev/null ; if [ $$? -eq 3 ]; then \