Merge branch 'master' of https://github.com/mitchellh/packer

builder/amazon/chroot/builder.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"runtime"
 
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mitchellh/multistep"
 	awscommon "github.com/mitchellh/packer/builder/amazon/common"
@@ -131,7 +132,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		return nil, err
 	}
 
-	ec2conn := ec2.New(config)
+	session := session.New(config)
+	ec2conn := ec2.New(session)
 
 	wrappedCommand := func(command string) (string, error) {
 		ctx := b.config.ctx

builder/amazon/common/artifact.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mitchellh/packer/packer"
 )
@@ -72,7 +73,8 @@ func (a *Artifact) Destroy() error {
 			Credentials: a.Conn.Config.Credentials,
 			Region:      aws.String(region),
 		}
-		regionConn := ec2.New(regionConfig)
+		sess := session.New(regionConfig)
+		regionConn := ec2.New(sess)
 
 		input := &ec2.DeregisterImageInput{
 			ImageId: &imageId,

builder/amazon/common/step_ami_region_copy.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 
 	"github.com/mitchellh/multistep"
@@ -87,7 +88,9 @@ func amiRegionCopy(state multistep.StateBag, config *AccessConfig, name string,
 	}
 	awsConfig.Region = aws.String(target)
 
-	regionconn := ec2.New(awsConfig)
+	sess := session.New(awsConfig)
+	regionconn := ec2.New(sess)
+
 	resp, err := regionconn.CopyImage(&ec2.CopyImageInput{
 		SourceRegion:  &source,
 		SourceImageId: &imageId,

builder/amazon/common/step_create_tags.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mitchellh/multistep"
 	"github.com/mitchellh/packer/packer"
@@ -33,11 +34,13 @@ func (s *StepCreateTags) Run(state multistep.StateBag) multistep.StepAction {
 
 			// Declare list of resources to tag
 			resourceIds := []*string{&ami}
-
-			regionconn := ec2.New(&aws.Config{
+			awsConfig := aws.Config{
 				Credentials: ec2conn.Config.Credentials,
 				Region:      aws.String(region),
-			})
+			}
+			session := session.New(&awsConfig)
+
+			regionconn := ec2.New(session)
 
 			// Retrieve image list for given AMI
 			imageResp, err := regionconn.DescribeImages(&ec2.DescribeImagesInput{

builder/amazon/common/step_modify_ami_attributes.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mitchellh/multistep"
 	"github.com/mitchellh/packer/packer"
@@ -88,10 +89,12 @@ func (s *StepModifyAMIAttributes) Run(state multistep.StateBag) multistep.StepAc
 
 	for region, ami := range amis {
 		ui.Say(fmt.Sprintf("Modifying attributes on AMI (%s)...", ami))
-		regionconn := ec2.New(&aws.Config{
+		awsConfig := aws.Config{
 			Credentials: ec2conn.Config.Credentials,
 			Region:      aws.String(region),
-		})
+		}
+		session := session.New(&awsConfig)
+		regionconn := ec2.New(session)
 		for name, input := range options {
 			ui.Message(fmt.Sprintf("Modifying: %s", name))
 			input.ImageId = &ami

builder/amazon/ebs/builder.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"log"
 
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mitchellh/multistep"
 	awscommon "github.com/mitchellh/packer/builder/amazon/common"
@@ -68,7 +69,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		return nil, err
 	}
 
-	ec2conn := ec2.New(config)
+	session := session.New(config)
+	ec2conn := ec2.New(session)
 
 	// Setup the state bag and initial state for the steps
 	state := new(multistep.BasicStateBag)

builder/amazon/ebs/builder_acc_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mitchellh/packer/builder/amazon/common"
 	builderT "github.com/mitchellh/packer/helper/builder/testing"
@@ -160,7 +161,8 @@ func testEC2Conn() (*ec2.EC2, error) {
 		return nil, err
 	}
 
-	return ec2.New(config), nil
+	session := session.New(config)
+	return ec2.New(session), nil
 }
 
 const testBuilderAccBasic = `

builder/amazon/instance/builder.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mitchellh/multistep"
 	awscommon "github.com/mitchellh/packer/builder/amazon/common"
@@ -159,7 +160,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		return nil, err
 	}
 
-	ec2conn := ec2.New(config)
+	session := session.New(config)
+	ec2conn := ec2.New(session)
 
 	// Setup the state bag and initial state for the steps
 	state := new(multistep.BasicStateBag)

builder/qemu/builder.go

@@ -88,6 +88,8 @@ type Config struct {
 	DiskSize        uint       `mapstructure:"disk_size"`
 	DiskCache       string     `mapstructure:"disk_cache"`
 	DiskDiscard     string     `mapstructure:"disk_discard"`
+	SkipCompaction  bool       `mapstructure:"skip_compaction"`
+	DiskCompression bool       `mapstructure:"disk_compression"`
 	FloppyFiles     []string   `mapstructure:"floppy_files"`
 	Format          string     `mapstructure:"format"`
 	Headless        bool       `mapstructure:"headless"`
@@ -242,6 +244,11 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 			errs, errors.New("invalid format, only 'qcow2' or 'raw' are allowed"))
 	}
 
+	if b.config.Format != "qcow2" {
+		b.config.SkipCompaction = true
+		b.config.DiskCompression = false
+	}
+
 	if _, ok := accels[b.config.Accelerator]; !ok {
 		errs = packer.MultiErrorAppend(
 			errs, errors.New("invalid accelerator, only 'kvm', 'tcg', 'xen', or 'none' are allowed"))
@@ -364,6 +371,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		},
 		new(common.StepProvision),
 		new(stepShutdown),
+		new(stepConvertDisk),
 	}
 
 	// Setup the state bag

builder/qemu/builder_test.go

@@ -132,6 +132,48 @@ func TestBuilderPrepare_BootWait(t *testing.T) {
 	}
 }
 
+func TestBuilderPrepare_DiskCompaction(t *testing.T) {
+	var b Builder
+	config := testConfig()
+
+	// Bad
+	config["skip_compaction"] = false
+	config["disk_compression"] = true
+	config["format"] = "img"
+	warns, err := b.Prepare(config)
+	if len(warns) > 0 {
+		t.Fatalf("bad: %#v", warns)
+	}
+	if err == nil {
+		t.Fatal("should have error")
+	}
+	if b.config.SkipCompaction != true {
+		t.Fatalf("SkipCompaction should be true")
+	}
+	if b.config.DiskCompression != false {
+		t.Fatalf("DiskCompression should be false")
+	}
+
+	// Good
+	config["skip_compaction"] = false
+	config["disk_compression"] = true
+	config["format"] = "qcow2"
+	b = Builder{}
+	warns, err = b.Prepare(config)
+	if len(warns) > 0 {
+		t.Fatalf("bad: %#v", warns)
+	}
+	if err != nil {
+		t.Fatalf("should not have error: %s", err)
+	}
+	if b.config.SkipCompaction != false {
+		t.Fatalf("SkipCompaction should be false")
+	}
+	if b.config.DiskCompression != true {
+		t.Fatalf("DiskCompression should be true")
+	}
+}
+
 func TestBuilderPrepare_DiskSize(t *testing.T) {
 	var b Builder
 	config := testConfig()

builder/qemu/step_convert_disk.go

@@ -0,0 +1,67 @@
+package qemu
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/mitchellh/multistep"
+	"github.com/mitchellh/packer/packer"
+
+	"os"
+)
+
+// This step converts the virtual disk that was used as the
+// hard drive for the virtual machine.
+type stepConvertDisk struct{}
+
+func (s *stepConvertDisk) Run(state multistep.StateBag) multistep.StepAction {
+	config := state.Get("config").(*Config)
+	driver := state.Get("driver").(Driver)
+	diskName := state.Get("disk_filename").(string)
+	ui := state.Get("ui").(packer.Ui)
+
+	if config.SkipCompaction && !config.DiskCompression {
+		return multistep.ActionContinue
+	}
+
+	name := diskName + ".convert"
+
+	sourcePath := filepath.Join(config.OutputDir, diskName)
+	targetPath := filepath.Join(config.OutputDir, name)
+
+	command := []string{
+		"convert",
+		"-q",
+	}
+
+	if config.DiskCompression {
+		command = append(command, "-c")
+	}
+
+	command = append(command, []string{
+		"-f", config.Format,
+		"-O", config.Format,
+		sourcePath,
+		targetPath,
+	}...,
+	)
+
+	ui.Say("Converting hard drive...")
+	if err := driver.QemuImg(command...); err != nil {
+		err := fmt.Errorf("Error converting hard drive: %s", err)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	if err := os.Rename(targetPath, sourcePath); err != nil {
+		err := fmt.Errorf("Error moving converted hard drive: %s", err)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	return multistep.ActionContinue
+}
+
+func (s *stepConvertDisk) Cleanup(state multistep.StateBag) {}

builder/vmware/iso/builder.go

@@ -40,7 +40,7 @@ type Config struct {
 	DiskSize            uint     `mapstructure:"disk_size"`
 	DiskTypeId          string   `mapstructure:"disk_type_id"`
 	FloppyFiles         []string `mapstructure:"floppy_files"`
-	Format              string   `mapstruture:"format"`
+	Format              string   `mapstructure:"format"`
 	GuestOSType         string   `mapstructure:"guest_os_type"`
 	Version             string   `mapstructure:"version"`
 	VMName              string   `mapstructure:"vm_name"`

config.go

@@ -10,7 +10,7 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/mitchellh/osext"
+	"github.com/kardianos/osext"
 	"github.com/mitchellh/packer/command"
 	"github.com/mitchellh/packer/packer"
 	"github.com/mitchellh/packer/packer/plugin"

provisioner/puppet-masterless/provisioner.go

@@ -22,6 +22,9 @@ type Config struct {
 	// The command used to execute Puppet.
 	ExecuteCommand string `mapstructure:"execute_command"`
 
+	// Additional arguments to pass when executing Puppet
+	ExtraArguments []string `mapstructure:"extra_arguments"`
+
 	// Additional facts to set when executing Puppet
 	Facter map[string]string
 
@@ -62,6 +65,7 @@ type ExecuteTemplate struct {
 	ManifestFile    string
 	ManifestDir     string
 	Sudo            bool
+	ExtraArguments  string
 }
 
 func (p *Provisioner) Prepare(raws ...interface{}) error {
@@ -86,6 +90,7 @@ func (p *Provisioner) Prepare(raws ...interface{}) error {
 			"{{if ne .HieraConfigPath \"\"}}--hiera_config='{{.HieraConfigPath}}' {{end}}" +
 			"{{if ne .ManifestDir \"\"}}--manifestdir='{{.ManifestDir}}' {{end}}" +
 			"--detailed-exitcodes " +
+			"{{if ne .ExtraArguments \"\"}}{{.ExtraArguments}} {{end}}" +
 			"{{.ManifestFile}}"
 	}
 
@@ -218,6 +223,7 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
 		ModulePath:      strings.Join(modulePaths, ":"),
 		Sudo:            !p.config.PreventSudo,
 		WorkingDir:      p.config.WorkingDir,
+		ExtraArguments:  strings.Join(p.config.ExtraArguments, " "),
 	}
 	command, err := interpolate.Render(p.config.ExecuteCommand, &p.config.ctx)
 	if err != nil {

provisioner/puppet-masterless/provisioner_test.go

@@ -1,10 +1,12 @@
 package puppetmasterless
 
 import (
-	"github.com/mitchellh/packer/packer"
 	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
+
+	"github.com/mitchellh/packer/packer"
 )
 
 func testConfig() map[string]interface{} {
@@ -177,3 +179,86 @@ func TestProvisionerPrepare_facterFacts(t *testing.T) {
 		t.Fatalf("err: Default facts are not set in the Puppet provisioner!")
 	}
 }
+
+func TestProvisionerPrepare_extraArguments(t *testing.T) {
+	config := testConfig()
+
+	// Test with missing parameter
+	delete(config, "extra_arguments")
+	p := new(Provisioner)
+	err := p.Prepare(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Test with malformed value
+	config["extra_arguments"] = "{{}}"
+	p = new(Provisioner)
+	err = p.Prepare(config)
+	if err == nil {
+		t.Fatal("should be an error")
+	}
+
+	// Test with valid values
+	config["extra_arguments"] = []string{
+		"arg",
+	}
+
+	p = new(Provisioner)
+	err = p.Prepare(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestProvisionerProvision_extraArguments(t *testing.T) {
+	config := testConfig()
+	ui := &packer.MachineReadableUi{
+		Writer: ioutil.Discard,
+	}
+	comm := new(packer.MockCommunicator)
+
+	extraArguments := []string{
+		"--some-arg=yup",
+		"--some-other-arg",
+	}
+	config["extra_arguments"] = extraArguments
+
+	// Test with valid values
+	p := new(Provisioner)
+	err := p.Prepare(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	err = p.Provision(ui, comm)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expectedArgs := strings.Join(extraArguments, " ")
+
+	if !strings.Contains(comm.StartCmd.Command, expectedArgs) {
+		t.Fatalf("Command %q doesn't contain the expected arguments %q", comm.StartCmd.Command, expectedArgs)
+	}
+
+	// Test with missing parameter
+	delete(config, "extra_arguments")
+
+	p = new(Provisioner)
+	err = p.Prepare(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	err = p.Provision(ui, comm)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Check the expected `extra_arguments` position for an empty value
+	splitCommand := strings.Split(comm.StartCmd.Command, " ")
+	if "" == splitCommand[len(splitCommand)-2] {
+		t.Fatalf("Command %q contains an extra-space which may cause arg parsing issues", comm.StartCmd.Command)
+	}
+}

scripts/build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # This script builds the application from source for multiple platforms.
 set -e

scripts/dist.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 # Get the parent directory of where this script is.

scripts/upload.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 # Get the parent directory of where this script is.

scripts/website_push.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Set the tmpdir
 if [ -z "$TMPDIR" ]; then

website/source/docs/builders/amazon-ebs.html.markdown

@@ -278,7 +278,7 @@ Here is an example using the optional AMI tags. This will add the tags
 
 -> **Note:** Packer uses pre-built AMIs as the source for building images.
 These source AMIs may include volumes that are not flagged to be destroyed on
-termiation of the instance building the new image. Packer will attempt to clean
+termination of the instance building the new image. Packer will attempt to clean
 up all residual volumes that are not designated by the user to remain after
 termination. If you need to preserve those source volumes, you can overwrite the
 termination setting by specifying `delete_on_termination=false` in the

website/source/docs/builders/amazon.html.markdown

@@ -101,6 +101,7 @@ Packer to work:
         "ec2:DeleteSecurityGroup",
         "ec2:AuthorizeSecurityGroupIngress",
         "ec2:CreateImage",
+        "ec2:CopyImage",
         "ec2:RunInstances",
         "ec2:TerminateInstances",
         "ec2:StopInstances",

website/source/docs/builders/qemu.html.markdown

@@ -136,6 +136,12 @@ builder.
 -   `disk_size` (integer) - The size, in megabytes, of the hard disk to create
     for the VM. By default, this is 40000 (about 40 GB).
 
+-   `skip_compaction` (boolean) - Packer compacts the QCOW2 image using `qemu-img convert`.
+    Set this option to `true` to disable compacting. Defaults to `false`.
+
+-   `disk_compression` (boolean) - Apply compression to the QCOW2 disk file
+    using `qemu-img convert`. Defaults to `false`.
+
 -   `floppy_files` (array of strings) - A list of files to place onto a floppy
     disk that is attached when the VM is booted. This is most useful for
     unattended Windows installs, which look for an `Autounattend.xml` file on

website/source/docs/other/debugging.html.markdown

@@ -58,10 +58,36 @@ any logging to be enabled.
 ### Debugging Packer in Powershell/Windows
 
 In Windows you can set the detailed logs environmental variable `PACKER_LOG` or
-the log variable `PACKER_LOG_PATH` using powershell environment variables. For example:
+the log variable `PACKER_LOG_PATH` using powershell environment variables. For
+example:
 
     $env:PACKER_LOG=1
     $env:PACKER_LOG_PATH="packerlog.txt" 
 
 If you find a bug with Packer, please include the detailed log by using a
 service such as [gist](http://gist.github.com).
+
+## Issues Installing Ubuntu Packages
+
+Issues may arise using and building Ubuntu AMIs where common packages that
+*should* be installed from Ubuntu's Main repository are not found during a
+provisioner step:
+
+    amazon-ebs: No candidate version found for build-essential
+    amazon-ebs: No candidate version found for build-essential
+
+This, obviously can cause problems where a build is unable to finish
+successfully as the proper packages cannot be provisioned correctly. The problem
+arises when cloud-init has not finished fully running on the source AMI by the
+time that packer starts any provisioning steps.
+
+Adding the following provisioner to the packer template, allows for the
+cloud-init process to fully finish before packer starts provisioning the source
+AMI.
+
+    {
+      "type": "shell",
+      "inline": [
+        "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done"
+      ]
+    }

website/source/docs/provisioners/puppet-masterless.html.markdown

@@ -59,6 +59,12 @@ Optional parameters:
     variables](/docs/templates/configuration-templates.html) available. See
     below for more information.
 
+-   `extra_arguments` (array of strings) - This is an array of additional options to
+    pass to the puppet command when executing puppet. This allows for
+    customization of the `execute_command` without having to completely replace
+    or include it's contents, making forward-compatible customizations much
+    easier.
+
 -   `facter` (object of key/value strings) - Additional
     [facts](http://puppetlabs.com/puppet/related-projects/facter) to make
     available when Puppet is running.

website/source/docs/provisioners/salt-masterless.html.markdown

@@ -38,6 +38,9 @@ Optional:
     has more detailed usage instructions. By default, no arguments are sent to
     the script.
 
+-   `disable_sudo` (boolean) - By default, the bootstrap install command is prefixed with `sudo`. When using a
+    Docker builder, you will likely want to pass `true` since `sudo` is often not pre-installed.
+
 -   `remote_pillar_roots` (string) - The path to your remote [pillar
     roots](http://docs.saltstack.com/ref/configuration/master.html#pillar-configuration).
     default: `/srv/pillar`.