Merge remote-tracking branch 'upstream/master' into packer-builder-profitbricks

CHANGELOG.md

@@ -27,6 +27,8 @@ IMPROVEMENTS:
   * builder/null: Can now be used with WinRM [GH-2525]
   * builder/parallels: Now pauses between `boot_command` entries when running
     with `-debug` [GH-3547]
+  * builder/parallels: Support future versions of Parallels by using the latest
+    driver [GH-3673]
   * builder/qemu: Added `vnc_bind_address` option [GH-3574]
   * builder/virtualbox: Now pauses between `boot_command` entries when running
     with `-debug` [GH-3542]

CONTRIBUTING.md

@@ -83,6 +83,20 @@ following steps in order to be able to compile and test Packer. These instructio
 7. If everything works well and the tests pass, run `go fmt` on your code
    before submitting a pull-request.
 
+### Opening an Pull Request
+
+When you are ready to open a pull-request, you will need to [fork packer](https://github.com/mitchellh/packer#fork-destination-box), push your changes to your fork, and then open a pull-request.
+
+For example, my github username is `cbednarski` so I would do the following:
+
+    git checkout -b f-my-feature
+    // develop a patch
+    git push https://github.com/cbednarski/packer f-my-feature
+
+From there, open your fork in your browser to open a new pull-request.
+
+**Note** Go infers package names from their filepaths. This means `go build` will break if you `git clone` your fork instead of using `go get` on the main packer project.
+
 ### Tips for Working on Packer
 
 #### Godeps
@@ -122,5 +136,5 @@ down to a specific resource to test, since testing all of them at once can
 sometimes take a very long time.
 
 Acceptance tests typically require other environment variables to be set for
-things such as access keys. The test itself should error early and tell you
+things such as API tokens and keys. Each test should error and tell you which
-what to set, so it is not documented here.
+credentials are missing, so those are not documented here.

builder/amazon/common/regions.go

@@ -4,6 +4,7 @@ func listEC2Regions() []string {
 	return []string{
 		"ap-northeast-1",
 		"ap-northeast-2",
+		"ap-south-1",
 		"ap-southeast-1",
 		"ap-southeast-2",
 		"cn-north-1",

builder/amazon/common/step_run_source_instance.go

@@ -45,7 +45,25 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi
 
 	securityGroupIds := make([]*string, len(tempSecurityGroupIds))
 	for i, sg := range tempSecurityGroupIds {
-		securityGroupIds[i] = aws.String(sg)
+		found := false
+		for i := 0; i < 5; i++ {
+			time.Sleep(time.Duration(i) * 5 * time.Second)
+			log.Printf("[DEBUG] Describing tempSecurityGroup to ensure it is available: %s", sg)
+			_, err := ec2conn.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
+				GroupIds: []*string{aws.String(sg)},
+			})
+			if err == nil {
+				log.Printf("[DEBUG] Found security group %s", sg)
+				found = true
+				break
+			}
+			log.Printf("[DEBUG] Error in querying security group %s", err)
+		}
+		if found {
+			securityGroupIds[i] = aws.String(sg)
+		} else {
+			state.Put("error", fmt.Errorf("Timeout waiting for security group %s to become available", sg))
+		}
 	}
 
 	userData := s.UserData

builder/amazon/common/step_source_ami_info.go

@@ -22,7 +22,7 @@ func (s *StepSourceAMIInfo) Run(state multistep.StateBag) multistep.StepAction {
 	ec2conn := state.Get("ec2").(*ec2.EC2)
 	ui := state.Get("ui").(packer.Ui)
 
-	ui.Say("Inspecting the source AMI...")
+	ui.Say(fmt.Sprintf("Inspecting the source AMI (%s)...", s.SourceAmi))
 	imageResp, err := ec2conn.DescribeImages(&ec2.DescribeImagesInput{ImageIds: []*string{&s.SourceAmi}})
 	if err != nil {
 		err := fmt.Errorf("Error querying AMI: %s", err)

builder/azure/arm/config_test.go

@@ -371,6 +371,7 @@ func TestConfigShouldRejectMalformedCaptureNamePrefix(t *testing.T) {
 		"storage_account":        "ignore",
 		"resource_group_name":    "ignore",
 		"subscription_id":        "ignore",
+		"communicator":           "none",
 		// Does not matter for this test case, just pick one.
 		"os_type": constants.Target_Linux,
 	}
@@ -421,6 +422,7 @@ func TestConfigShouldRejectMalformedCaptureContainerName(t *testing.T) {
 		"storage_account":     "ignore",
 		"resource_group_name": "ignore",
 		"subscription_id":     "ignore",
+		"communicator":        "none",
 		// Does not matter for this test case, just pick one.
 		"os_type": constants.Target_Linux,
 	}

builder/parallels/common/driver.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"os/exec"
 	"runtime"
+	"strconv"
 	"strings"
 )
 
@@ -125,6 +126,14 @@ func NewDriver() (Driver, error) {
 		supportedVersions = append(supportedVersions, v)
 	}
 
+	latestDriver := 11
+	version, _ := drivers[strconv.Itoa(latestDriver)].Version()
+	majVer, _ := strconv.Atoi(strings.SplitN(version, ".", 2)[0])
+	if majVer > latestDriver {
+		log.Printf("Your version of Parallels Desktop for Mac is %s, Packer will use driver for version %d.", version, latestDriver)
+		return drivers[strconv.Itoa(latestDriver)], nil
+	}
+
 	return nil, fmt.Errorf(
 		"Unable to initialize any driver. Supported Parallels Desktop versions: "+
 			"%s\n", strings.Join(supportedVersions, ", "))

common/download.go

@@ -105,21 +105,29 @@ func (d *DownloadClient) Get() (string, error) {
 		return d.config.TargetPath, nil
 	}
 
-	url, err := url.Parse(d.config.Url)
+	u, err := url.Parse(d.config.Url)
 	if err != nil {
 		return "", err
 	}
 
-	log.Printf("Parsed URL: %#v", url)
+	log.Printf("Parsed URL: %#v", u)
 
 	// Files when we don't copy the file are special cased.
 	var f *os.File
 	var finalPath string
 	sourcePath := ""
-	if url.Scheme == "file" && !d.config.CopyFile {
+	if u.Scheme == "file" && !d.config.CopyFile {
+		// This is special case for relative path in this case user specify
+		// file:../ and after parse destination goes to Opaque
+		if u.Path != "" {
+			// If url.Path is set just use this
+			finalPath = u.Path
+		} else if u.Opaque != "" {
+			// otherwise try url.Opaque
+			finalPath = u.Opaque
+		}
 		// This is a special case where we use a source file that already exists
 		// locally and we don't make a copy. Normally we would copy or download.
-		finalPath = url.Path
 		log.Printf("[DEBUG] Using local file: %s", finalPath)
 
 		// Remove forward slash on absolute Windows file URLs before processing
@@ -128,13 +136,16 @@ func (d *DownloadClient) Get() (string, error) {
 		}
 		// Keep track of the source so we can make sure not to delete this later
 		sourcePath = finalPath
+		if _, err = os.Stat(finalPath); err != nil {
+			return "", err
+		}
 	} else {
 		finalPath = d.config.TargetPath
 
 		var ok bool
-		d.downloader, ok = d.config.DownloaderMap[url.Scheme]
+		d.downloader, ok = d.config.DownloaderMap[u.Scheme]
 		if !ok {
-			return "", fmt.Errorf("No downloader for scheme: %s", url.Scheme)
+			return "", fmt.Errorf("No downloader for scheme: %s", u.Scheme)
 		}
 
 		// Otherwise, download using the downloader.
@@ -143,8 +154,8 @@ func (d *DownloadClient) Get() (string, error) {
 			return "", err
 		}
 
-		log.Printf("[DEBUG] Downloading: %s", url.String())
+		log.Printf("[DEBUG] Downloading: %s", u.String())
-		err = d.downloader.Download(f, url)
+		err = d.downloader.Download(f, u)
 		f.Close()
 		if err != nil {
 			return "", err

common/iso_config.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"net/url"
 	"os"
@@ -125,14 +124,11 @@ func (c *ISOConfig) Prepare(ctx *interpolate.Context) ([]string, []error) {
 }
 
 func (c *ISOConfig) parseCheckSumFile(rd *bufio.Reader) error {
+	errNotFound := fmt.Errorf("No checksum for %q found at: %s", filepath.Base(c.ISOUrls[0]), c.ISOChecksumURL)
 	for {
 		line, err := rd.ReadString('\n')
-		if err != nil {
+		if err != nil && line == "" {
-			if err == io.EOF {
+			break
-				return fmt.Errorf("No checksum for \"%s\" found at: %s", filepath.Base(c.ISOUrls[0]), c.ISOChecksumURL)
-			} else {
-				return fmt.Errorf("Error getting checksum from url: %s , %s", c.ISOChecksumURL, err.Error())
-			}
 		}
 		parts := strings.Fields(line)
 		if len(parts) < 2 {
@@ -142,7 +138,7 @@ func (c *ISOConfig) parseCheckSumFile(rd *bufio.Reader) error {
 			// BSD-style checksum
 			if parts[1] == fmt.Sprintf("(%s)", filepath.Base(c.ISOUrls[0])) {
 				c.ISOChecksum = parts[3]
-				break
+				return nil
 			}
 		} else {
 			// Standard checksum
@@ -152,9 +148,9 @@ func (c *ISOConfig) parseCheckSumFile(rd *bufio.Reader) error {
 			}
 			if parts[1] == filepath.Base(c.ISOUrls[0]) {
 				c.ISOChecksum = parts[0]
-				break
+				return nil
 			}
 		}
 	}
-	return nil
+	return errNotFound
 }

common/iso_config_test.go

@@ -26,6 +26,14 @@ bAr0 *the-OS.iso
 baZ0  other.iso
 `
 
+var cs_bsd_style_no_newline = `
+MD5 (other.iso) = bAr
+MD5 (the-OS.iso) = baZ`
+
+var cs_gnu_style_no_newline = `
+bAr0 *the-OS.iso
+baZ0  other.iso`
+
 func TestISOConfigPrepare_ISOChecksum(t *testing.T) {
 	i := testISOConfig()
 
@@ -100,6 +108,43 @@ func TestISOConfigPrepare_ISOChecksumURL(t *testing.T) {
 	if i.ISOChecksum != "bar0" {
 		t.Fatalf("should've found \"bar0\" got: %s", i.ISOChecksum)
 	}
+
+	// Test good - ISOChecksumURL BSD style no newline
+	i = testISOConfig()
+	i.ISOChecksum = ""
+	cs_file, _ = ioutil.TempFile("", "packer-test-")
+	ioutil.WriteFile(cs_file.Name(), []byte(cs_bsd_style_no_newline), 0666)
+	i.ISOChecksumURL = fmt.Sprintf("file://%s", cs_file.Name())
+	warns, err = i.Prepare(nil)
+	if len(warns) > 0 {
+		t.Fatalf("bad: %#v", warns)
+	}
+	if err != nil {
+		t.Fatalf("should not have error: %s", err)
+	}
+
+	if i.ISOChecksum != "baz" {
+		t.Fatalf("should've found \"baz\" got: %s", i.ISOChecksum)
+	}
+
+	// Test good - ISOChecksumURL GNU style no newline
+	i = testISOConfig()
+	i.ISOChecksum = ""
+	cs_file, _ = ioutil.TempFile("", "packer-test-")
+	ioutil.WriteFile(cs_file.Name(), []byte(cs_gnu_style_no_newline), 0666)
+	i.ISOChecksumURL = fmt.Sprintf("file://%s", cs_file.Name())
+	warns, err = i.Prepare(nil)
+	if len(warns) > 0 {
+		t.Fatalf("bad: %#v", warns)
+	}
+	if err != nil {
+		t.Fatalf("should not have error: %s", err)
+	}
+
+	if i.ISOChecksum != "bar0" {
+		t.Fatalf("should've found \"bar0\" got: %s", i.ISOChecksum)
+	}
+
 }
 
 func TestISOConfigPrepare_ISOChecksumType(t *testing.T) {

communicator/none/communicator.go

@@ -38,3 +38,7 @@ func (c *comm) UploadDir(dst string, src string, excl []string) error {
 func (c *comm) Download(path string, output io.Writer) error {
 	return errors.New("Download is not implemented when communicator = 'none'")
 }
+
+func (c *comm) DownloadDir(dst string, src string, excl []string) error {
+	return errors.New("DownloadDir is not implemented when communicator = 'none'")
+}

communicator/none/communicator_test.go

@@ -0,0 +1,16 @@
+package none
+
+import (
+	"testing"
+
+	"github.com/mitchellh/packer/packer"
+)
+
+func TestCommIsCommunicator(t *testing.T) {
+	var raw interface{}
+	raw = &comm{}
+	if _, ok := raw.(packer.Communicator); !ok {
+		t.Fatalf("comm must be a communicator")
+	}
+}
+

post-processor/checksum/post-processor.go

@@ -103,7 +103,9 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac
 			if _, err := os.Stat(checksumFile); err != nil {
 				newartifact.files = append(newartifact.files, checksumFile)
 			}
-
+			if err := os.MkdirAll(filepath.Dir(checksumFile), os.FileMode(0755)); err != nil {
+				return nil, false, fmt.Errorf("unable to create dir: %s", err.Error())
+			}
 			fw, err := os.OpenFile(checksumFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0644))
 			if err != nil {
 				return nil, false, fmt.Errorf("unable to create file %s: %s", checksumFile, err.Error())

provisioner/ansible/provisioner.go

@@ -20,6 +20,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"unicode"
 
 	"golang.org/x/crypto/ssh"
 
@@ -329,12 +330,21 @@ func (p *Provisioner) executeAnsible(ui packer.Ui, comm packer.Communicator, pri
 
 	wg := sync.WaitGroup{}
 	repeat := func(r io.ReadCloser) {
-		scanner := bufio.NewScanner(r)
+		reader := bufio.NewReader(r)
-		for scanner.Scan() {
+		for {
-			ui.Message(scanner.Text())
+			line, err := reader.ReadString('\n')
-		}
+			if line != "" {
-		if err := scanner.Err(); err != nil {
+				line = strings.TrimRightFunc(line, unicode.IsSpace)
-			ui.Error(err.Error())
+				ui.Message(line)
+			}
+			if err != nil {
+				if err == io.EOF {
+					break
+				} else {
+					ui.Error(err.Error())
+					break
+				}
+			}
 		}
 		wg.Done()
 	}

provisioner/ansible/provisioner_test.go

@@ -1,6 +1,7 @@
 package ansible
 
 import (
+	"bytes"
 	"crypto/rand"
 	"fmt"
 	"io"
@@ -243,3 +244,32 @@ func TestProvisionerPrepare_LocalPort(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 }
+
+func TestAnsibleGetVersion(t *testing.T) {
+	if os.Getenv("PACKER_ACC") == "" {
+		t.Skip("This test is only run with PACKER_ACC=1 and it requires Ansible to be installed")
+	}
+
+	var p Provisioner
+	p.config.Command = "ansible-playbook"
+	p.getVersion()
+}
+
+func TestAnsibleLongMessages(t *testing.T) {
+	if os.Getenv("PACKER_ACC") == "" {
+		t.Skip("This test is only run with PACKER_ACC=1 and it requires Ansible to be installed")
+	}
+
+	var p Provisioner
+	p.config.Command = "ansible-playbook"
+	p.config.PlaybookFile = "./test-fixtures/long-debug-message.yml"
+	p.Prepare()
+
+	comm := &packer.MockCommunicator{}
+	ui := &packer.BasicUi{
+		Reader: new(bytes.Buffer),
+		Writer: new(bytes.Buffer),
+	}
+
+	p.Provision(ui, comm)
+}

provisioner/ansible/test-fixtures/long-debug-message.yml

@@ -0,0 +1,8 @@
+- name: Stub for Packer testing long Ansible messages
+  hosts: localhost
+  connection: local
+
+  tasks:
+    - name: Very long Ansible output (>65535 chars) (Issue https://github.com/mitchellh/packer/issues/3268)
+      debug:
+        msg: "{{ lipsum(n=300, html=false) }}"

website/source/docs/provisioners/powershell.html.md

@@ -71,7 +71,7 @@ Optional parameters:
     Windows user.
 
 -   `remote_path` (string) - The path where the script will be uploaded to in
-    the machine. This defaults to "/tmp/script.sh". This value must be a
+    the machine. This defaults to "c:/Windows/Temp/script.ps1". This value must be a
     writable location and any parent directories must already exist.
 
 -   `start_retry_timeout` (string) - The amount of time to attempt to *start*

website/source/docs/provisioners/windows-shell.html.md

@@ -65,7 +65,7 @@ Optional parameters:
     to run, and `Vars`, which is the list of `environment_vars`, if configured.
 
 -   `remote_path` (string) - The path where the script will be uploaded to in
-    the machine. This defaults to "/tmp/script.sh". This value must be a
+    the machine. This defaults to "c:/Windows/Temp/script.bat". This value must be a
     writable location and any parent directories must already exist.
 
 -   `start_retry_timeout` (string) - The amount of time to attempt to *start*