Merge branch 'master' of https://github.com/hashicorp/packer into azure-sdk-update

builder/proxmox/step_type_boot_command.go

@@ -26,7 +26,7 @@ type stepTypeBootCommand struct {
 
 type bootCommandTemplateData struct {
 	HTTPIP   string
-	HTTPPort uint
+	HTTPPort int
 }
 
 type commandTyper interface {
@@ -66,7 +66,7 @@ func (s *stepTypeBootCommand) Run(ctx context.Context, state multistep.StateBag)
 	common.SetHTTPIP(httpIP)
 	s.Ctx.Data = &bootCommandTemplateData{
 		HTTPIP:   httpIP,
-		HTTPPort: state.Get("http_port").(uint),
+		HTTPPort: state.Get("http_port").(int),
 	}
 
 	ui.Say("Typing the boot command")

builder/proxmox/step_type_boot_command_test.go

@@ -106,7 +106,7 @@ func TestTypeBootCommand(t *testing.T) {
 			state := new(multistep.BasicStateBag)
 			state.Put("ui", packer.TestUi(t))
 			state.Put("config", c.builderConfig)
-			state.Put("http_port", uint(0))
+			state.Put("http_port", int(0))
 			state.Put("vmRef", proxmox.NewVmRef(1))
 			state.Put("proxmoxClient", typer)
 

common/retry/retry.go

@@ -25,6 +25,17 @@ type Config struct {
 	ShouldRetry func(error) bool
 }
 
+type RetryExhaustedError struct {
+	Err error
+}
+
+func (err *RetryExhaustedError) Error() string {
+	if err == nil || err.Err == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("retry count exhausted. Last err: %s", err.Err)
+}
+
 // Run fn until context is cancelled up until StartTimeout time has passed.
 func (cfg Config) Run(ctx context.Context, fn func(context.Context) error) error {
 	retryDelay := func() time.Duration { return 2 * time.Second }
@@ -40,10 +51,10 @@ func (cfg Config) Run(ctx context.Context, fn func(context.Context) error) error
 		startTimeout = time.After(cfg.StartTimeout)
 	}
 
+	var err error
 	for try := 0; ; try++ {
-		var err error
 		if cfg.Tries != 0 && try == cfg.Tries {
-			return err
+			return &RetryExhaustedError{err}
 		}
 		if err = fn(ctx); err == nil {
 			return nil

common/retry/retry_test.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"testing"
 	"time"
+
+	"github.com/google/go-cmp/cmp"
 )
 
 func success(context.Context) error { return nil }
@@ -18,12 +20,23 @@ var failErr = errors.New("woops !")
 
 func fail(context.Context) error { return failErr }
 
+type failOnce bool
+
+func (ran *failOnce) Run(context.Context) error {
+	if !*ran {
+		*ran = true
+		return failErr
+	}
+	return nil
+}
+
 func TestConfig_Run(t *testing.T) {
 	cancelledCtx, cancel := context.WithCancel(context.Background())
 	cancel()
 	type fields struct {
 		StartTimeout time.Duration
 		RetryDelay   func() time.Duration
+		Tries        int
 	}
 	type args struct {
 		ctx context.Context
@@ -36,26 +49,37 @@ func TestConfig_Run(t *testing.T) {
 		wantErr error
 	}{
 		{"success",
-			fields{StartTimeout: time.Second, RetryDelay: nil},
+			fields{StartTimeout: time.Second},
 			args{context.Background(), success},
 			nil},
 		{"context cancelled",
-			fields{StartTimeout: time.Second, RetryDelay: nil},
+			fields{StartTimeout: time.Second},
 			args{cancelledCtx, wait},
 			context.Canceled},
 		{"timeout",
 			fields{StartTimeout: 20 * time.Millisecond, RetryDelay: func() time.Duration { return 10 * time.Millisecond }},
 			args{cancelledCtx, fail},
 			failErr},
+		{"success after one failure",
+			fields{Tries: 2, RetryDelay: func() time.Duration { return 0 }},
+			args{context.Background(), new(failOnce).Run},
+			nil},
+		{"fail after one failure",
+			fields{Tries: 1, RetryDelay: func() time.Duration { return 0 }},
+			args{context.Background(), new(failOnce).Run},
+			&RetryExhaustedError{failErr},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := Config{
 				StartTimeout: tt.fields.StartTimeout,
 				RetryDelay:   tt.fields.RetryDelay,
+				Tries:        tt.fields.Tries,
 			}
-			if err := cfg.Run(tt.args.ctx, tt.args.fn); err != tt.wantErr {
+			err := cfg.Run(tt.args.ctx, tt.args.fn)
-				t.Fatalf("Config.Run() error = %v, wantErr %v", err, tt.wantErr)
+			if diff := cmp.Diff(err, tt.wantErr, DeepAllowUnexported(RetryExhaustedError{}, errors.New(""))); diff != "" {
+				t.Fatalf("Config.Run() unexpected error: %s", diff)
 			}
 		})
 	}

common/retry/utils_test.go

@@ -0,0 +1,48 @@
+package retry
+
+import (
+	"reflect"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func DeepAllowUnexported(vs ...interface{}) cmp.Option {
+	m := make(map[reflect.Type]struct{})
+	for _, v := range vs {
+		structTypes(reflect.ValueOf(v), m)
+	}
+	var typs []interface{}
+	for t := range m {
+		typs = append(typs, reflect.New(t).Elem().Interface())
+	}
+	return cmp.AllowUnexported(typs...)
+}
+
+func structTypes(v reflect.Value, m map[reflect.Type]struct{}) {
+	if !v.IsValid() {
+		return
+	}
+	switch v.Kind() {
+	case reflect.Ptr:
+		if !v.IsNil() {
+			structTypes(v.Elem(), m)
+		}
+	case reflect.Interface:
+		if !v.IsNil() {
+			structTypes(v.Elem(), m)
+		}
+	case reflect.Slice, reflect.Array:
+		for i := 0; i < v.Len(); i++ {
+			structTypes(v.Index(i), m)
+		}
+	case reflect.Map:
+		for _, k := range v.MapKeys() {
+			structTypes(v.MapIndex(k), m)
+		}
+	case reflect.Struct:
+		m[v.Type()] = struct{}{}
+		for i := 0; i < v.NumField(); i++ {
+			structTypes(v.Field(i), m)
+		}
+	}
+}

website/source/docs/builders/amazon-ebs.html.md.erb

@@ -726,3 +726,5 @@ be easily added to the provisioner section.
 
 }
 ```
+
+<%= partial "partials/builders/aws-ssh-differentiation-table" %>

website/source/docs/builders/amazon-ebssurrogate.html.md.erb

@@ -626,3 +626,6 @@ These source AMIs may include volumes that are not flagged to be destroyed on
 termination of the instance building the new image. In addition to those
 volumes created by this builder, any volumes inn the source AMI which are not
 marked for deletion on termination will remain in your account.
+
+
+<%= partial "partials/builders/aws-ssh-differentiation-table" %>

website/source/docs/builders/amazon-ebsvolume.html.md.erb

@@ -564,3 +564,6 @@ These source AMIs may include volumes that are not flagged to be destroyed on
 termination of the instance building the new image. In addition to those
 volumes created by this builder, any volumes inn the source AMI which are not
 marked for deletion on termination will remain in your account.
+
+
+<%= partial "partials/builders/aws-ssh-differentiation-table" %>

website/source/docs/builders/amazon-instance.html.md.erb

@@ -705,3 +705,6 @@ this:
 ```
 
 You may wish to constrain the resource to a specific bucket.
+
+
+<%= partial "partials/builders/aws-ssh-differentiation-table" %>

website/source/docs/builders/googlecompute.html.md

@@ -166,6 +166,9 @@ setting a valid `account_file` and `project_id`:
 }
 ```
 
+-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code:
+https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/
+
 This build can take up to 15 min.
 
 ### Nested Hypervisor Example

website/source/docs/builders/hyperv-iso.html.md.erb

@@ -876,6 +876,9 @@ Finish proxy after sysprep -->
 </unattend>
 ```
 
+-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code:
+https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/
+
 ## Example For Ubuntu Vivid Generation 2
 
 If you are running Windows under virtualization, you may need to create a

website/source/docs/builders/hyperv-vmcx.html.md.erb

@@ -893,6 +893,9 @@ Finish proxy after sysprep -->
 </unattend>
 ```
 
+-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code:
+https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/
+
 ## Example For Ubuntu Vivid Generation 2
 
 If you are running Windows under virtualization, you may need to create a

website/source/docs/builders/ncloud.html.md

@@ -90,6 +90,9 @@ Here is a basic example for windows server.
       ]
     }
 
+-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code:
+https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/
+
 Here is a basic example for linux server.
 
     {

website/source/docs/builders/proxmox.html.md

@@ -71,8 +71,9 @@ builder.
 -   `sockets` (int) - How many CPU sockets to give the virtual machine.
     Defaults to `1`
 
--   `os` (string) - The operating system. Can be `linux`, `windows`, `solaris`
+-   `os` (string) - The operating system. Can be `wxp`, `w2k`, `w2k3`, `w2k8`,
-    or `other`. Defaults to `other`.
+    `wvista`, `win7`, `win8`, `win10`, `l24` (Linux 2.4), `l26` (Linux 2.6+),
+    `solaris` or `other`. Defaults to `other`.
 
 -   `network_adapters` (array of objects) - Network adapters attached to the
     virtual machine. Example:

website/source/docs/builders/vmware-iso.html.md.erb

@@ -496,69 +496,7 @@ variables isn't required, however.
 -   `Version` - The Hardware version VMWare will execute this vm under. Also
     known as the `virtualhw.version`.
 
-## Building on a Remote vSphere Hypervisor
+<%= partial "partials/builders/building_on_remote_vsphere_hypervisor" %>
-
-In addition to using the desktop products of VMware locally to build virtual
-machines, Packer can use a remote VMware Hypervisor to build the virtual
-machine.
-
--&gt; **Note:** Packer supports ESXi 5.1 and above.
-
-Before using a remote vSphere Hypervisor, you need to enable GuestIPHack by
-running the following command:
-
-``` text
-esxcli system settings advanced set -o /Net/GuestIPHack -i 1
-```
-
-When using a remote VMware Hypervisor, the builder still downloads the ISO and
-various files locally, and uploads these to the remote machine. Packer currently
-uses SSH to communicate to the ESXi machine rather than the vSphere API. At some
-point, the vSphere API may be used.
-
-Packer also requires VNC to issue boot commands during a build, which may be
-disabled on some remote VMware Hypervisors. Please consult the appropriate
-documentation on how to update VMware Hypervisor's firewall to allow these
-connections. VNC can be disabled by not setting a `boot_command` and setting
-`disable_vnc` to `true`.
-
-To use a remote VMware vSphere Hypervisor to build your virtual machine, fill in
-the required `remote_*` configurations:
-
--   `remote_type` - This must be set to "esx5".
-
--   `remote_host` - The host of the remote machine.
-
-Additionally, there are some optional configurations that you'll likely have to
-modify as well:
-
--   `remote_port` - The SSH port of the remote machine
-
--   `remote_datastore` - The path to the datastore where the VM will be stored
-    on the ESXi machine.
-
--   `remote_cache_datastore` - The path to the datastore where supporting files
-    will be stored during the build on the remote machine.
-
--   `remote_cache_directory` - The path where the ISO and/or floppy files will
-    be stored during the build on the remote machine. The path is relative to
-    the `remote_cache_datastore` on the remote machine.
-
--   `remote_username` - The SSH username used to access the remote machine.
-
--   `remote_password` - The SSH password for access to the remote machine.
-
--   `remote_private_key_file` - The SSH key for access to the remote machine.
-
--   `format` (string) - Either "ovf", "ova" or "vmx", this specifies the output
-    format of the exported virtual machine. This defaults to "ovf".
-    Before using this option, you need to install `ovftool`. This option
-    currently only works when option remote_type is set to "esx5".
-    Since ovftool is only capable of password based authentication
-    `remote_password` must be set when exporting the VM.
-
--   `vnc_disable_password` - This must be set to "true" when using VNC with
-    ESXi 6.5 or 6.7.
 
 ### VNC port discovery
 

website/source/docs/builders/vmware-vmx.html.md.erb

@@ -360,3 +360,5 @@ Ubuntu 12.04 installer:
 
 For more examples of various boot commands, see the sample projects from our
 [community templates page](/community-tools.html#templates).
+
+<%= partial "partials/builders/building_on_remote_vsphere_hypervisor" %>

website/source/docs/post-processors/vagrant.html.md

@@ -150,3 +150,9 @@ The following Docker input artifacts are supported:
 
 The `libvirt` provider supports QEMU artifacts built using any these
 accelerators: none, kvm, tcg, or hvf.
+
+### VMWare
+
+If you are using the Vagrant post-processor with the `vmware-esxi` builder, you
+must export the builder artifact locally; the Vagrant post-processor will
+not work on remote artifacts.

website/source/docs/provisioners/ansible.html.md.erb

@@ -322,6 +322,9 @@ Platform:
 }
 ```
 
+-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code:
+https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/
+
 ### Post i/o timeout errors
 
 If you see

website/source/guides/isotime-template-function.html.md

@@ -0,0 +1,89 @@
+---
+layout: guides
+sidebar_current: isotime-template-function
+page_title: Using the isotime template function - Guides
+description: |-
+  It can be a bit confusing to figure out how to format your isotime using the
+  golang reference date string. Here is a small guide and some examples.
+---
+
+# Using the Isotime template function with a format string
+
+The way you format isotime in golang is a bit nontraditional compared to how
+you may be used to formatting datetime strings.
+
+Full docs and examples for the golang time formatting function can be found
+[here](https://golang.org/pkg/time/#example_Time_Format)
+
+However, the formatting basics are worth describing here. From the [golang docs](https://golang.org/pkg/time/#pkg-constants):
+
+
+>These are predefined layouts for use in Time.Format and time.Parse. The
+>reference time used in the layouts is the specific time:
+>
+>Mon Jan 2 15:04:05 MST 2006
+>
+>which is Unix time 1136239445. Since MST is GMT-0700, the reference time
+>can be thought of as
+>
+>01/02 03:04:05PM '06 -0700
+>
+> To define your own format, write down what the reference time would look like
+> formatted your way; see the values of constants like ANSIC, StampMicro or
+> Kitchen for examples. The model is to demonstrate what the reference time
+> looks like so that the Format and Parse methods can apply the same
+> transformation to a general time value.
+
+
+So what does that look like in a Packer template function?
+
+``` json
+{
+	"variables":
+	{
+		"myvar": "packer-{{isotime \"2006-01-02 03:04:05\"}}"
+	},
+	"builders": [
+		{
+			"type": "null",
+			"communicator": "none"
+		}
+	],
+	"provisioners": [
+		{
+			"type": "shell-local",
+			"inline": ["echo {{ user `myvar`}}"]
+		}
+	]
+}
+```
+
+You can switch out the variables section above with the following examples to
+get different timestamps:
+
+Date only, not time:
+
+```json
+	"variables":
+	{
+		"myvar": "packer-{{isotime \"2006-01-02\"}}"
+	},
+```
+
+A timestamp down to the millisecond:
+
+```json
+	"variables":
+	{
+		"myvar": "packer-{{isotime \"Jan-_2-15:04:05.000\"}}"
+	},
+```
+
+Or just the time as it would appear on a digital clock:
+
+```json
+	"variables":
+	{
+		"myvar": "packer-{{isotime \"3:04PM\"}}"
+	},
+```

website/source/guides/use-packer-with-comment.html.md

@@ -0,0 +1,77 @@
+---
+layout: guides
+sidebar_current: use-packer-with-comment
+page_title: Use jq and Packer to comment your templates - Guides
+description: |-
+  You can add detailed comments beyond the root-level underscore-prefixed field
+  supported by Packer, and remove them using jq.
+---
+
+#How to use jq to strip unsupported comments from a Packer template
+
+One of the biggest complaints we get about packer is that json doesn't use comments. We're in the process of moving to HCL2, the same config language used by Terraform, which does allow comments. But in the meantime, you can add detailed comments beyond the root-level underscore-prefixed field supported by Packer, and remove them using jq.
+
+Let's say we have a file named commented_template.json
+
+``` json
+{
+  	"_comment": [
+	    "this is",
+	    "a multi-line",
+	    "comment"
+  	],
+    "builders": [
+        {
+            "_comment": "this is a comment inside a builder",
+            "type": "null",
+            "communicator": "none"
+        }
+    ],
+    "_comment": "this is a root level comment",
+    "provisioners": [
+        {
+          "_comment": "this is a different comment",
+          "type": "shell",
+          "_comment": "this is yet another comment",
+          "inline": ["echo hellooooo"]
+        }
+    ]
+}
+```
+
+```sh
+jq 'walk(if type == "object" then del(._comment) else . end)' commented_template.json > uncommented_template.json
+```
+
+will produce a new file containing:
+
+```json
+{
+  "builders": [
+    {
+      "type": "null",
+      "communicator": "none"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "inline": [
+        "echo hellooooo"
+      ]
+    }
+  ]
+}
+```
+
+Once you've got your uncommented file, you can call `packer build` on it like
+you normally would.
+
+## The walk function
+If your install of jq does not have the walk function and you get an error like
+
+```
+jq: error: walk/1 is not defined at <top-level>,
+```
+
+You can create a file `~/.jq` and add the [walk function](https://github.com/stedolan/jq/blob/ad9fc9f559e78a764aac20f669f23cdd020cd943/src/builtin.jq#L255-L262) to it by hand.

website/source/intro/getting-started/build-image.html.md

@@ -404,6 +404,9 @@ Start-Service -Name WinRM
 </powershell>
 ```
 
+-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code:
+https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/
+
 Save the above code in a file named `bootstrap_win.txt`.
 
 -> **A quick aside/warning:**<br />

website/source/layouts/guides.erb

@@ -1,9 +1,15 @@
 <% wrap_layout :inner do %>
   <% content_for :sidebar do %>
     <ul class="nav docs-sidenav">
+      <li<%= sidebar_current("isotime-template-function") %>>
+        <a href="/guides/isotime-template-function.html">Isotime Template Function</a>
+      </li>
       <li<%= sidebar_current("guides-veewee-to-packer") %>>
         <a href="/guides/veewee-to-packer.html">Veewee to Packer</a>
       </li>
+      <li<%= sidebar_current("use-packer-with-comment") %>>
+        <a href="/guides/use-packer-with-comment.html">Use jq to strip comments from a Packer template</a>
+      </li>
       <li<%= sidebar_current("guides-packer-on-cicd") %>>
         <a href="/guides/packer-on-cicd/index.html">Build Immutable Infrastructure with Packer in CI/CD</a>
         <ul class="nav">

website/source/partials/builders/_aws-ssh-differentiation-table.html.md

@@ -0,0 +1,11 @@
+## Which SSH Options to use:
+
+This chart breaks down what Packer does if you set any of the below SSH options:
+
+| ssh_password | ssh_private_key_file | ssh_keypair_name | temporary_key_pair_name | Packer will... |
+| --- | --- | --- | --- | --- |
+| X | - | - | - | ssh authenticating with username and given password |
+| - | X | - | - | ssh authenticating with private key file |
+| - | X | X | - | ssh authenticating with given private key file and "attaching" the keypair to the instance |
+| - | - | - | X | Create a temporary ssh keypair with a particular name, clean it up |
+| - | - | - | - | Create a temporary ssh keypair with a default name, clean it up |

website/source/partials/builders/_building_on_remote_vsphere_hypervisor.html.md

@@ -0,0 +1,66 @@
+## Building on a Remote vSphere Hypervisor
+
+In addition to using the desktop products of VMware locally to build virtual
+machines, Packer can use a remote VMware Hypervisor to build the virtual
+machine.
+
+-> **Note:** Packer supports ESXi 5.1 and above.
+
+Before using a remote vSphere Hypervisor, you need to enable GuestIPHack by
+running the following command:
+
+``` text
+esxcli system settings advanced set -o /Net/GuestIPHack -i 1
+```
+
+When using a remote VMware Hypervisor, the builder still downloads the ISO and
+various files locally, and uploads these to the remote machine. Packer currently
+uses SSH to communicate to the ESXi machine rather than the vSphere API. At some
+point, the vSphere API may be used.
+
+Packer also requires VNC to issue boot commands during a build, which may be
+disabled on some remote VMware Hypervisors. Please consult the appropriate
+documentation on how to update VMware Hypervisor's firewall to allow these
+connections. VNC can be disabled by not setting a `boot_command` and setting
+`disable_vnc` to `true`.
+
+Please note that you should disable vMotion for the host you intend to run
+Packer builds on; a vMotion event will cause the Packer build to fail.
+
+To use a remote VMware vSphere Hypervisor to build your virtual machine, fill in
+the required `remote_*` configurations:
+
+-   `remote_type` - This must be set to "esx5".
+
+-   `remote_host` - The host of the remote machine.
+
+Additionally, there are some optional configurations that you'll likely have to
+modify as well:
+
+-   `remote_port` - The SSH port of the remote machine
+
+-   `remote_datastore` - The path to the datastore where the VM will be stored
+    on the ESXi machine.
+
+-   `remote_cache_datastore` - The path to the datastore where supporting files
+    will be stored during the build on the remote machine.
+
+-   `remote_cache_directory` - The path where the ISO and/or floppy files will
+    be stored during the build on the remote machine. The path is relative to
+    the `remote_cache_datastore` on the remote machine.
+
+-   `remote_username` - The SSH username used to access the remote machine.
+
+-   `remote_password` - The SSH password for access to the remote machine.
+
+-   `remote_private_key_file` - The SSH key for access to the remote machine.
+
+-   `format` (string) - Either "ovf", "ova" or "vmx", this specifies the output
+    format of the exported virtual machine. This defaults to "ovf".
+    Before using this option, you need to install `ovftool`. This option
+    currently only works when option remote_type is set to "esx5".
+    Since ovftool is only capable of password based authentication
+    `remote_password` must be set when exporting the VM.
+
+-   `vnc_disable_password` - This must be set to "true" when using VNC with
+    ESXi 6.5 or 6.7.