Improve upgrade error and transform all variables with template eng to locals

command/hcl2_upgrade.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	texttemplate "text/template"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/hcl/v2/hclwrite"
 	hcl2shim "github.com/hashicorp/packer-plugin-sdk/hcl2helper"
 	"github.com/hashicorp/packer-plugin-sdk/template"
@@ -279,28 +280,29 @@ func transposeTemplatingCalls(s []byte) []byte {
 
 		return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...)
 	}
-	funcMap := templateCommonFunctionMap()
 
-	tpl, err := texttemplate.New("hcl2_upgrade").
+	funcErrors := &multierror.Error{
-		Funcs(funcMap).
+		ErrorFormat: func(es []error) string {
-		Parse(string(s))
+			if len(es) == 1 {
+				return fmt.Sprintf("# 1 error occurred upgrading the following block:\n\t# %s\n", es[0])
+			}
 
-	if err != nil {
+			points := make([]string, len(es))
-		return fallbackReturn(err)
+			for i, err := range es {
+				if i == len(es)-1 {
+					points[i] = fmt.Sprintf("# %s", err)
+					continue
+				}
+				points[i] = fmt.Sprintf("# %s\n", err)
+			}
+
+			return fmt.Sprintf(
+				"# %d errors occurred upgrading the following block:\n\t%s",
+				len(es), strings.Join(points, "\n\t"))
+		},
 	}
 
-	str := &bytes.Buffer{}
+	funcMap := texttemplate.FuncMap{
-	// PASSTHROUGHS is a map of variable-specific golang text template fields
-	// that should remain in the text template format.
-	if err := tpl.Execute(str, PASSTHROUGHS); err != nil {
-		return fallbackReturn(err)
-	}
-
-	return str.Bytes()
-}
-
-func templateCommonFunctionMap() texttemplate.FuncMap {
-	return texttemplate.FuncMap{
 		"aws_secretsmanager": func(a ...string) string {
 			if len(a) == 2 {
 				for key, config := range amazonSecretsManagerMap {
@@ -365,49 +367,55 @@ func templateCommonFunctionMap() texttemplate.FuncMap {
 		"uuid": func() string {
 			return fmt.Sprintf("${uuidv4()}")
 		},
-		"lower": func(_ string) (string, error) {
+		"lower": func(a string) (string, error) {
-			return "", UnhandleableArgumentError{
+			funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{
 				"lower",
 				"`lower(var.example)`",
 				"https://www.packer.io/docs/templates/hcl_templates/functions/string/lower",
-			}
+			})
+			return fmt.Sprintf("{{ lower `%s` }}", a), nil
 		},
-		"upper": func(_ string) (string, error) {
+		"upper": func(a string) (string, error) {
-			return "", UnhandleableArgumentError{
+			funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{
 				"upper",
 				"`upper(var.example)`",
 				"https://www.packer.io/docs/templates/hcl_templates/functions/string/upper",
-			}
+			})
+			return fmt.Sprintf("{{ upper `%s` }}", a), nil
 		},
-		"split": func(_, _ string, _ int) (string, error) {
+		"split": func(a, b string, n int) (string, error) {
-			return "", UnhandleableArgumentError{
+			funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{
 				"split",
 				"`split(separator, string)`",
 				"https://www.packer.io/docs/templates/hcl_templates/functions/string/split",
-			}
+			})
+			return fmt.Sprintf("{{ split `%s` `%s` %d }}", a, b, n), nil
 		},
-		"replace": func(_, _, _ string, _ int) (string, error) {
+		"replace": func(a, b string, n int, c string) (string, error) {
-			return "", UnhandleableArgumentError{
+			funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{
 				"replace",
 				"`replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`",
 				"https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace",
-			}
+			})
+			return fmt.Sprintf("{{ replace `%s` `%s` `%s` %d }}", a, b, c, n), nil
 		},
-		"replace_all": func(_, _, _ string) (string, error) {
+		"replace_all": func(a, b, c string) (string, error) {
-			return "", UnhandleableArgumentError{
+			funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{
 				"replace_all",
 				"`replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`",
 				"https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace",
-			}
+			})
+			return fmt.Sprintf("{{ replace_all `%s` `%s` `%s` }}", a, b, c), nil
 		},
-		"clean_resource_name": func(_ string) (string, error) {
+		"clean_resource_name": func(a string) (string, error) {
-			return "", UnhandleableArgumentError{
+			funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{
 				"clean_resource_name",
 				"use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`",
 				"https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules" +
 					" , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace" +
 					" or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace",
-			}
+			})
+			return fmt.Sprintf("{{ clean_resource_name `%s` }}", a), nil
 		},
 		"build_name": func() string {
 			return fmt.Sprintf("${build.name}")
@@ -416,6 +424,28 @@ func templateCommonFunctionMap() texttemplate.FuncMap {
 			return fmt.Sprintf("${build.type}")
 		},
 	}
+
+	tpl, err := texttemplate.New("hcl2_upgrade").
+		Funcs(funcMap).
+		Parse(string(s))
+
+	if err != nil {
+		return fallbackReturn(err)
+	}
+
+	str := &bytes.Buffer{}
+	// PASSTHROUGHS is a map of variable-specific golang text template fields
+	// that should remain in the text template format.
+	if err := tpl.Execute(str, PASSTHROUGHS); err != nil {
+		return fallbackReturn(err)
+	}
+
+	out := str.Bytes()
+
+	if funcErrors.Len() > 0 {
+		return append([]byte(fmt.Sprintf("\n%s", funcErrors)), out...)
+	}
+	return out
 }
 
 // variableTransposeTemplatingCalls executes parts of blocks as go template files and replaces
@@ -440,27 +470,32 @@ func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) {
 	// Make locals from variables using valid template engine,
 	// expect the ones using only 'env'
 	// ref: https://www.packer.io/docs/templates/legacy_json_templates/engine#template-engine
-	funcMap := templateCommonFunctionMap()
+	funcMap := texttemplate.FuncMap{
-	funcMap["aws_secretsmanager"] = setIsLocal
+		"aws_secretsmanager": setIsLocal,
-	funcMap["user"] = setIsLocal
+		"timestamp":          setIsLocal,
-	funcMap["isotime"] = setIsLocal
+		"isotime":            setIsLocal,
-	funcMap["timestamp"] = setIsLocal
+		"user":               setIsLocal,
-	funcMap["template_dir"] = setIsLocal
+		"env": func(in string) string {
-	funcMap["lower"] = setIsLocal
+			return fmt.Sprintf("${env(%q)}", in)
-	funcMap["upper"] = setIsLocal
+		},
-	funcMap["uuid"] = setIsLocal
+		"template_dir":   setIsLocal,
-	funcMap["pwd"] = setIsLocal
+		"pwd":            setIsLocal,
-	funcMap["split"] = func(_, _ string, _ int) (string, error) {
+		"packer_version": setIsLocal,
-		isLocal = true
+		"uuid":           setIsLocal,
-		return "", nil
+		"lower":          setIsLocal,
-	}
+		"upper":          setIsLocal,
-	funcMap["replace"] = func(_, _, _ string, _ int) (string, error) {
+		"split": func(_, _ string, _ int) (string, error) {
-		isLocal = true
+			isLocal = true
-		return "", nil
+			return "", nil
-	}
+		},
-	funcMap["replace_all"] = func(_, _, _ string) (string, error) {
+		"replace": func(_, _ string, _ int, _ string) (string, error) {
-		isLocal = true
+			isLocal = true
-		return "", nil
+			return "", nil
+		},
+		"replace_all": func(_, _, _ string) (string, error) {
+			isLocal = true
+			return "", nil
+		},
 	}
 
 	tpl, err := texttemplate.New("hcl2_upgrade").

command/hcl2_upgrade_test.go

@@ -28,6 +28,7 @@ func Test_hcl2_upgrade(t *testing.T) {
 		{folder: "aws-access-config", flags: []string{}},
 		{folder: "variables-only", flags: []string{}},
 		{folder: "variables-with-variables", flags: []string{}},
+		{folder: "complete-variables-with-template-engine", flags: []string{}},
 	}
 
 	for _, tc := range tc {

command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl

@@ -0,0 +1,58 @@
+
+variable "env_test" {
+  type    = string
+  default = "${env("TEST_ENV")}"
+}
+
+locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
+
+# 5 errors occurred upgrading the following block:
+# unhandled "lower" call:
+# there is no way to automatically upgrade the "lower" call.
+# Please manually upgrade to `lower(var.example)`
+# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/lower for more infos.
+
+# unhandled "replace" call:
+# there is no way to automatically upgrade the "replace" call.
+# Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
+# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
+
+# unhandled "replace_all" call:
+# there is no way to automatically upgrade the "replace_all" call.
+# Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
+# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
+
+# unhandled "split" call:
+# there is no way to automatically upgrade the "split" call.
+# Please manually upgrade to `split(separator, string)`
+# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/split for more infos.
+
+# unhandled "upper" call:
+# there is no way to automatically upgrade the "upper" call.
+# Please manually upgrade to `upper(var.example)`
+# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/upper for more infos.
+locals {
+  build_timestamp = "${local.timestamp}"
+  iso_datetime    = "${local.timestamp}"
+  lower           = "{{ lower `HELLO` }}"
+  pwd             = "${path.cwd}"
+  replace         = "{{ replace `b` `c` `ababa` 2 }}"
+  replace_all     = "{{ replace_all `b` `c` `ababa` }}"
+  split           = "{{ split `aba` `b` 1 }}"
+  temp_directory  = "${path.root}"
+  upper           = "{{ upper `hello` }}"
+  uuid            = "${uuidv4()}"
+}
+
+source "null" "autogenerated_1" {
+  communicator = "none"
+}
+
+build {
+  sources = ["source.null.autogenerated_1"]
+
+  provisioner "shell-local" {
+    inline = ["echo ${local.build_timestamp}", "echo ${local.temp_directory}", "echo ${local.iso_datetime}", "echo ${local.uuid}", "echo ${var.env_test}", "echo ${local.lower}", "echo ${local.upper}", "echo ${local.pwd}", "echo ${local.replace}", "echo ${local.replace_all}", "echo ${local.split}"]
+  }
+
+}

command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/input.json

@@ -0,0 +1,35 @@
+{
+    "variables": {
+        "build_timestamp": "{{timestamp}}",
+        "temp_directory": "{{template_dir}}",
+        "uuid": "{{uuid}}",
+        "env_test": "{{env `TEST_ENV`}}",
+        "lower": "{{lower `HELLO`}}",
+        "upper": "{{upper `hello`}}",
+        "pwd": "{{pwd}}",
+        "replace": "{{replace `b` `c` 2 `ababa`}}",
+        "replace_all": "{{replace_all `b` `c` `ababa`}}",
+        "split": "{{split `aba` `b` 1}}",
+        "iso_datetime": "{{isotime `2006-01-02T15:04:05Z07:00`}}"
+    },
+    "builders": [{
+        "type": "null",
+        "communicator": "none"
+    }],
+    "provisioners": [{
+        "type": "shell-local",
+        "inline": [
+            "echo {{ user `build_timestamp`}}",
+            "echo {{ user `temp_directory`}}",
+            "echo {{ user `iso_datetime`}}",
+            "echo {{ user `uuid`}}",
+            "echo {{ user `env_test`}}",
+            "echo {{ user `lower`}}",
+            "echo {{ user `upper`}}",
+            "echo {{ user `pwd`}}",
+            "echo {{ user `replace`}}",
+            "echo {{ user `replace_all`}}",
+            "echo {{ user `split`}}"
+        ]
+    }]
+}

command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl

@@ -187,34 +187,38 @@ build {
   }
 
 
-  # template: hcl2_upgrade:2:38: executing "hcl2_upgrade" at <clean_resource_name>: error calling clean_resource_name: unhandled "clean_resource_name" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "clean_resource_name" call:
   # there is no way to automatically upgrade the "clean_resource_name" call.
   # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
   provisioner "shell" {
-    inline = ["echo mybuild-{{isotime | clean_resource_name}}"]
+    inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"]
   }
 
 
-  # template: hcl2_upgrade:2:35: executing "hcl2_upgrade" at <lower>: error calling lower: unhandled "lower" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "lower" call:
   # there is no way to automatically upgrade the "lower" call.
   # Please manually upgrade to `lower(var.example)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/lower for more infos.
   provisioner "shell" {
-    inline = ["echo {{ `SOMETHING` | lower }}"]
+    inline = ["echo {{ lower `SOMETHING` }}"]
   }
 
 
-  # template: hcl2_upgrade:2:35: executing "hcl2_upgrade" at <upper>: error calling upper: unhandled "upper" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "upper" call:
   # there is no way to automatically upgrade the "upper" call.
   # Please manually upgrade to `upper(var.example)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/upper for more infos.
   provisioner "shell" {
-    inline = ["echo {{ `something` | upper }}"]
+    inline = ["echo {{ upper `something` }}"]
   }
 
 
-  # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at <split `some-string` `-` 0>: error calling split: unhandled "split" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "split" call:
   # there is no way to automatically upgrade the "split" call.
   # Please manually upgrade to `split(separator, string)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/split for more infos.
@@ -223,16 +227,18 @@ build {
   }
 
 
-  # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at <replace_all `-` `/` build_name>: error calling replace_all: unhandled "replace_all" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "replace_all" call:
   # there is no way to automatically upgrade the "replace_all" call.
   # Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
   provisioner "shell" {
-    inline = ["echo {{ replace_all `-` `/` build_name }}"]
+    inline = ["echo {{ replace_all `-` `/` `${build.name}` }}"]
   }
 
 
-  # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at <replace `some-string` `-` `/` 1>: error calling replace: unhandled "replace" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "replace" call:
   # there is no way to automatically upgrade the "replace" call.
   # Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.

command/test-fixtures/hcl2_upgrade/complete/input.json

@@ -180,7 +180,7 @@
         {
             "type": "shell",
             "inline": [
-                "echo {{ replace `some-string` `-` `/` 1 }}"
+                "echo {{ replace `some-string` `-` 1 `/` }}"
             ]
         },
         {

command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl

@@ -145,34 +145,38 @@ build {
   }
 
 
-  # template: hcl2_upgrade:2:38: executing "hcl2_upgrade" at <clean_resource_name>: error calling clean_resource_name: unhandled "clean_resource_name" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "clean_resource_name" call:
   # there is no way to automatically upgrade the "clean_resource_name" call.
   # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
   provisioner "shell" {
-    inline = ["echo mybuild-{{isotime | clean_resource_name}}"]
+    inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"]
   }
 
 
-  # template: hcl2_upgrade:2:35: executing "hcl2_upgrade" at <lower>: error calling lower: unhandled "lower" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "lower" call:
   # there is no way to automatically upgrade the "lower" call.
   # Please manually upgrade to `lower(var.example)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/lower for more infos.
   provisioner "shell" {
-    inline = ["echo {{ `SOMETHING` | lower }}"]
+    inline = ["echo {{ lower `SOMETHING` }}"]
   }
 
 
-  # template: hcl2_upgrade:2:35: executing "hcl2_upgrade" at <upper>: error calling upper: unhandled "upper" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "upper" call:
   # there is no way to automatically upgrade the "upper" call.
   # Please manually upgrade to `upper(var.example)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/upper for more infos.
   provisioner "shell" {
-    inline = ["echo {{ `something` | upper }}"]
+    inline = ["echo {{ upper `something` }}"]
   }
 
 
-  # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at <split `some-string` `-` 0>: error calling split: unhandled "split" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "split" call:
   # there is no way to automatically upgrade the "split" call.
   # Please manually upgrade to `split(separator, string)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/split for more infos.
@@ -181,16 +185,18 @@ build {
   }
 
 
-  # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at <replace_all `-` `/` build_name>: error calling replace_all: unhandled "replace_all" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "replace_all" call:
   # there is no way to automatically upgrade the "replace_all" call.
   # Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
   provisioner "shell" {
-    inline = ["echo {{ replace_all `-` `/` build_name }}"]
+    inline = ["echo {{ replace_all `-` `/` `${build.name}` }}"]
   }
 
 
-  # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at <replace `some-string` `-` `/` 1>: error calling replace: unhandled "replace" call:
+  # 1 error occurred upgrading the following block:
+  # unhandled "replace" call:
   # there is no way to automatically upgrade the "replace" call.
   # Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.

command/test-fixtures/hcl2_upgrade/without-annotations/input.json

@@ -180,7 +180,7 @@
         {
             "type": "shell",
             "inline": [
-                "echo {{ replace `some-string` `-` `/` 1 }}"
+                "echo {{ replace `some-string` `-` 1 `/` }}"
             ]
         },
         {