From 5ccbd27b726ac15bd6c698f8317281d9670da02c Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 23 Feb 2021 16:51:13 +0100 Subject: [PATCH] Improve upgrade error and transform all variables with template eng to locals --- command/hcl2_upgrade.go | 149 +++++++++++------- command/hcl2_upgrade_test.go | 1 + .../expected.pkr.hcl | 58 +++++++ .../input.json | 35 ++++ .../hcl2_upgrade/complete/expected.pkr.hcl | 26 +-- .../hcl2_upgrade/complete/input.json | 2 +- .../without-annotations/expected.pkr.hcl | 26 +-- .../without-annotations/input.json | 2 +- 8 files changed, 220 insertions(+), 79 deletions(-) create mode 100644 command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl create mode 100644 command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/input.json diff --git a/command/hcl2_upgrade.go b/command/hcl2_upgrade.go index a979cf751..b0647a23e 100644 --- a/command/hcl2_upgrade.go +++ b/command/hcl2_upgrade.go @@ -12,6 +12,7 @@ import ( "strings" texttemplate "text/template" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl/v2/hclwrite" hcl2shim "github.com/hashicorp/packer-plugin-sdk/hcl2helper" "github.com/hashicorp/packer-plugin-sdk/template" @@ -279,28 +280,29 @@ func transposeTemplatingCalls(s []byte) []byte { return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...) } - funcMap := templateCommonFunctionMap() - tpl, err := texttemplate.New("hcl2_upgrade"). - Funcs(funcMap). - Parse(string(s)) + funcErrors := &multierror.Error{ + ErrorFormat: func(es []error) string { + if len(es) == 1 { + return fmt.Sprintf("# 1 error occurred upgrading the following block:\n\t# %s\n", es[0]) + } - if err != nil { - return fallbackReturn(err) + points := make([]string, len(es)) + for i, err := range es { + if i == len(es)-1 { + points[i] = fmt.Sprintf("# %s", err) + continue + } + points[i] = fmt.Sprintf("# %s\n", err) + } + + return fmt.Sprintf( + "# %d errors occurred upgrading the following block:\n\t%s", + len(es), strings.Join(points, "\n\t")) + }, } - str := &bytes.Buffer{} - // PASSTHROUGHS is a map of variable-specific golang text template fields - // that should remain in the text template format. - if err := tpl.Execute(str, PASSTHROUGHS); err != nil { - return fallbackReturn(err) - } - - return str.Bytes() -} - -func templateCommonFunctionMap() texttemplate.FuncMap { - return texttemplate.FuncMap{ + funcMap := texttemplate.FuncMap{ "aws_secretsmanager": func(a ...string) string { if len(a) == 2 { for key, config := range amazonSecretsManagerMap { @@ -365,49 +367,55 @@ func templateCommonFunctionMap() texttemplate.FuncMap { "uuid": func() string { return fmt.Sprintf("${uuidv4()}") }, - "lower": func(_ string) (string, error) { - return "", UnhandleableArgumentError{ + "lower": func(a string) (string, error) { + funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{ "lower", "`lower(var.example)`", "https://www.packer.io/docs/templates/hcl_templates/functions/string/lower", - } + }) + return fmt.Sprintf("{{ lower `%s` }}", a), nil }, - "upper": func(_ string) (string, error) { - return "", UnhandleableArgumentError{ + "upper": func(a string) (string, error) { + funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{ "upper", "`upper(var.example)`", "https://www.packer.io/docs/templates/hcl_templates/functions/string/upper", - } + }) + return fmt.Sprintf("{{ upper `%s` }}", a), nil }, - "split": func(_, _ string, _ int) (string, error) { - return "", UnhandleableArgumentError{ + "split": func(a, b string, n int) (string, error) { + funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{ "split", "`split(separator, string)`", "https://www.packer.io/docs/templates/hcl_templates/functions/string/split", - } + }) + return fmt.Sprintf("{{ split `%s` `%s` %d }}", a, b, n), nil }, - "replace": func(_, _, _ string, _ int) (string, error) { - return "", UnhandleableArgumentError{ + "replace": func(a, b string, n int, c string) (string, error) { + funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{ "replace", "`replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`", "https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace", - } + }) + return fmt.Sprintf("{{ replace `%s` `%s` `%s` %d }}", a, b, c, n), nil }, - "replace_all": func(_, _, _ string) (string, error) { - return "", UnhandleableArgumentError{ + "replace_all": func(a, b, c string) (string, error) { + funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{ "replace_all", "`replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`", "https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace", - } + }) + return fmt.Sprintf("{{ replace_all `%s` `%s` `%s` }}", a, b, c), nil }, - "clean_resource_name": func(_ string) (string, error) { - return "", UnhandleableArgumentError{ + "clean_resource_name": func(a string) (string, error) { + funcErrors = multierror.Append(funcErrors, UnhandleableArgumentError{ "clean_resource_name", "use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`", "https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules" + " , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace" + " or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace", - } + }) + return fmt.Sprintf("{{ clean_resource_name `%s` }}", a), nil }, "build_name": func() string { return fmt.Sprintf("${build.name}") @@ -416,6 +424,28 @@ func templateCommonFunctionMap() texttemplate.FuncMap { return fmt.Sprintf("${build.type}") }, } + + tpl, err := texttemplate.New("hcl2_upgrade"). + Funcs(funcMap). + Parse(string(s)) + + if err != nil { + return fallbackReturn(err) + } + + str := &bytes.Buffer{} + // PASSTHROUGHS is a map of variable-specific golang text template fields + // that should remain in the text template format. + if err := tpl.Execute(str, PASSTHROUGHS); err != nil { + return fallbackReturn(err) + } + + out := str.Bytes() + + if funcErrors.Len() > 0 { + return append([]byte(fmt.Sprintf("\n%s", funcErrors)), out...) + } + return out } // variableTransposeTemplatingCalls executes parts of blocks as go template files and replaces @@ -440,27 +470,32 @@ func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) { // Make locals from variables using valid template engine, // expect the ones using only 'env' // ref: https://www.packer.io/docs/templates/legacy_json_templates/engine#template-engine - funcMap := templateCommonFunctionMap() - funcMap["aws_secretsmanager"] = setIsLocal - funcMap["user"] = setIsLocal - funcMap["isotime"] = setIsLocal - funcMap["timestamp"] = setIsLocal - funcMap["template_dir"] = setIsLocal - funcMap["lower"] = setIsLocal - funcMap["upper"] = setIsLocal - funcMap["uuid"] = setIsLocal - funcMap["pwd"] = setIsLocal - funcMap["split"] = func(_, _ string, _ int) (string, error) { - isLocal = true - return "", nil - } - funcMap["replace"] = func(_, _, _ string, _ int) (string, error) { - isLocal = true - return "", nil - } - funcMap["replace_all"] = func(_, _, _ string) (string, error) { - isLocal = true - return "", nil + funcMap := texttemplate.FuncMap{ + "aws_secretsmanager": setIsLocal, + "timestamp": setIsLocal, + "isotime": setIsLocal, + "user": setIsLocal, + "env": func(in string) string { + return fmt.Sprintf("${env(%q)}", in) + }, + "template_dir": setIsLocal, + "pwd": setIsLocal, + "packer_version": setIsLocal, + "uuid": setIsLocal, + "lower": setIsLocal, + "upper": setIsLocal, + "split": func(_, _ string, _ int) (string, error) { + isLocal = true + return "", nil + }, + "replace": func(_, _ string, _ int, _ string) (string, error) { + isLocal = true + return "", nil + }, + "replace_all": func(_, _, _ string) (string, error) { + isLocal = true + return "", nil + }, } tpl, err := texttemplate.New("hcl2_upgrade"). diff --git a/command/hcl2_upgrade_test.go b/command/hcl2_upgrade_test.go index 1d8818e4e..1e3ce491f 100644 --- a/command/hcl2_upgrade_test.go +++ b/command/hcl2_upgrade_test.go @@ -28,6 +28,7 @@ func Test_hcl2_upgrade(t *testing.T) { {folder: "aws-access-config", flags: []string{}}, {folder: "variables-only", flags: []string{}}, {folder: "variables-with-variables", flags: []string{}}, + {folder: "complete-variables-with-template-engine", flags: []string{}}, } for _, tc := range tc { diff --git a/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl new file mode 100644 index 000000000..1116310d6 --- /dev/null +++ b/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl @@ -0,0 +1,58 @@ + +variable "env_test" { + type = string + default = "${env("TEST_ENV")}" +} + +locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } + +# 5 errors occurred upgrading the following block: +# unhandled "lower" call: +# there is no way to automatically upgrade the "lower" call. +# Please manually upgrade to `lower(var.example)` +# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/lower for more infos. + +# unhandled "replace" call: +# there is no way to automatically upgrade the "replace" call. +# Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` +# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. + +# unhandled "replace_all" call: +# there is no way to automatically upgrade the "replace_all" call. +# Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` +# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. + +# unhandled "split" call: +# there is no way to automatically upgrade the "split" call. +# Please manually upgrade to `split(separator, string)` +# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/split for more infos. + +# unhandled "upper" call: +# there is no way to automatically upgrade the "upper" call. +# Please manually upgrade to `upper(var.example)` +# Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/upper for more infos. +locals { + build_timestamp = "${local.timestamp}" + iso_datetime = "${local.timestamp}" + lower = "{{ lower `HELLO` }}" + pwd = "${path.cwd}" + replace = "{{ replace `b` `c` `ababa` 2 }}" + replace_all = "{{ replace_all `b` `c` `ababa` }}" + split = "{{ split `aba` `b` 1 }}" + temp_directory = "${path.root}" + upper = "{{ upper `hello` }}" + uuid = "${uuidv4()}" +} + +source "null" "autogenerated_1" { + communicator = "none" +} + +build { + sources = ["source.null.autogenerated_1"] + + provisioner "shell-local" { + inline = ["echo ${local.build_timestamp}", "echo ${local.temp_directory}", "echo ${local.iso_datetime}", "echo ${local.uuid}", "echo ${var.env_test}", "echo ${local.lower}", "echo ${local.upper}", "echo ${local.pwd}", "echo ${local.replace}", "echo ${local.replace_all}", "echo ${local.split}"] + } + +} diff --git a/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/input.json b/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/input.json new file mode 100644 index 000000000..17117221f --- /dev/null +++ b/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/input.json @@ -0,0 +1,35 @@ +{ + "variables": { + "build_timestamp": "{{timestamp}}", + "temp_directory": "{{template_dir}}", + "uuid": "{{uuid}}", + "env_test": "{{env `TEST_ENV`}}", + "lower": "{{lower `HELLO`}}", + "upper": "{{upper `hello`}}", + "pwd": "{{pwd}}", + "replace": "{{replace `b` `c` 2 `ababa`}}", + "replace_all": "{{replace_all `b` `c` `ababa`}}", + "split": "{{split `aba` `b` 1}}", + "iso_datetime": "{{isotime `2006-01-02T15:04:05Z07:00`}}" + }, + "builders": [{ + "type": "null", + "communicator": "none" + }], + "provisioners": [{ + "type": "shell-local", + "inline": [ + "echo {{ user `build_timestamp`}}", + "echo {{ user `temp_directory`}}", + "echo {{ user `iso_datetime`}}", + "echo {{ user `uuid`}}", + "echo {{ user `env_test`}}", + "echo {{ user `lower`}}", + "echo {{ user `upper`}}", + "echo {{ user `pwd`}}", + "echo {{ user `replace`}}", + "echo {{ user `replace_all`}}", + "echo {{ user `split`}}" + ] + }] +} \ No newline at end of file diff --git a/command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl index 7cf8d558c..4293048e5 100644 --- a/command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl +++ b/command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl @@ -187,34 +187,38 @@ build { } - # template: hcl2_upgrade:2:38: executing "hcl2_upgrade" at : error calling clean_resource_name: unhandled "clean_resource_name" call: + # 1 error occurred upgrading the following block: + # unhandled "clean_resource_name" call: # there is no way to automatically upgrade the "clean_resource_name" call. # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. provisioner "shell" { - inline = ["echo mybuild-{{isotime | clean_resource_name}}"] + inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"] } - # template: hcl2_upgrade:2:35: executing "hcl2_upgrade" at : error calling lower: unhandled "lower" call: + # 1 error occurred upgrading the following block: + # unhandled "lower" call: # there is no way to automatically upgrade the "lower" call. # Please manually upgrade to `lower(var.example)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/lower for more infos. provisioner "shell" { - inline = ["echo {{ `SOMETHING` | lower }}"] + inline = ["echo {{ lower `SOMETHING` }}"] } - # template: hcl2_upgrade:2:35: executing "hcl2_upgrade" at : error calling upper: unhandled "upper" call: + # 1 error occurred upgrading the following block: + # unhandled "upper" call: # there is no way to automatically upgrade the "upper" call. # Please manually upgrade to `upper(var.example)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/upper for more infos. provisioner "shell" { - inline = ["echo {{ `something` | upper }}"] + inline = ["echo {{ upper `something` }}"] } - # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at : error calling split: unhandled "split" call: + # 1 error occurred upgrading the following block: + # unhandled "split" call: # there is no way to automatically upgrade the "split" call. # Please manually upgrade to `split(separator, string)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/split for more infos. @@ -223,16 +227,18 @@ build { } - # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at : error calling replace_all: unhandled "replace_all" call: + # 1 error occurred upgrading the following block: + # unhandled "replace_all" call: # there is no way to automatically upgrade the "replace_all" call. # Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. provisioner "shell" { - inline = ["echo {{ replace_all `-` `/` build_name }}"] + inline = ["echo {{ replace_all `-` `/` `${build.name}` }}"] } - # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at : error calling replace: unhandled "replace" call: + # 1 error occurred upgrading the following block: + # unhandled "replace" call: # there is no way to automatically upgrade the "replace" call. # Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. diff --git a/command/test-fixtures/hcl2_upgrade/complete/input.json b/command/test-fixtures/hcl2_upgrade/complete/input.json index 5efa2c419..3c7461ed6 100644 --- a/command/test-fixtures/hcl2_upgrade/complete/input.json +++ b/command/test-fixtures/hcl2_upgrade/complete/input.json @@ -180,7 +180,7 @@ { "type": "shell", "inline": [ - "echo {{ replace `some-string` `-` `/` 1 }}" + "echo {{ replace `some-string` `-` 1 `/` }}" ] }, { diff --git a/command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl index 779fa6ed9..93c51bdc4 100644 --- a/command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl +++ b/command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl @@ -145,34 +145,38 @@ build { } - # template: hcl2_upgrade:2:38: executing "hcl2_upgrade" at : error calling clean_resource_name: unhandled "clean_resource_name" call: + # 1 error occurred upgrading the following block: + # unhandled "clean_resource_name" call: # there is no way to automatically upgrade the "clean_resource_name" call. # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. provisioner "shell" { - inline = ["echo mybuild-{{isotime | clean_resource_name}}"] + inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"] } - # template: hcl2_upgrade:2:35: executing "hcl2_upgrade" at : error calling lower: unhandled "lower" call: + # 1 error occurred upgrading the following block: + # unhandled "lower" call: # there is no way to automatically upgrade the "lower" call. # Please manually upgrade to `lower(var.example)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/lower for more infos. provisioner "shell" { - inline = ["echo {{ `SOMETHING` | lower }}"] + inline = ["echo {{ lower `SOMETHING` }}"] } - # template: hcl2_upgrade:2:35: executing "hcl2_upgrade" at : error calling upper: unhandled "upper" call: + # 1 error occurred upgrading the following block: + # unhandled "upper" call: # there is no way to automatically upgrade the "upper" call. # Please manually upgrade to `upper(var.example)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/upper for more infos. provisioner "shell" { - inline = ["echo {{ `something` | upper }}"] + inline = ["echo {{ upper `something` }}"] } - # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at : error calling split: unhandled "split" call: + # 1 error occurred upgrading the following block: + # unhandled "split" call: # there is no way to automatically upgrade the "split" call. # Please manually upgrade to `split(separator, string)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/split for more infos. @@ -181,16 +185,18 @@ build { } - # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at : error calling replace_all: unhandled "replace_all" call: + # 1 error occurred upgrading the following block: + # unhandled "replace_all" call: # there is no way to automatically upgrade the "replace_all" call. # Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. provisioner "shell" { - inline = ["echo {{ replace_all `-` `/` build_name }}"] + inline = ["echo {{ replace_all `-` `/` `${build.name}` }}"] } - # template: hcl2_upgrade:2:21: executing "hcl2_upgrade" at : error calling replace: unhandled "replace" call: + # 1 error occurred upgrading the following block: + # unhandled "replace" call: # there is no way to automatically upgrade the "replace" call. # Please manually upgrade to `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. diff --git a/command/test-fixtures/hcl2_upgrade/without-annotations/input.json b/command/test-fixtures/hcl2_upgrade/without-annotations/input.json index 1b4eb5dec..fdd5cf257 100644 --- a/command/test-fixtures/hcl2_upgrade/without-annotations/input.json +++ b/command/test-fixtures/hcl2_upgrade/without-annotations/input.json @@ -180,7 +180,7 @@ { "type": "shell", "inline": [ - "echo {{ replace `some-string` `-` `/` 1 }}" + "echo {{ replace `some-string` `-` 1 `/` }}" ] }, {