HCL2: add templatefile function (#10776)

* tests
* docs

.gitattributes

@@ -6,6 +6,7 @@
 *.mdx text eol=lf
 *.ps1 text eol=lf
 *.hcl text eol=lf
+*.tmpl text eol=lf
 *.txt text eol=lf
 go.mod text eol=lf
 go.sum text eol=lf

hcl2template/function/templatefile.go

@@ -0,0 +1,140 @@
+package function
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/go-cty-funcs/filesystem"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function"
+)
+
+// MakeTemplateFileFunc constructs a function that takes a file path and
+// an arbitrary object of named values and attempts to render the referenced
+// file as a template using HCL template syntax.
+//
+// The template itself may recursively call other functions so a callback
+// must be provided to get access to those functions. The template cannot,
+// however, access any variables defined in the scope: it is restricted only to
+// those variables provided in the second function argument.
+//
+// As a special exception, a referenced template file may not recursively call
+// the templatefile function, since that would risk the same file being
+// included into itself indefinitely.
+func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Function) function.Function {
+
+	params := []function.Parameter{
+		{
+			Name: "path",
+			Type: cty.String,
+		},
+		{
+			Name: "vars",
+			Type: cty.DynamicPseudoType,
+		},
+	}
+
+	loadTmpl := func(fn string) (hcl.Expression, error) {
+		// We re-use File here to ensure the same filename interpretation
+		// as it does, along with its other safety checks.
+		tmplVal, err := filesystem.File(baseDir, cty.StringVal(fn))
+		if err != nil {
+			return nil, err
+		}
+
+		expr, diags := hclsyntax.ParseTemplate([]byte(tmplVal.AsString()), fn, hcl.Pos{Line: 1, Column: 1})
+		if diags.HasErrors() {
+			return nil, diags
+		}
+
+		return expr, nil
+	}
+
+	renderTmpl := func(expr hcl.Expression, varsVal cty.Value) (cty.Value, error) {
+		if varsTy := varsVal.Type(); !(varsTy.IsMapType() || varsTy.IsObjectType()) {
+			return cty.DynamicVal, function.NewArgErrorf(1, "invalid vars value: must be a map") // or an object, but we don't strongly distinguish these most of the time
+		}
+
+		ctx := &hcl.EvalContext{
+			Variables: varsVal.AsValueMap(),
+		}
+
+		// We require all of the variables to be valid HCL identifiers, because
+		// otherwise there would be no way to refer to them in the template
+		// anyway. Rejecting this here gives better feedback to the user
+		// than a syntax error somewhere in the template itself.
+		for n := range ctx.Variables {
+			if !hclsyntax.ValidIdentifier(n) {
+				// This error message intentionally doesn't describe _all_ of
+				// the different permutations that are technically valid as an
+				// HCL identifier, but rather focuses on what we might
+				// consider to be an "idiomatic" variable name.
+				return cty.DynamicVal, function.NewArgErrorf(1, "invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
+			}
+		}
+
+		// We'll pre-check references in the template here so we can give a
+		// more specialized error message than HCL would by default, so it's
+		// clearer that this problem is coming from a templatefile call.
+		for _, traversal := range expr.Variables() {
+			root := traversal.RootName()
+			if _, ok := ctx.Variables[root]; !ok {
+				return cty.DynamicVal, function.NewArgErrorf(1, "vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
+			}
+		}
+
+		givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems
+		funcs := make(map[string]function.Function, len(givenFuncs))
+		for name, fn := range givenFuncs {
+			if name == "templatefile" {
+				// We stub this one out to prevent recursive calls.
+				funcs[name] = function.New(&function.Spec{
+					Params: params,
+					Type: func(args []cty.Value) (cty.Type, error) {
+						return cty.NilType, fmt.Errorf("cannot recursively call templatefile from inside templatefile call")
+					},
+				})
+				continue
+			}
+			funcs[name] = fn
+		}
+		ctx.Functions = funcs
+
+		val, diags := expr.Value(ctx)
+		if diags.HasErrors() {
+			return cty.DynamicVal, diags
+		}
+		return val, nil
+	}
+
+	return function.New(&function.Spec{
+		Params: params,
+		Type: func(args []cty.Value) (cty.Type, error) {
+			if !(args[0].IsKnown() && args[1].IsKnown()) {
+				return cty.DynamicPseudoType, nil
+			}
+
+			// We'll render our template now to see what result type it
+			// produces. A template consisting only of a single interpolation
+			// can potentially return any type.
+			expr, err := loadTmpl(args[0].AsString())
+			if err != nil {
+				return cty.DynamicPseudoType, err
+			}
+
+			// This is safe even if args[1] contains unknowns because the HCL
+			// template renderer itself knows how to short-circuit those.
+			val, err := renderTmpl(expr, args[1])
+			return val.Type(), err
+		},
+		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+			expr, err := loadTmpl(args[0].AsString())
+			if err != nil {
+				return cty.DynamicVal, err
+			}
+			return renderTmpl(expr, args[1])
+		},
+	})
+
+}

hcl2template/function/templatefile_test.go

@@ -0,0 +1,151 @@
+package function
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/go-cty-funcs/filesystem"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function"
+	"github.com/zclconf/go-cty/cty/function/stdlib"
+)
+
+func TestTemplateFile(t *testing.T) {
+	tests := []struct {
+		Path cty.Value
+		Vars cty.Value
+		Want cty.Value
+		Err  string
+	}{
+		{
+			cty.StringVal("testdata/hello.txt"),
+			cty.EmptyObjectVal,
+			cty.StringVal("Hello World"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/icon.png"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`contents of testdata/icon.png are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead`,
+		},
+		{
+			cty.StringVal("testdata/missing"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`no file exists at ` + filepath.Clean("testdata/missing"),
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.MapVal(map[string]cty.Value{
+				"name": cty.StringVal("Jodie"),
+			}),
+			cty.StringVal("Hello, Jodie!"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.MapVal(map[string]cty.Value{
+				"name!": cty.StringVal("Jodie"),
+			}),
+			cty.NilVal,
+			`invalid template variable name "name!": must start with a letter, followed by zero or more letters, digits, and underscores`,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"name": cty.StringVal("Jimbo"),
+			}),
+			cty.StringVal("Hello, Jimbo!"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`vars map does not contain key "name", referenced at testdata/hello.tmpl:1,10-14`,
+		},
+		{
+			cty.StringVal("testdata/func.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+				}),
+			}),
+			cty.StringVal("The items are a, b, c"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/recursive.tmpl"),
+			cty.MapValEmpty(cty.String),
+			cty.NilVal,
+			`testdata/recursive.tmpl:1,3-16: Error in function call; Call to function "templatefile" failed: cannot recursively call templatefile from inside templatefile call.`,
+		},
+		{
+			cty.StringVal("testdata/list.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+				}),
+			}),
+			cty.StringVal("- a\n- b\n- c\n"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/list.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.True,
+			}),
+			cty.NilVal,
+			`testdata/list.tmpl:1,13-17: Iteration over non-iterable value; A value of type bool cannot be used as the collection in a 'for' expression.`,
+		},
+		{
+			cty.StringVal("testdata/bare.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"val": cty.True,
+			}),
+			cty.True, // since this template contains only an interpolation, its true value shines through
+			``,
+		},
+	}
+
+	templateFileFn := MakeTemplateFileFunc(".", func() map[string]function.Function {
+		return map[string]function.Function{
+			"join":         stdlib.JoinFunc,
+			"templatefile": filesystem.MakeFileFunc(".", false), // just a placeholder, since templatefile itself overrides this
+		}
+	})
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("TemplateFile(%#v, %#v)", test.Path, test.Vars), func(t *testing.T) {
+			got, err := templateFileFn.Call([]cty.Value{test.Path, test.Vars})
+
+			if argErr, ok := err.(function.ArgError); ok {
+				if argErr.Index < 0 || argErr.Index > 1 {
+					t.Errorf("ArgError index %d is out of range for templatefile (must be 0 or 1)", argErr.Index)
+				}
+			}
+
+			if test.Err != "" {
+				if err == nil {
+					t.Fatal("succeeded; want error")
+				}
+				if got, want := err.Error(), test.Err; got != want {
+					t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
+				return
+			} else if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+
+			if !got.RawEquals(test.Want) {
+				t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, test.Want)
+			}
+		})
+	}
+}

hcl2template/function/testdata/bare.tmpl

@@ -0,0 +1 @@
+${val}

hcl2template/function/testdata/func.tmpl

@@ -0,0 +1 @@
+The items are ${join(", ", list)}

hcl2template/function/testdata/hello.tmpl

@@ -0,0 +1 @@
+Hello, ${name}!

hcl2template/function/testdata/hello.txt

@@ -0,0 +1 @@
+Hello World

hcl2template/function/testdata/list.tmpl

@@ -0,0 +1,3 @@
+%{ for x in list ~}
+- ${x}
+%{ endfor ~}

hcl2template/function/testdata/recursive.tmpl

@@ -0,0 +1 @@
+${templatefile("recursive.tmpl", {})}

hcl2template/functions.go

@@ -118,6 +118,12 @@ func Functions(basedir string) map[string]function.Function {
 		"zipmap":             stdlib.ZipmapFunc,
 	}
 
+	funcs["templatefile"] = pkrfunction.MakeTemplateFileFunc(basedir, func() map[string]function.Function {
+		// The templatefile function prevents recursive calls to itself
+		// by copying this map and overwriting the "templatefile" entry.
+		return funcs
+	})
+
 	return funcs
 }
 

website/content/docs/templates/hcl_templates/functions/file/fileexists.mdx

@@ -32,4 +32,5 @@ fileexists("custom-section.sh") ? file("custom-section.sh") : local.default_cont
 
 ## Related Functions
 
-- [`file`](/docs/templates/hcl_templates/functions/file) reads the contents of a file at a given path
+- [`file`](/docs/templates/hcl_templates/functions/file/file) reads the contents
+  of a file at a given path.

website/content/docs/templates/hcl_templates/functions/file/templatefile.mdx

@@ -0,0 +1,133 @@
+---
+page_title: templatefile - Functions - Configuration Language
+sidebar_title: templatefile
+description: |-
+  The templatefile function reads the file at the given path and renders its
+  content as a template using a supplied set of template variables.
+---
+
+# `templatefile` Function
+
+-> *Recommendation:* we recommend using the `.pkrtpl.hcl` file extension when
+using the `templatefile` function. Template files *are* hcl treated as files but
+also templates and therefore have slightly different set of features
+than the ones offered in a `.pkr.hcl` Packer template. While you are not
+required to use this extension, doing so will enable syntax highlighters to
+properly understand your file.
+
+`templatefile` reads the file at the given path and renders its content as a
+template using a supplied set of template variables.
+
+```hcl
+templatefile(path, vars)
+```
+
+The template syntax is the same as for string templates in the main HCL2
+language, including interpolation sequences delimited with `${ ... }`. This
+function just allows longer template sequences to be factored out into a
+separate file for readability.
+
+The "vars" argument must be a map. Within the template file, each of the keys in
+the map is available as a variable for interpolation. The template may also use
+any other function available in Packer, except that recursive calls to
+templatefile are not permitted. Variable names must each start with a letter,
+followed by zero or more letters, digits, or underscores.
+
+Strings in HCL2 are sequences of Unicode characters, so this function will
+interpret the file contents as UTF-8 encoded text and return the resulting
+Unicode characters. If the file contains invalid UTF-8 sequences then this
+function will produce an error.
+
+This function can be used only with files that already exist on disk at the
+beginning of a run.
+
+## Examples
+
+### Lists
+
+Given a template file backends.tpl with the following content:
+
+```hcl
+%{ for addr in ip_addrs ~}
+backend ${addr}:${port}
+%{ endfor ~}
+```
+The templatefile function renders the template:
+
+```shell-session
+> templatefile("${path.root}/backends.tmpl", { port = 8080, ip_addrs = ["10.0.0.1", "10.0.0.2"] })
+backend 10.0.0.1:8080
+backend 10.0.0.2:8080
+```
+
+### Maps
+
+Given a template file config.tmpl with the following content:
+```hcl
+%{ for config_key, config_value in config }
+set ${config_key} = ${config_value}
+%{ endfor ~}
+```
+The templatefile function renders the template:
+```shell-session
+> templatefile(
+               "${path.root}/config.tmpl",
+               {
+                 config = {
+                   "x"   = "y"
+                   "foo" = "bar"
+                   "key" = "value"
+                 }
+               }
+              )
+set foo = bar
+set key = value
+set x = y
+```
+
+### Generating JSON or YAML from a template
+
+If the string you want to generate will be in JSON or YAML syntax, it's often
+tricky and tedious to write a template that will generate valid JSON or YAML
+that will be interpreted correctly when using lots of individual interpolation
+sequences and directives.
+
+Instead, you can write a template that consists only of a single interpolated
+call to either jsonencode or yamlencode, specifying the value to encode using
+normal expression syntax as in the following examples:
+
+```hcl
+${jsonencode({
+  "backends": [for addr in ip_addrs : "${addr}:${port}"],
+})}
+${yamlencode({
+  "backends": [for addr in ip_addrs : "${addr}:${port}"],
+})}
+```
+
+Given the same input as the `backends.tmpl` example in the previous section,
+this will produce a valid JSON or YAML representation of the given data
+structure, without the need to manually handle escaping or delimiters. In the
+latest examples above, the repetition based on elements of ip_addrs is achieved
+by using a for expression rather than by using template directives.
+
+```
+{"backends":["10.0.0.1:8080","10.0.0.2:8080"]}
+```
+If the resulting template is small, you can choose instead to write jsonencode or yamlencode calls inline in your main configuration files, and avoid creating separate template files at all:
+
+locals {
+  backend_config_json = jsonencode({
+    "backends": [for addr in ip_addrs : "${addr}:${port}"],
+  })
+}
+For more information, see the main documentation for jsonencode and yamlencode.
+
+»
+
+## Related Functions
+
+- [`file`](/docs/templates/hcl_templates/functions/file/file) reads the contents
+  of a file at a given path.
+- [`fileexists`](/docs/templates/hcl_templates/functions/file/fileexists)
+  determines whether a file exists at a given path.

website/data/docs-nav-data.json

@@ -449,6 +449,10 @@
                   {
                     "title": "pathexpand",
                     "path": "templates/hcl_templates/functions/file/pathexpand"
+                  },
+                  {
+                    "title": "templatefile",
+                    "path": "templates/hcl_templates/functions/file/templatefile"
                   }
                 ]
               },

website/data/docs-navigation.js

@@ -137,6 +137,7 @@ export default [
                   'fileexists',
                   'fileset',
                   'pathexpand',
+                  'templatefile',
                 ],
               },
               {