diff --git a/.gitattributes b/.gitattributes index 0eb67ecaf..af45aba81 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,6 +6,7 @@ *.mdx text eol=lf *.ps1 text eol=lf *.hcl text eol=lf +*.tmpl text eol=lf *.txt text eol=lf go.mod text eol=lf go.sum text eol=lf diff --git a/hcl2template/function/templatefile.go b/hcl2template/function/templatefile.go new file mode 100644 index 000000000..d7edf23b2 --- /dev/null +++ b/hcl2template/function/templatefile.go @@ -0,0 +1,140 @@ +package function + +import ( + "fmt" + + "github.com/hashicorp/go-cty-funcs/filesystem" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" +) + +// MakeTemplateFileFunc constructs a function that takes a file path and +// an arbitrary object of named values and attempts to render the referenced +// file as a template using HCL template syntax. +// +// The template itself may recursively call other functions so a callback +// must be provided to get access to those functions. The template cannot, +// however, access any variables defined in the scope: it is restricted only to +// those variables provided in the second function argument. +// +// As a special exception, a referenced template file may not recursively call +// the templatefile function, since that would risk the same file being +// included into itself indefinitely. +func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Function) function.Function { + + params := []function.Parameter{ + { + Name: "path", + Type: cty.String, + }, + { + Name: "vars", + Type: cty.DynamicPseudoType, + }, + } + + loadTmpl := func(fn string) (hcl.Expression, error) { + // We re-use File here to ensure the same filename interpretation + // as it does, along with its other safety checks. + tmplVal, err := filesystem.File(baseDir, cty.StringVal(fn)) + if err != nil { + return nil, err + } + + expr, diags := hclsyntax.ParseTemplate([]byte(tmplVal.AsString()), fn, hcl.Pos{Line: 1, Column: 1}) + if diags.HasErrors() { + return nil, diags + } + + return expr, nil + } + + renderTmpl := func(expr hcl.Expression, varsVal cty.Value) (cty.Value, error) { + if varsTy := varsVal.Type(); !(varsTy.IsMapType() || varsTy.IsObjectType()) { + return cty.DynamicVal, function.NewArgErrorf(1, "invalid vars value: must be a map") // or an object, but we don't strongly distinguish these most of the time + } + + ctx := &hcl.EvalContext{ + Variables: varsVal.AsValueMap(), + } + + // We require all of the variables to be valid HCL identifiers, because + // otherwise there would be no way to refer to them in the template + // anyway. Rejecting this here gives better feedback to the user + // than a syntax error somewhere in the template itself. + for n := range ctx.Variables { + if !hclsyntax.ValidIdentifier(n) { + // This error message intentionally doesn't describe _all_ of + // the different permutations that are technically valid as an + // HCL identifier, but rather focuses on what we might + // consider to be an "idiomatic" variable name. + return cty.DynamicVal, function.NewArgErrorf(1, "invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n) + } + } + + // We'll pre-check references in the template here so we can give a + // more specialized error message than HCL would by default, so it's + // clearer that this problem is coming from a templatefile call. + for _, traversal := range expr.Variables() { + root := traversal.RootName() + if _, ok := ctx.Variables[root]; !ok { + return cty.DynamicVal, function.NewArgErrorf(1, "vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange()) + } + } + + givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems + funcs := make(map[string]function.Function, len(givenFuncs)) + for name, fn := range givenFuncs { + if name == "templatefile" { + // We stub this one out to prevent recursive calls. + funcs[name] = function.New(&function.Spec{ + Params: params, + Type: func(args []cty.Value) (cty.Type, error) { + return cty.NilType, fmt.Errorf("cannot recursively call templatefile from inside templatefile call") + }, + }) + continue + } + funcs[name] = fn + } + ctx.Functions = funcs + + val, diags := expr.Value(ctx) + if diags.HasErrors() { + return cty.DynamicVal, diags + } + return val, nil + } + + return function.New(&function.Spec{ + Params: params, + Type: func(args []cty.Value) (cty.Type, error) { + if !(args[0].IsKnown() && args[1].IsKnown()) { + return cty.DynamicPseudoType, nil + } + + // We'll render our template now to see what result type it + // produces. A template consisting only of a single interpolation + // can potentially return any type. + expr, err := loadTmpl(args[0].AsString()) + if err != nil { + return cty.DynamicPseudoType, err + } + + // This is safe even if args[1] contains unknowns because the HCL + // template renderer itself knows how to short-circuit those. + val, err := renderTmpl(expr, args[1]) + return val.Type(), err + }, + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + expr, err := loadTmpl(args[0].AsString()) + if err != nil { + return cty.DynamicVal, err + } + return renderTmpl(expr, args[1]) + }, + }) + +} diff --git a/hcl2template/function/templatefile_test.go b/hcl2template/function/templatefile_test.go new file mode 100644 index 000000000..658d2f368 --- /dev/null +++ b/hcl2template/function/templatefile_test.go @@ -0,0 +1,151 @@ +package function + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/hashicorp/go-cty-funcs/filesystem" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" + "github.com/zclconf/go-cty/cty/function/stdlib" +) + +func TestTemplateFile(t *testing.T) { + tests := []struct { + Path cty.Value + Vars cty.Value + Want cty.Value + Err string + }{ + { + cty.StringVal("testdata/hello.txt"), + cty.EmptyObjectVal, + cty.StringVal("Hello World"), + ``, + }, + { + cty.StringVal("testdata/icon.png"), + cty.EmptyObjectVal, + cty.NilVal, + `contents of testdata/icon.png are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead`, + }, + { + cty.StringVal("testdata/missing"), + cty.EmptyObjectVal, + cty.NilVal, + `no file exists at ` + filepath.Clean("testdata/missing"), + }, + { + cty.StringVal("testdata/hello.tmpl"), + cty.MapVal(map[string]cty.Value{ + "name": cty.StringVal("Jodie"), + }), + cty.StringVal("Hello, Jodie!"), + ``, + }, + { + cty.StringVal("testdata/hello.tmpl"), + cty.MapVal(map[string]cty.Value{ + "name!": cty.StringVal("Jodie"), + }), + cty.NilVal, + `invalid template variable name "name!": must start with a letter, followed by zero or more letters, digits, and underscores`, + }, + { + cty.StringVal("testdata/hello.tmpl"), + cty.ObjectVal(map[string]cty.Value{ + "name": cty.StringVal("Jimbo"), + }), + cty.StringVal("Hello, Jimbo!"), + ``, + }, + { + cty.StringVal("testdata/hello.tmpl"), + cty.EmptyObjectVal, + cty.NilVal, + `vars map does not contain key "name", referenced at testdata/hello.tmpl:1,10-14`, + }, + { + cty.StringVal("testdata/func.tmpl"), + cty.ObjectVal(map[string]cty.Value{ + "list": cty.ListVal([]cty.Value{ + cty.StringVal("a"), + cty.StringVal("b"), + cty.StringVal("c"), + }), + }), + cty.StringVal("The items are a, b, c"), + ``, + }, + { + cty.StringVal("testdata/recursive.tmpl"), + cty.MapValEmpty(cty.String), + cty.NilVal, + `testdata/recursive.tmpl:1,3-16: Error in function call; Call to function "templatefile" failed: cannot recursively call templatefile from inside templatefile call.`, + }, + { + cty.StringVal("testdata/list.tmpl"), + cty.ObjectVal(map[string]cty.Value{ + "list": cty.ListVal([]cty.Value{ + cty.StringVal("a"), + cty.StringVal("b"), + cty.StringVal("c"), + }), + }), + cty.StringVal("- a\n- b\n- c\n"), + ``, + }, + { + cty.StringVal("testdata/list.tmpl"), + cty.ObjectVal(map[string]cty.Value{ + "list": cty.True, + }), + cty.NilVal, + `testdata/list.tmpl:1,13-17: Iteration over non-iterable value; A value of type bool cannot be used as the collection in a 'for' expression.`, + }, + { + cty.StringVal("testdata/bare.tmpl"), + cty.ObjectVal(map[string]cty.Value{ + "val": cty.True, + }), + cty.True, // since this template contains only an interpolation, its true value shines through + ``, + }, + } + + templateFileFn := MakeTemplateFileFunc(".", func() map[string]function.Function { + return map[string]function.Function{ + "join": stdlib.JoinFunc, + "templatefile": filesystem.MakeFileFunc(".", false), // just a placeholder, since templatefile itself overrides this + } + }) + + for _, test := range tests { + t.Run(fmt.Sprintf("TemplateFile(%#v, %#v)", test.Path, test.Vars), func(t *testing.T) { + got, err := templateFileFn.Call([]cty.Value{test.Path, test.Vars}) + + if argErr, ok := err.(function.ArgError); ok { + if argErr.Index < 0 || argErr.Index > 1 { + t.Errorf("ArgError index %d is out of range for templatefile (must be 0 or 1)", argErr.Index) + } + } + + if test.Err != "" { + if err == nil { + t.Fatal("succeeded; want error") + } + if got, want := err.Error(), test.Err; got != want { + t.Errorf("wrong error\ngot: %s\nwant: %s", got, want) + } + return + } else if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + if !got.RawEquals(test.Want) { + t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, test.Want) + } + }) + } +} diff --git a/hcl2template/function/testdata/bare.tmpl b/hcl2template/function/testdata/bare.tmpl new file mode 100644 index 000000000..da7cbab0e --- /dev/null +++ b/hcl2template/function/testdata/bare.tmpl @@ -0,0 +1 @@ +${val} \ No newline at end of file diff --git a/hcl2template/function/testdata/func.tmpl b/hcl2template/function/testdata/func.tmpl new file mode 100644 index 000000000..33a240000 --- /dev/null +++ b/hcl2template/function/testdata/func.tmpl @@ -0,0 +1 @@ +The items are ${join(", ", list)} \ No newline at end of file diff --git a/hcl2template/function/testdata/hello.tmpl b/hcl2template/function/testdata/hello.tmpl new file mode 100644 index 000000000..f112ef899 --- /dev/null +++ b/hcl2template/function/testdata/hello.tmpl @@ -0,0 +1 @@ +Hello, ${name}! \ No newline at end of file diff --git a/hcl2template/function/testdata/hello.txt b/hcl2template/function/testdata/hello.txt new file mode 100644 index 000000000..5e1c309da --- /dev/null +++ b/hcl2template/function/testdata/hello.txt @@ -0,0 +1 @@ +Hello World \ No newline at end of file diff --git a/hcl2template/function/testdata/icon.png b/hcl2template/function/testdata/icon.png new file mode 100644 index 000000000..a474f146f Binary files /dev/null and b/hcl2template/function/testdata/icon.png differ diff --git a/hcl2template/function/testdata/list.tmpl b/hcl2template/function/testdata/list.tmpl new file mode 100644 index 000000000..da8f4749e --- /dev/null +++ b/hcl2template/function/testdata/list.tmpl @@ -0,0 +1,3 @@ +%{ for x in list ~} +- ${x} +%{ endfor ~} diff --git a/hcl2template/function/testdata/recursive.tmpl b/hcl2template/function/testdata/recursive.tmpl new file mode 100644 index 000000000..f121b604e --- /dev/null +++ b/hcl2template/function/testdata/recursive.tmpl @@ -0,0 +1 @@ +${templatefile("recursive.tmpl", {})} \ No newline at end of file diff --git a/hcl2template/functions.go b/hcl2template/functions.go index fdd900ee4..3ac79cfec 100644 --- a/hcl2template/functions.go +++ b/hcl2template/functions.go @@ -118,6 +118,12 @@ func Functions(basedir string) map[string]function.Function { "zipmap": stdlib.ZipmapFunc, } + funcs["templatefile"] = pkrfunction.MakeTemplateFileFunc(basedir, func() map[string]function.Function { + // The templatefile function prevents recursive calls to itself + // by copying this map and overwriting the "templatefile" entry. + return funcs + }) + return funcs } diff --git a/website/content/docs/templates/hcl_templates/functions/file/fileexists.mdx b/website/content/docs/templates/hcl_templates/functions/file/fileexists.mdx index 8acb0565a..9d7ae7207 100644 --- a/website/content/docs/templates/hcl_templates/functions/file/fileexists.mdx +++ b/website/content/docs/templates/hcl_templates/functions/file/fileexists.mdx @@ -32,4 +32,5 @@ fileexists("custom-section.sh") ? file("custom-section.sh") : local.default_cont ## Related Functions -- [`file`](/docs/templates/hcl_templates/functions/file) reads the contents of a file at a given path +- [`file`](/docs/templates/hcl_templates/functions/file/file) reads the contents + of a file at a given path. diff --git a/website/content/docs/templates/hcl_templates/functions/file/templatefile.mdx b/website/content/docs/templates/hcl_templates/functions/file/templatefile.mdx new file mode 100644 index 000000000..7d07133bd --- /dev/null +++ b/website/content/docs/templates/hcl_templates/functions/file/templatefile.mdx @@ -0,0 +1,133 @@ +--- +page_title: templatefile - Functions - Configuration Language +sidebar_title: templatefile +description: |- + The templatefile function reads the file at the given path and renders its + content as a template using a supplied set of template variables. +--- + +# `templatefile` Function + +-> *Recommendation:* we recommend using the `.pkrtpl.hcl` file extension when +using the `templatefile` function. Template files *are* hcl treated as files but +also templates and therefore have slightly different set of features +than the ones offered in a `.pkr.hcl` Packer template. While you are not +required to use this extension, doing so will enable syntax highlighters to +properly understand your file. + +`templatefile` reads the file at the given path and renders its content as a +template using a supplied set of template variables. + +```hcl +templatefile(path, vars) +``` + +The template syntax is the same as for string templates in the main HCL2 +language, including interpolation sequences delimited with `${ ... }`. This +function just allows longer template sequences to be factored out into a +separate file for readability. + +The "vars" argument must be a map. Within the template file, each of the keys in +the map is available as a variable for interpolation. The template may also use +any other function available in Packer, except that recursive calls to +templatefile are not permitted. Variable names must each start with a letter, +followed by zero or more letters, digits, or underscores. + +Strings in HCL2 are sequences of Unicode characters, so this function will +interpret the file contents as UTF-8 encoded text and return the resulting +Unicode characters. If the file contains invalid UTF-8 sequences then this +function will produce an error. + +This function can be used only with files that already exist on disk at the +beginning of a run. + +## Examples + +### Lists + +Given a template file backends.tpl with the following content: + +```hcl +%{ for addr in ip_addrs ~} +backend ${addr}:${port} +%{ endfor ~} +``` +The templatefile function renders the template: + +```shell-session +> templatefile("${path.root}/backends.tmpl", { port = 8080, ip_addrs = ["10.0.0.1", "10.0.0.2"] }) +backend 10.0.0.1:8080 +backend 10.0.0.2:8080 +``` + +### Maps + +Given a template file config.tmpl with the following content: +```hcl +%{ for config_key, config_value in config } +set ${config_key} = ${config_value} +%{ endfor ~} +``` +The templatefile function renders the template: +```shell-session +> templatefile( + "${path.root}/config.tmpl", + { + config = { + "x" = "y" + "foo" = "bar" + "key" = "value" + } + } + ) +set foo = bar +set key = value +set x = y +``` + +### Generating JSON or YAML from a template + +If the string you want to generate will be in JSON or YAML syntax, it's often +tricky and tedious to write a template that will generate valid JSON or YAML +that will be interpreted correctly when using lots of individual interpolation +sequences and directives. + +Instead, you can write a template that consists only of a single interpolated +call to either jsonencode or yamlencode, specifying the value to encode using +normal expression syntax as in the following examples: + +```hcl +${jsonencode({ + "backends": [for addr in ip_addrs : "${addr}:${port}"], +})} +${yamlencode({ + "backends": [for addr in ip_addrs : "${addr}:${port}"], +})} +``` + +Given the same input as the `backends.tmpl` example in the previous section, +this will produce a valid JSON or YAML representation of the given data +structure, without the need to manually handle escaping or delimiters. In the +latest examples above, the repetition based on elements of ip_addrs is achieved +by using a for expression rather than by using template directives. + +``` +{"backends":["10.0.0.1:8080","10.0.0.2:8080"]} +``` +If the resulting template is small, you can choose instead to write jsonencode or yamlencode calls inline in your main configuration files, and avoid creating separate template files at all: + +locals { + backend_config_json = jsonencode({ + "backends": [for addr in ip_addrs : "${addr}:${port}"], + }) +} +For more information, see the main documentation for jsonencode and yamlencode. + +ยป + +## Related Functions + +- [`file`](/docs/templates/hcl_templates/functions/file/file) reads the contents + of a file at a given path. +- [`fileexists`](/docs/templates/hcl_templates/functions/file/fileexists) + determines whether a file exists at a given path. diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index f4a652250..ada4ff3c0 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -449,6 +449,10 @@ { "title": "pathexpand", "path": "templates/hcl_templates/functions/file/pathexpand" + }, + { + "title": "templatefile", + "path": "templates/hcl_templates/functions/file/templatefile" } ] }, diff --git a/website/data/docs-navigation.js b/website/data/docs-navigation.js index a65fd5c5a..118101f7c 100644 --- a/website/data/docs-navigation.js +++ b/website/data/docs-navigation.js @@ -137,6 +137,7 @@ export default [ 'fileexists', 'fileset', 'pathexpand', + 'templatefile', ], }, {