From c8300b620a581f3c2fa9f505621f35d568f4a25a Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Thu, 12 Mar 2020 14:27:56 +0100 Subject: [PATCH] allow to use hcl files as var files in HCL mode fix #8781 --- command/build.go | 2 +- command/meta.go | 14 +++++++++++++- hcl2template/common_test.go | 3 ++- hcl2template/parser.go | 19 +++++++++++++++++-- hcl2template/types.build_test.go | 12 ++++++------ hcl2template/types.packer_config.go | 4 ++-- hcl2template/types.packer_config_test.go | 10 +++++----- hcl2template/types.source_test.go | 10 +++++----- hcl2template/types.variables_test.go | 18 +++++++++--------- helper/flag-kv/flag_strings.go | 16 ++++++++++++++++ 10 files changed, 76 insertions(+), 32 deletions(-) create mode 100644 helper/flag-kv/flag_strings.go diff --git a/command/build.go b/command/build.go index 2ea4fcc85..64a352678 100644 --- a/command/build.go +++ b/command/build.go @@ -104,7 +104,7 @@ func (c *BuildCommand) GetBuildsFromHCL(path string) ([]packer.Build, int) { PostProcessorsSchemas: c.CoreConfig.Components.PostProcessorStore, } - builds, diags := parser.Parse(path, c.flagVars) + builds, diags := parser.Parse(path, c.varFiles, c.flagVars) { // write HCL errors/diagnostics if any. b := bytes.NewBuffer(nil) diff --git a/command/meta.go b/command/meta.go index b92116428..8d28a1924 100644 --- a/command/meta.go +++ b/command/meta.go @@ -32,6 +32,7 @@ type Meta struct { Version string // These are set by command-line flags + varFiles []string flagVars map[string]string } @@ -41,6 +42,17 @@ func (m *Meta) Core(tpl *template.Template) (*packer.Core, error) { // Copy the config so we don't modify it config := *m.CoreConfig config.Template = tpl + + fj := &kvflag.FlagJSON{} + for _, file := range m.varFiles { + err := fj.Set(file) + if err != nil { + return nil, err + } + } + for k, v := range *fj { + m.flagVars[k] = v + } config.Variables = m.flagVars // Init the core @@ -117,7 +129,7 @@ func (m *Meta) FlagSet(n string, fs FlagSetFlags) *flag.FlagSet { // FlagSetVars tells us what variables to use if fs&FlagSetVars != 0 { f.Var((*kvflag.Flag)(&m.flagVars), "var", "") - f.Var((*kvflag.FlagJSON)(&m.flagVars), "var-file", "") + f.Var((*kvflag.StringSlice)(&m.varFiles), "var-file", "") } // Create an io.Writer that writes to our Ui properly for errors. diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index b0dfedb82..fd472cf3b 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -37,6 +37,7 @@ func getBasicParser() *Parser { type parseTestArgs struct { filename string vars map[string]string + varFiles []string } type parseTest struct { @@ -58,7 +59,7 @@ func testParse(t *testing.T, tests []parseTest) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotCfg, gotDiags := tt.parser.parse(tt.args.filename, tt.args.vars) + gotCfg, gotDiags := tt.parser.parse(tt.args.filename, tt.args.varFiles, tt.args.vars) if tt.parseWantDiags == (gotDiags == nil) { t.Fatalf("Parser.parse() unexpected %q diagnostics.", gotDiags) } diff --git a/hcl2template/parser.go b/hcl2template/parser.go index 3bf48294e..35061f932 100644 --- a/hcl2template/parser.go +++ b/hcl2template/parser.go @@ -52,7 +52,7 @@ const ( hcl2VarJsonFileExt = ".auto.pkrvars.json" ) -func (p *Parser) parse(filename string, vars map[string]string) (*PackerConfig, hcl.Diagnostics) { +func (p *Parser) parse(filename string, varFiles []string, argVars map[string]string) (*PackerConfig, hcl.Diagnostics) { var files []*hcl.File var diags hcl.Diagnostics @@ -60,6 +60,7 @@ func (p *Parser) parse(filename string, vars map[string]string) (*PackerConfig, // parse config files { hclFiles, jsonFiles, moreDiags := GetHCL2Files(filename, hcl2FileExt, hcl2JsonFileExt) + diags = append(diags, moreDiags...) if len(hclFiles)+len(jsonFiles) == 0 { diags = append(moreDiags, &hcl.Diagnostic{ Severity: hcl.DiagError, @@ -111,6 +112,20 @@ func (p *Parser) parse(filename string, vars map[string]string) (*PackerConfig, { hclVarFiles, jsonVarFiles, moreDiags := GetHCL2Files(filename, hcl2VarFileExt, hcl2VarJsonFileExt) diags = append(diags, moreDiags...) + for _, file := range varFiles { + switch filepath.Ext(file) { + case ".hcl": + hclVarFiles = append(hclVarFiles, file) + case ".json": + jsonVarFiles = append(jsonVarFiles, file) + default: + diags = append(moreDiags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Could not guess format of " + file, + Detail: "A var file must be suffixed with `.hcl` or `.json`.", + }) + } + } var varFiles []*hcl.File for _, filename := range hclVarFiles { f, moreDiags := p.ParseHCLFile(filename) @@ -123,7 +138,7 @@ func (p *Parser) parse(filename string, vars map[string]string) (*PackerConfig, varFiles = append(varFiles, f) } - diags = append(diags, cfg.collectInputVariableValues(os.Environ(), varFiles, vars)...) + diags = append(diags, cfg.collectInputVariableValues(os.Environ(), varFiles, argVars)...) } _, moreDiags := cfg.InputVariables.Values() diff --git a/hcl2template/types.build_test.go b/hcl2template/types.build_test.go index dba35bf3a..7d45d022d 100644 --- a/hcl2template/types.build_test.go +++ b/hcl2template/types.build_test.go @@ -13,7 +13,7 @@ func TestParse_build(t *testing.T) { tests := []parseTest{ {"basic build no src", defaultParser, - parseTestArgs{"testdata/build/basic.pkr.hcl", nil}, + parseTestArgs{"testdata/build/basic.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "build"), Builds: Builds{ @@ -47,7 +47,7 @@ func TestParse_build(t *testing.T) { }, {"untyped provisioner", defaultParser, - parseTestArgs{"testdata/build/provisioner_untyped.pkr.hcl", nil}, + parseTestArgs{"testdata/build/provisioner_untyped.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "build"), Builds: nil, @@ -58,7 +58,7 @@ func TestParse_build(t *testing.T) { }, {"inexistent provisioner", defaultParser, - parseTestArgs{"testdata/build/provisioner_inexistent.pkr.hcl", nil}, + parseTestArgs{"testdata/build/provisioner_inexistent.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "build"), Builds: nil, @@ -69,7 +69,7 @@ func TestParse_build(t *testing.T) { }, {"untyped post-processor", defaultParser, - parseTestArgs{"testdata/build/post-processor_untyped.pkr.hcl", nil}, + parseTestArgs{"testdata/build/post-processor_untyped.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "build"), Builds: nil, @@ -80,7 +80,7 @@ func TestParse_build(t *testing.T) { }, {"inexistent post-processor", defaultParser, - parseTestArgs{"testdata/build/post-processor_inexistent.pkr.hcl", nil}, + parseTestArgs{"testdata/build/post-processor_inexistent.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "build"), Builds: nil, @@ -91,7 +91,7 @@ func TestParse_build(t *testing.T) { }, {"invalid source", defaultParser, - parseTestArgs{"testdata/build/invalid_source_reference.pkr.hcl", nil}, + parseTestArgs{"testdata/build/invalid_source_reference.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "build"), Builds: nil, diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 0d61e3069..3f52092d7 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -287,8 +287,8 @@ func (p *Parser) getBuilds(cfg *PackerConfig) ([]packer.Build, hcl.Diagnostics) // // Parse then return a slice of packer.Builds; which are what packer core uses // to run builds. -func (p *Parser) Parse(path string, vars map[string]string) ([]packer.Build, hcl.Diagnostics) { - cfg, diags := p.parse(path, vars) +func (p *Parser) Parse(path string, varFiles []string, argVars map[string]string) ([]packer.Build, hcl.Diagnostics) { + cfg, diags := p.parse(path, varFiles, argVars) if diags.HasErrors() { return nil, diags } diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index d84cef1fc..ae0b5030d 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -18,7 +18,7 @@ func TestParser_complete(t *testing.T) { tests := []parseTest{ {"working build", defaultParser, - parseTestArgs{"testdata/complete", nil}, + parseTestArgs{"testdata/complete", nil, nil}, &PackerConfig{ Basedir: "testdata/complete", InputVariables: Variables{ @@ -128,7 +128,7 @@ func TestParser_complete(t *testing.T) { }, {"dir with no config files", defaultParser, - parseTestArgs{"testdata/empty", nil}, + parseTestArgs{"testdata/empty", nil, nil}, nil, true, true, nil, @@ -136,14 +136,14 @@ func TestParser_complete(t *testing.T) { }, {name: "inexistent dir", parser: defaultParser, - args: parseTestArgs{"testdata/inexistent", nil}, + args: parseTestArgs{"testdata/inexistent", nil, nil}, parseWantCfg: nil, parseWantDiags: true, parseWantDiagHasErrors: true, }, {name: "folder named build.pkr.hcl with an unknown src", parser: defaultParser, - args: parseTestArgs{"testdata/build.pkr.hcl", nil}, + args: parseTestArgs{"testdata/build.pkr.hcl", nil, nil}, parseWantCfg: &PackerConfig{ Basedir: "testdata/build.pkr.hcl", Builds: Builds{ @@ -166,7 +166,7 @@ func TestParser_complete(t *testing.T) { }, {name: "unknown block type", parser: defaultParser, - args: parseTestArgs{"testdata/unknown", nil}, + args: parseTestArgs{"testdata/unknown", nil, nil}, parseWantCfg: &PackerConfig{ Basedir: "testdata/unknown", }, diff --git a/hcl2template/types.source_test.go b/hcl2template/types.source_test.go index f5c9d0855..423d20422 100644 --- a/hcl2template/types.source_test.go +++ b/hcl2template/types.source_test.go @@ -13,7 +13,7 @@ func TestParse_source(t *testing.T) { tests := []parseTest{ {"two basic sources", defaultParser, - parseTestArgs{"testdata/sources/basic.pkr.hcl", nil}, + parseTestArgs{"testdata/sources/basic.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "sources"), Sources: map[SourceRef]*SourceBlock{ @@ -32,7 +32,7 @@ func TestParse_source(t *testing.T) { }, {"untyped source", defaultParser, - parseTestArgs{"testdata/sources/untyped.pkr.hcl", nil}, + parseTestArgs{"testdata/sources/untyped.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "sources"), }, @@ -42,7 +42,7 @@ func TestParse_source(t *testing.T) { }, {"unnamed source", defaultParser, - parseTestArgs{"testdata/sources/unnamed.pkr.hcl", nil}, + parseTestArgs{"testdata/sources/unnamed.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "sources"), }, @@ -52,7 +52,7 @@ func TestParse_source(t *testing.T) { }, {"inexistent source", defaultParser, - parseTestArgs{"testdata/sources/inexistent.pkr.hcl", nil}, + parseTestArgs{"testdata/sources/inexistent.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "sources"), }, @@ -62,7 +62,7 @@ func TestParse_source(t *testing.T) { }, {"duplicate source", defaultParser, - parseTestArgs{"testdata/sources/duplicate.pkr.hcl", nil}, + parseTestArgs{"testdata/sources/duplicate.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "sources"), Sources: map[SourceRef]*SourceBlock{ diff --git a/hcl2template/types.variables_test.go b/hcl2template/types.variables_test.go index f6d22d219..e53ff02b6 100644 --- a/hcl2template/types.variables_test.go +++ b/hcl2template/types.variables_test.go @@ -20,7 +20,7 @@ func TestParse_variables(t *testing.T) { tests := []parseTest{ {"basic variables", defaultParser, - parseTestArgs{"testdata/variables/basic.pkr.hcl", nil}, + parseTestArgs{"testdata/variables/basic.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "variables"), InputVariables: Variables{ @@ -75,7 +75,7 @@ func TestParse_variables(t *testing.T) { }, {"duplicate variable", defaultParser, - parseTestArgs{"testdata/variables/duplicate_variable.pkr.hcl", nil}, + parseTestArgs{"testdata/variables/duplicate_variable.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "variables"), InputVariables: Variables{ @@ -90,7 +90,7 @@ func TestParse_variables(t *testing.T) { }, {"duplicate variable in variables", defaultParser, - parseTestArgs{"testdata/variables/duplicate_variables.pkr.hcl", nil}, + parseTestArgs{"testdata/variables/duplicate_variables.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "variables"), InputVariables: Variables{ @@ -105,7 +105,7 @@ func TestParse_variables(t *testing.T) { }, {"invalid default type", defaultParser, - parseTestArgs{"testdata/variables/invalid_default.pkr.hcl", nil}, + parseTestArgs{"testdata/variables/invalid_default.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "variables"), InputVariables: Variables{ @@ -121,7 +121,7 @@ func TestParse_variables(t *testing.T) { {"unknown key", defaultParser, - parseTestArgs{"testdata/variables/unknown_key.pkr.hcl", nil}, + parseTestArgs{"testdata/variables/unknown_key.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "variables"), InputVariables: Variables{ @@ -138,7 +138,7 @@ func TestParse_variables(t *testing.T) { {"unset used variable", defaultParser, - parseTestArgs{"testdata/variables/unset_used_string_variable.pkr.hcl", nil}, + parseTestArgs{"testdata/variables/unset_used_string_variable.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "variables"), InputVariables: Variables{ @@ -154,7 +154,7 @@ func TestParse_variables(t *testing.T) { {"unset unused variable", defaultParser, - parseTestArgs{"testdata/variables/unset_unused_string_variable.pkr.hcl", nil}, + parseTestArgs{"testdata/variables/unset_unused_string_variable.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "variables"), InputVariables: Variables{ @@ -189,7 +189,7 @@ func TestParse_variables(t *testing.T) { {"locals within another locals usage in different files", defaultParser, - parseTestArgs{"testdata/variables/complicated", nil}, + parseTestArgs{"testdata/variables/complicated", nil, nil}, &PackerConfig{ Basedir: "testdata/variables/complicated", InputVariables: Variables{ @@ -231,7 +231,7 @@ func TestParse_variables(t *testing.T) { }, {"recursive locals", defaultParser, - parseTestArgs{"testdata/variables/recursive_locals.pkr.hcl", nil}, + parseTestArgs{"testdata/variables/recursive_locals.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "variables"), LocalVariables: Variables{}, diff --git a/helper/flag-kv/flag_strings.go b/helper/flag-kv/flag_strings.go new file mode 100644 index 000000000..2d63af9fa --- /dev/null +++ b/helper/flag-kv/flag_strings.go @@ -0,0 +1,16 @@ +package kvflag + +import ( + "strings" +) + +type StringSlice []string + +func (s *StringSlice) String() string { + return strings.Join(*s, ", ") +} + +func (s *StringSlice) Set(value string) error { + *s = append(*s, value) + return nil +}