Allow to have `dynamic` blocks in a `build` block + tests (#10825)
This : * allows to have a `build.dynamic` block * add tests * makes sure to show a correct message when a source was not found * display only name of source (instead of a weird map printout) * use a "Did you mean %q" feature where possible Because dynamic blocks need all variables to be evaluated and available, I moved parsing of everything that is not a variable to "after" variables are extrapolated. Meaning that dynamic block get expanded in the `init` phase and then only we start interpreting HCL2 content. After #10819 fix #10657
This commit is contained in:
parent
a588808270
commit
77a29fc2f8
|
@ -383,6 +383,19 @@ func TestBuild(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "hcl - dynamic source blocks in a build block",
|
||||||
|
args: []string{
|
||||||
|
testFixture("hcl", "dynamic", "build.pkr.hcl"),
|
||||||
|
},
|
||||||
|
fileCheck: fileCheck{
|
||||||
|
expectedContent: map[string]string{
|
||||||
|
"dummy.txt": "layers/base/main/files",
|
||||||
|
"postgres/13.txt": "layers/base/main/files\nlayers/base/init/files\nlayers/postgres/files",
|
||||||
|
},
|
||||||
|
expected: []string{"dummy-fooo.txt", "dummy-baar.txt", "postgres/13-fooo.txt", "postgres/13-baar.txt"},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tc {
|
for _, tt := range tc {
|
||||||
|
|
|
@ -0,0 +1,47 @@
|
||||||
|
|
||||||
|
source "file" "base" {
|
||||||
|
}
|
||||||
|
|
||||||
|
variables {
|
||||||
|
images = {
|
||||||
|
dummy = {
|
||||||
|
image = "dummy"
|
||||||
|
layers = ["base/main"]
|
||||||
|
}
|
||||||
|
postgres = {
|
||||||
|
image = "postgres/13"
|
||||||
|
layers = ["base/main", "base/init", "postgres"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
locals {
|
||||||
|
files = {
|
||||||
|
foo = {
|
||||||
|
destination = "fooo"
|
||||||
|
}
|
||||||
|
bar = {
|
||||||
|
destination = "baar"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
build {
|
||||||
|
dynamic "source" {
|
||||||
|
for_each = var.images
|
||||||
|
labels = ["file.base"]
|
||||||
|
content {
|
||||||
|
name = source.key
|
||||||
|
target = "${source.value.image}.txt"
|
||||||
|
content = join("\n", formatlist("layers/%s/files", var.images[source.key].layers))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dynamic "provisioner" {
|
||||||
|
for_each = local.files
|
||||||
|
labels = ["shell-local"]
|
||||||
|
content {
|
||||||
|
inline = ["echo '' > ${var.images[source.name].image}-${provisioner.value.destination}.txt"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -76,12 +76,15 @@ const (
|
||||||
|
|
||||||
// Parse will Parse all HCL files in filename. Path can be a folder or a file.
|
// Parse will Parse all HCL files in filename. Path can be a folder or a file.
|
||||||
//
|
//
|
||||||
// Parse will first Parse variables and then the rest; so that interpolation
|
// Parse will first Parse packer and variables blocks, omitting the rest, which
|
||||||
// can happen.
|
// can be expanded with dynamic blocks. We need to evaluate all variables for
|
||||||
|
// that, so that data sources can expand dynamic blocks too.
|
||||||
//
|
//
|
||||||
// Parse returns a PackerConfig that contains configuration layout of a packer
|
// Parse returns a PackerConfig that contains configuration layout of a packer
|
||||||
// build; sources(builders)/provisioners/posts-processors will not be started
|
// build; sources(builders)/provisioners/posts-processors will not be started
|
||||||
// and their contents wont be verified; Most syntax errors will cause an error.
|
// and their contents wont be verified; Most syntax errors will cause an error,
|
||||||
|
// init should be called next to expand dynamic blocks and verify that used
|
||||||
|
// things do exist.
|
||||||
func (p *Parser) Parse(filename string, varFiles []string, argVars map[string]string) (*PackerConfig, hcl.Diagnostics) {
|
func (p *Parser) Parse(filename string, varFiles []string, argVars map[string]string) (*PackerConfig, hcl.Diagnostics) {
|
||||||
var files []*hcl.File
|
var files []*hcl.File
|
||||||
var diags hcl.Diagnostics
|
var diags hcl.Diagnostics
|
||||||
|
@ -235,10 +238,6 @@ func (p *Parser) Parse(filename string, varFiles []string, argVars map[string]st
|
||||||
diags = append(diags, cfg.collectInputVariableValues(os.Environ(), varFiles, argVars)...)
|
diags = append(diags, cfg.collectInputVariableValues(os.Environ(), varFiles, argVars)...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// parse the actual content // rest
|
|
||||||
for _, file := range cfg.files {
|
|
||||||
diags = append(diags, cfg.parser.parseConfig(file, cfg)...)
|
|
||||||
}
|
|
||||||
return cfg, diags
|
return cfg, diags
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -321,6 +320,11 @@ func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnosti
|
||||||
filterVarsFromLogs(cfg.InputVariables)
|
filterVarsFromLogs(cfg.InputVariables)
|
||||||
filterVarsFromLogs(cfg.LocalVariables)
|
filterVarsFromLogs(cfg.LocalVariables)
|
||||||
|
|
||||||
|
// parse the actual content // rest
|
||||||
|
for _, file := range cfg.files {
|
||||||
|
diags = append(diags, cfg.parser.parseConfig(file, cfg)...)
|
||||||
|
}
|
||||||
|
|
||||||
diags = append(diags, cfg.initializeBlocks()...)
|
diags = append(diags, cfg.initializeBlocks()...)
|
||||||
|
|
||||||
return diags
|
return diags
|
||||||
|
@ -332,6 +336,7 @@ func (p *Parser) parseConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics {
|
||||||
var diags hcl.Diagnostics
|
var diags hcl.Diagnostics
|
||||||
|
|
||||||
body := f.Body
|
body := f.Body
|
||||||
|
body = dynblock.Expand(body, cfg.EvalContext(DatasourceContext, nil))
|
||||||
content, moreDiags := body.Content(configSchema)
|
content, moreDiags := body.Content(configSchema)
|
||||||
diags = append(diags, moreDiags...)
|
diags = append(diags, moreDiags...)
|
||||||
|
|
||||||
|
@ -380,7 +385,7 @@ func (p *Parser) parseConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics {
|
||||||
func (p *Parser) decodeDatasources(file *hcl.File, cfg *PackerConfig) hcl.Diagnostics {
|
func (p *Parser) decodeDatasources(file *hcl.File, cfg *PackerConfig) hcl.Diagnostics {
|
||||||
var diags hcl.Diagnostics
|
var diags hcl.Diagnostics
|
||||||
|
|
||||||
body := dynblock.Expand(file.Body, cfg.EvalContext(DatasourceContext, nil))
|
body := file.Body
|
||||||
content, moreDiags := body.Content(configSchema)
|
content, moreDiags := body.Content(configSchema)
|
||||||
diags = append(diags, moreDiags...)
|
diags = append(diags, moreDiags...)
|
||||||
|
|
||||||
|
|
|
@ -7,7 +7,7 @@ import (
|
||||||
"runtime"
|
"runtime"
|
||||||
|
|
||||||
"github.com/hashicorp/hcl/v2"
|
"github.com/hashicorp/hcl/v2"
|
||||||
"github.com/hashicorp/hcl/v2/ext/dynblock"
|
"github.com/hashicorp/packer-plugin-sdk/didyoumean"
|
||||||
pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin"
|
pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin"
|
||||||
plugingetter "github.com/hashicorp/packer/packer/plugin-getter"
|
plugingetter "github.com/hashicorp/packer/packer/plugin-getter"
|
||||||
)
|
)
|
||||||
|
@ -109,7 +109,7 @@ func (cfg *PackerConfig) detectPluginBinaries() hcl.Diagnostics {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics {
|
func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics {
|
||||||
// verify that all used plugins do exist and expand dynamic bodies
|
// verify that all used plugins do exist
|
||||||
var diags hcl.Diagnostics
|
var diags hcl.Diagnostics
|
||||||
|
|
||||||
for _, build := range cfg.Builds {
|
for _, build := range cfg.Builds {
|
||||||
|
@ -129,12 +129,16 @@ func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics {
|
||||||
|
|
||||||
sourceDefinition, found := cfg.Sources[srcUsage.SourceRef]
|
sourceDefinition, found := cfg.Sources[srcUsage.SourceRef]
|
||||||
if !found {
|
if !found {
|
||||||
|
availableSrcs := listAvailableSourceNames(cfg.Sources)
|
||||||
|
detail := fmt.Sprintf("Known: %v", availableSrcs)
|
||||||
|
if sugg := didyoumean.NameSuggestion(srcUsage.SourceRef.String(), availableSrcs); sugg != "" {
|
||||||
|
detail = fmt.Sprintf("Did you mean to use %q?", sugg)
|
||||||
|
}
|
||||||
diags = append(diags, &hcl.Diagnostic{
|
diags = append(diags, &hcl.Diagnostic{
|
||||||
Summary: "Unknown " + sourceLabel + " " + srcUsage.String(),
|
Summary: "Unknown " + sourceLabel + " " + srcUsage.SourceRef.String(),
|
||||||
Subject: build.HCL2Ref.DefRange.Ptr(),
|
Subject: build.HCL2Ref.DefRange.Ptr(),
|
||||||
Severity: hcl.DiagError,
|
Severity: hcl.DiagError,
|
||||||
Detail: fmt.Sprintf("Known: %v", cfg.Sources),
|
Detail: detail,
|
||||||
// TODO: show known sources as a string slice here ^.
|
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -144,8 +148,6 @@ func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics {
|
||||||
// merge additions into source definition to get a new body.
|
// merge additions into source definition to get a new body.
|
||||||
body = hcl.MergeBodies([]hcl.Body{body, srcUsage.Body})
|
body = hcl.MergeBodies([]hcl.Body{body, srcUsage.Body})
|
||||||
}
|
}
|
||||||
// expand any dynamic block.
|
|
||||||
body = dynblock.Expand(body, cfg.EvalContext(BuildContext, nil))
|
|
||||||
|
|
||||||
srcUsage.Body = body
|
srcUsage.Body = body
|
||||||
}
|
}
|
||||||
|
@ -159,8 +161,6 @@ func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics {
|
||||||
Severity: hcl.DiagError,
|
Severity: hcl.DiagError,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
// Allow rest of the body to have dynamic blocks
|
|
||||||
provBlock.HCL2Ref.Rest = dynblock.Expand(provBlock.HCL2Ref.Rest, cfg.EvalContext(BuildContext, nil))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if build.ErrorCleanupProvisionerBlock != nil {
|
if build.ErrorCleanupProvisionerBlock != nil {
|
||||||
|
@ -172,8 +172,6 @@ func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics {
|
||||||
Severity: hcl.DiagError,
|
Severity: hcl.DiagError,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
// Allow rest of the body to have dynamic blocks
|
|
||||||
build.ErrorCleanupProvisionerBlock.HCL2Ref.Rest = dynblock.Expand(build.ErrorCleanupProvisionerBlock.HCL2Ref.Rest, cfg.EvalContext(BuildContext, nil))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ppList := range build.PostProcessorsLists {
|
for _, ppList := range build.PostProcessorsLists {
|
||||||
|
@ -186,8 +184,6 @@ func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics {
|
||||||
Severity: hcl.DiagError,
|
Severity: hcl.DiagError,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
// Allow the rest of the body to have dynamic blocks
|
|
||||||
ppBlock.HCL2Ref.Rest = dynblock.Expand(ppBlock.HCL2Ref.Rest, cfg.EvalContext(BuildContext, nil))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -81,6 +81,7 @@ type Builds []*BuildBlock
|
||||||
// load the references to the contents of the build block.
|
// load the references to the contents of the build block.
|
||||||
func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildBlock, hcl.Diagnostics) {
|
func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildBlock, hcl.Diagnostics) {
|
||||||
build := &BuildBlock{}
|
build := &BuildBlock{}
|
||||||
|
body := block.Body
|
||||||
|
|
||||||
var b struct {
|
var b struct {
|
||||||
Name string `hcl:"name,optional"`
|
Name string `hcl:"name,optional"`
|
||||||
|
@ -88,7 +89,7 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB
|
||||||
FromSources []string `hcl:"sources,optional"`
|
FromSources []string `hcl:"sources,optional"`
|
||||||
Config hcl.Body `hcl:",remain"`
|
Config hcl.Body `hcl:",remain"`
|
||||||
}
|
}
|
||||||
diags := gohcl.DecodeBody(block.Body, nil, &b)
|
diags := gohcl.DecodeBody(body, nil, &b)
|
||||||
if diags.HasErrors() {
|
if diags.HasErrors() {
|
||||||
return nil, diags
|
return nil, diags
|
||||||
}
|
}
|
||||||
|
@ -118,7 +119,8 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB
|
||||||
build.Sources = append(build.Sources, SourceUseBlock{SourceRef: ref})
|
build.Sources = append(build.Sources, SourceUseBlock{SourceRef: ref})
|
||||||
}
|
}
|
||||||
|
|
||||||
content, moreDiags := b.Config.Content(buildSchema)
|
body = b.Config
|
||||||
|
content, moreDiags := body.Content(buildSchema)
|
||||||
diags = append(diags, moreDiags...)
|
diags = append(diags, moreDiags...)
|
||||||
if diags.HasErrors() {
|
if diags.HasErrors() {
|
||||||
return nil, diags
|
return nil, diags
|
||||||
|
|
|
@ -538,65 +538,8 @@ func TestParser_no_init(t *testing.T) {
|
||||||
Type: cty.List(cty.String),
|
Type: cty.List(cty.String),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Sources: map[SourceRef]SourceBlock{
|
Sources: nil,
|
||||||
refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"},
|
Builds: nil,
|
||||||
refAWSV3MyImage: {Type: "amazon-v3-ebs", Name: "my-image"},
|
|
||||||
},
|
|
||||||
Builds: Builds{
|
|
||||||
&BuildBlock{
|
|
||||||
Sources: []SourceUseBlock{
|
|
||||||
{
|
|
||||||
SourceRef: refVBIsoUbuntu1204,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
SourceRef: refAWSV3MyImage,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
ProvisionerBlocks: []*ProvisionerBlock{
|
|
||||||
{
|
|
||||||
PType: "shell",
|
|
||||||
PName: "provisioner that does something",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PType: "file",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
PostProcessorsLists: [][]*PostProcessorBlock{
|
|
||||||
{
|
|
||||||
{
|
|
||||||
PType: "amazon-import",
|
|
||||||
PName: "something",
|
|
||||||
KeepInputArtifact: pTrue,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
{
|
|
||||||
PType: "amazon-import",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
{
|
|
||||||
PType: "amazon-import",
|
|
||||||
PName: "first-nested-post-processor",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PType: "amazon-import",
|
|
||||||
PName: "second-nested-post-processor",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
{
|
|
||||||
PType: "amazon-import",
|
|
||||||
PName: "third-nested-post-processor",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PType: "amazon-import",
|
|
||||||
PName: "fourth-nested-post-processor",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
false, false,
|
false, false,
|
||||||
[]packersdk.Build{},
|
[]packersdk.Build{},
|
||||||
|
|
|
@ -2,6 +2,7 @@ package hcl2template
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/hashicorp/hcl/v2"
|
"github.com/hashicorp/hcl/v2"
|
||||||
|
@ -170,3 +171,12 @@ var NoSource SourceRef
|
||||||
func (r SourceRef) String() string {
|
func (r SourceRef) String() string {
|
||||||
return fmt.Sprintf("%s.%s", r.Type, r.Name)
|
return fmt.Sprintf("%s.%s", r.Type, r.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func listAvailableSourceNames(srcs map[SourceRef]SourceBlock) []string {
|
||||||
|
res := make([]string, 0, len(srcs))
|
||||||
|
for k := range srcs {
|
||||||
|
res = append(res, k.String())
|
||||||
|
}
|
||||||
|
sort.Strings(res)
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue