Allow to override some source fields from the build section

Using `hcl.MergeBodies`

hcl2template/parser.go

@@ -2,6 +2,7 @@ package hcl2template
 
 import (
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 
@@ -191,7 +192,7 @@ func (p *Parser) decodeConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics {
 			}
 
 			ref := source.Ref()
-			if existing := cfg.Sources[ref]; existing != nil {
+			if existing, found := cfg.Sources[ref]; found {
 				diags = append(diags, &hcl.Diagnostic{
 					Severity: hcl.DiagError,
 					Summary:  "Duplicate " + sourceLabel + " block",
@@ -205,9 +206,10 @@ func (p *Parser) decodeConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics {
 			}
 
 			if cfg.Sources == nil {
-				cfg.Sources = map[SourceRef]*SourceBlock{}
+				cfg.Sources = map[SourceRef]SourceBlock{}
 			}
 			cfg.Sources[ref] = source
+			log.Printf("adding %s source to Available sources", ref)
 
 		case buildLabel:
 			build, moreDiags := p.decodeBuildConfig(block)

hcl2template/testdata/complete/build.pkr.hcl

@@ -5,6 +5,10 @@ build {
         "source.virtualbox-iso.ubuntu-1204",
     ]
 
+    source "source.amazon-ebs.ubuntu-1604" {
+        string = "setting from build section"
+    }
+
     provisioner "shell" {
         name     = "provisioner that does something"
         not_squashed = var.foo

hcl2template/testdata/complete/sources.pkr.hcl

@@ -1,3 +1,9 @@
+
+source "amazon-ebs" "ubuntu-1604" {
+    int = 42
+}
+
+
 source "virtualbox-iso" "ubuntu-1204" {
     string   = "string"
     int      = 42
@@ -85,4 +91,4 @@ source "virtualbox-iso" "ubuntu-1204" {
             ["c","d"]
         ]
     }
-}
+}

hcl2template/types.build.go

@@ -19,6 +19,7 @@ const (
 var buildSchema = &hcl.BodySchema{
 	Blocks: []hcl.BlockHeaderSchema{
 		{Type: buildFromLabel, LabelNames: []string{"type"}},
+		{Type: sourceLabel, LabelNames: []string{"reference"}},
 		{Type: buildProvisionerLabel, LabelNames: []string{"type"}},
 		{Type: buildPostProcessorLabel, LabelNames: []string{"type"}},
 	},
@@ -92,6 +93,13 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block) (*BuildBlock, hcl.Diagnosti
 	}
 	for _, block := range content.Blocks {
 		switch block.Type {
+		case sourceLabel:
+			ref, moreDiags := p.decodeBuildSource(block)
+			diags = append(diags, moreDiags...)
+			if moreDiags.HasErrors() {
+				continue
+			}
+			build.Sources = append(build.Sources, ref)
 		case buildProvisionerLabel:
 			p, moreDiags := p.decodeProvisioner(block)
 			diags = append(diags, moreDiags...)

hcl2template/types.build.post-processor.go

@@ -49,7 +49,7 @@ func (p *Parser) decodePostProcessor(block *hcl.Block) (*PostProcessorBlock, hcl
 	return postProcessor, diags
 }
 
-func (cfg *PackerConfig) startPostProcessor(source *SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.PostProcessor, hcl.Diagnostics) {
+func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.PostProcessor, hcl.Diagnostics) {
 	// ProvisionerBlock represents a detected but unparsed provisioner
 	var diags hcl.Diagnostics
 

hcl2template/types.build.provisioners.go

@@ -77,7 +77,7 @@ func (p *Parser) decodeProvisioner(block *hcl.Block) (*ProvisionerBlock, hcl.Dia
 	return provisioner, diags
 }
 
-func (cfg *PackerConfig) startProvisioner(source *SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.Provisioner, hcl.Diagnostics) {
+func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.Provisioner, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
 
 	provisioner, err := cfg.provisionersSchemas.Start(pb.PType)

hcl2template/types.packer_config.go

@@ -18,7 +18,7 @@ type PackerConfig struct {
 	Basedir string
 
 	// Available Source blocks
-	Sources map[SourceRef]*SourceBlock
+	Sources map[SourceRef]SourceBlock
 
 	// InputVariables and LocalVariables are the list of defined input and
 	// local variables. They are of the same type but are not used in the same
@@ -194,7 +194,7 @@ func (c *PackerConfig) evaluateLocalVariable(local *Local) hcl.Diagnostics {
 
 // getCoreBuildProvisioners takes a list of provisioner block, starts according
 // provisioners and sends parsed HCL2 over to it.
-func (cfg *PackerConfig) getCoreBuildProvisioners(source *SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) {
+func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
 	res := []packer.CoreBuildProvisioner{}
 	for _, pb := range blocks {
@@ -234,7 +234,7 @@ func (cfg *PackerConfig) getCoreBuildProvisioners(source *SourceBlock, blocks []
 
 // getCoreBuildProvisioners takes a list of post processor block, starts
 // according provisioners and sends parsed HCL2 over to it.
-func (cfg *PackerConfig) getCoreBuildPostProcessors(source *SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) {
+func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
 	res := []packer.CoreBuildPostProcessor{}
 	for _, ppb := range blocks {
@@ -262,15 +262,17 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build
 
 	for _, build := range cfg.Builds {
 		for _, from := range build.Sources {
-			src, found := cfg.Sources[from]
+			src, found := cfg.Sources[from.Ref()]
 			if !found {
 				diags = append(diags, &hcl.Diagnostic{
 					Summary:  "Unknown " + sourceLabel + " " + from.String(),
 					Subject:  build.HCL2Ref.DefRange.Ptr(),
 					Severity: hcl.DiagError,
+					Detail:   fmt.Sprintf("Known: %v", cfg.Sources),
 				})
 				continue
 			}
+			src.addition = from.addition
 
 			// Apply the -only and -except command-line options to exclude matching builds.
 			buildName := fmt.Sprintf("%s.%s", src.Type, src.Name)

hcl2template/types.packer_config_test.go

@@ -1,10 +1,9 @@
 package hcl2template
 
 import (
-	"path/filepath"
 	"testing"
-	"time"
 
+	. "github.com/hashicorp/packer/hcl2template/internal"
 	"github.com/hashicorp/packer/packer"
 	"github.com/zclconf/go-cty/cty"
 )
@@ -72,12 +71,16 @@ func TestParser_complete(t *testing.T) {
 						}),
 					},
 				},
-				Sources: map[SourceRef]*SourceBlock{
-					refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"},
+				Sources: map[SourceRef]SourceBlock{
+					refVBIsoUbuntu1204:  {Type: "virtualbox-iso", Name: "ubuntu-1204"},
+					refAWSEBSUbuntu1204: {Type: "amazon-ebs", Name: "ubuntu-1604"},
 				},
 				Builds: Builds{
 					&BuildBlock{
-						Sources: []SourceRef{refVBIsoUbuntu1204},
+						Sources: []SourceRef{
+							refVBIsoUbuntu1204,
+							refAWSEBSUbuntu1204,
+						},
 						ProvisionerBlocks: []*ProvisionerBlock{
 							{
 								PType: "shell",
@@ -125,136 +128,40 @@ func TestParser_complete(t *testing.T) {
 						},
 					},
 				},
-			},
-			false,
-		},
-		{"dir with no config files",
-			defaultParser,
-			parseTestArgs{"testdata/empty", nil, nil},
-			nil,
-			true, true,
-			nil,
-			false,
-		},
-		{name: "inexistent dir",
-			parser:                 defaultParser,
-			args:                   parseTestArgs{"testdata/inexistent", nil, nil},
-			parseWantCfg:           nil,
-			parseWantDiags:         true,
-			parseWantDiagHasErrors: true,
-		},
-		{name: "folder named build.pkr.hcl with an unknown src",
-			parser: defaultParser,
-			args:   parseTestArgs{"testdata/build.pkr.hcl", nil, nil},
-			parseWantCfg: &PackerConfig{
-				Basedir: "testdata/build.pkr.hcl",
-				Builds: Builds{
-					&BuildBlock{
-						Sources: []SourceRef{refAWSEBSUbuntu1204, refVBIsoUbuntu1204},
-						ProvisionerBlocks: []*ProvisionerBlock{
-							{PType: "shell"},
-							{PType: "file"},
-						},
-						PostProcessors: []*PostProcessorBlock{
-							{PType: "amazon-import"},
-						},
-					},
-				},
-			},
-			parseWantDiags:         false,
-			parseWantDiagHasErrors: false,
-			getBuildsWantBuilds:    []packer.Build{},
-			getBuildsWantDiags:     true,
-		},
-		{name: "unknown block type",
-			parser: defaultParser,
-			args:   parseTestArgs{"testdata/unknown", nil, nil},
-			parseWantCfg: &PackerConfig{
-				Basedir: "testdata/unknown",
-			},
-			parseWantDiags:         true,
-			parseWantDiagHasErrors: true,
-		},
-		{"provisioner with wrappers pause_before and max_retriers",
-			defaultParser,
-			parseTestArgs{"testdata/build/provisioner_paused_before_retry.pkr.hcl", nil, nil},
-			&PackerConfig{
-				Basedir: filepath.Join("testdata", "build"),
-				Sources: map[SourceRef]*SourceBlock{
-					refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"},
-				},
-				Builds: Builds{
-					&BuildBlock{
-						Sources: []SourceRef{refVBIsoUbuntu1204},
-						ProvisionerBlocks: []*ProvisionerBlock{
-							{
-								PType:       "shell",
-								PauseBefore: time.Second * 10,
-								MaxRetries:  5,
-							},
-						},
-					},
-				},
-			},
-			false, false,
-			[]packer.Build{
 				&packer.CoreBuild{
-					Type:     "virtualbox-iso.ubuntu-1204",
+					Type:     "amazon-ebs.ubuntu-1604",
 					Prepared: true,
-					Builder:  emptyMockBuilder,
+					Builder: &MockBuilder{
+						Config: MockConfig{
+							NestedMockConfig: NestedMockConfig{
+								String: "setting from build section",
+								Int:    42,
+								Tags:   []MockTag{},
+							},
+							NestedSlice: []NestedMockConfig{},
+						},
+					},
 					Provisioners: []packer.CoreBuildProvisioner{
 						{
-							PType: "shell",
-							Provisioner: &packer.RetriedProvisioner{
-								MaxRetries: 5,
-								Provisioner: &packer.PausedProvisioner{
-									PauseBefore: time.Second * 10,
-									Provisioner: emptyMockProvisioner,
-								},
-							},
+							PType:       "shell",
+							PName:       "provisioner that does something",
+							Provisioner: basicMockProvisioner,
 						},
+						{PType: "file", Provisioner: basicMockProvisioner},
 					},
-					PostProcessors: [][]packer.CoreBuildPostProcessor{},
-				},
-			},
-			false,
-		},
-		{"provisioner with wrappers timeout",
-			defaultParser,
-			parseTestArgs{"testdata/build/provisioner_timeout.pkr.hcl", nil, nil},
-			&PackerConfig{
-				Basedir: filepath.Join("testdata", "build"),
-				Sources: map[SourceRef]*SourceBlock{
-					refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"},
-				},
-				Builds: Builds{
-					&BuildBlock{
-						Sources: []SourceRef{refVBIsoUbuntu1204},
-						ProvisionerBlocks: []*ProvisionerBlock{
-							{
-								PType:   "shell",
-								Timeout: time.Second * 10,
-							},
-						},
-					},
-				},
-			},
-			false, false,
-			[]packer.Build{
-				&packer.CoreBuild{
-					Type:     "virtualbox-iso.ubuntu-1204",
-					Prepared: true,
-					Builder:  emptyMockBuilder,
-					Provisioners: []packer.CoreBuildProvisioner{
+					PostProcessors: [][]packer.CoreBuildPostProcessor{
 						{
-							PType: "shell",
-							Provisioner: &packer.TimeoutProvisioner{
-								Timeout:     time.Second * 10,
-								Provisioner: emptyMockProvisioner,
+							{
+								PType:         "amazon-import",
+								PName:         "something",
+								PostProcessor: basicMockPostProcessor,
+							},
+							{
+								PType:         "amazon-import",
+								PostProcessor: basicMockPostProcessor,
 							},
 						},
 					},
-					PostProcessors: [][]packer.CoreBuildPostProcessor{},
 				},
 			},
 			false,

hcl2template/types.source.go

@@ -16,10 +16,24 @@ type SourceBlock struct {
 	Name string
 
 	block *hcl.Block
+
+	// addition will be merged into block to allow to override builder settings
+	// per build.source block.
+	addition hcl.Body
 }
 
-func (p *Parser) decodeSource(block *hcl.Block) (*SourceBlock, hcl.Diagnostics) {
-	source := &SourceBlock{
+// decodeBuildSource reads a used source block from a build:
+//  build {
+//   source "type.example" {}
+//  }
+func (p *Parser) decodeBuildSource(block *hcl.Block) (SourceRef, hcl.Diagnostics) {
+	ref := sourceRefFromString(block.Labels[0])
+	ref.addition = block.Body
+	return ref, nil
+}
+
+func (p *Parser) decodeSource(block *hcl.Block) (SourceBlock, hcl.Diagnostics) {
+	source := SourceBlock{
 		Type:  block.Labels[0],
 		Name:  block.Labels[1],
 		block: block,
@@ -33,13 +47,13 @@ func (p *Parser) decodeSource(block *hcl.Block) (*SourceBlock, hcl.Diagnostics)
 			Detail:   fmt.Sprintf("known builders: %v", p.BuilderSchemas.List()),
 			Severity: hcl.DiagError,
 		})
-		return nil, diags
+		return source, diags
 	}
 
 	return source, diags
 }
 
-func (cfg *PackerConfig) startBuilder(source *SourceBlock, ectx *hcl.EvalContext, opts packer.GetBuildsOptions) (packer.Builder, hcl.Diagnostics, []string) {
+func (cfg *PackerConfig) startBuilder(source SourceBlock, ectx *hcl.EvalContext, opts packer.GetBuildsOptions) (packer.Builder, hcl.Diagnostics, []string) {
 	var diags hcl.Diagnostics
 
 	builder, err := cfg.builderSchemas.Start(source.Type)
@@ -52,7 +66,12 @@ func (cfg *PackerConfig) startBuilder(source *SourceBlock, ectx *hcl.EvalContext
 		return builder, diags, nil
 	}
 
-	decoded, moreDiags := decodeHCL2Spec(source.block.Body, ectx, builder)
+	body := source.block.Body
+	if source.addition != nil {
+		body = hcl.MergeBodies([]hcl.Body{source.block.Body, source.addition})
+	}
+
+	decoded, moreDiags := decodeHCL2Spec(body, ectx, builder)
 	diags = append(diags, moreDiags...)
 	if moreDiags.HasErrors() {
 		return nil, diags, nil
@@ -92,6 +111,19 @@ func (source *SourceBlock) Ref() SourceRef {
 type SourceRef struct {
 	Type string
 	Name string
+
+	// The content of this body will be merged into a new block to allow to
+	// override builder settings per build section.
+	addition hcl.Body
+}
+
+// the 'addition' field makes of ref a different entry in the soruces map, so
+// Ref is here to make sure only one is returned.
+func (r *SourceRef) Ref() SourceRef {
+	return SourceRef{
+		Type: r.Type,
+		Name: r.Name,
+	}
 }
 
 // NoSource is the zero value of sourceRef, representing the absense of an

hcl2template/types.source_test.go

@@ -16,7 +16,7 @@ func TestParse_source(t *testing.T) {
 			parseTestArgs{"testdata/sources/basic.pkr.hcl", nil, nil},
 			&PackerConfig{
 				Basedir: filepath.Join("testdata", "sources"),
-				Sources: map[SourceRef]*SourceBlock{
+				Sources: map[SourceRef]SourceBlock{
 					{
 						Type: "virtualbox-iso",
 						Name: "ubuntu-1204",
@@ -65,7 +65,7 @@ func TestParse_source(t *testing.T) {
 			parseTestArgs{"testdata/sources/duplicate.pkr.hcl", nil, nil},
 			&PackerConfig{
 				Basedir: filepath.Join("testdata", "sources"),
-				Sources: map[SourceRef]*SourceBlock{
+				Sources: map[SourceRef]SourceBlock{
 					{
 						Type: "virtualbox-iso",
 						Name: "ubuntu-1204",

hcl2template/types.variables_test.go

@@ -162,15 +162,17 @@ func TestParse_variables(t *testing.T) {
 						Name: "foo",
 					},
 				},
-				Sources: map[SourceRef]*SourceBlock{
-					SourceRef{"null", "null-builder"}: &SourceBlock{
+				Sources: map[SourceRef]SourceBlock{
+					SourceRef{"null", "null-builder", nil}: SourceBlock{
 						Name: "null-builder",
 						Type: "null",
 					},
 				},
 				Builds: Builds{
 					&BuildBlock{
-						Sources: []SourceRef{SourceRef{"null", "null-builder"}},
+						Sources: []SourceRef{
+							{"null", "null-builder", nil},
+						},
 					},
 				},
 			},