iSharkFly-Docs
/
packer-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HCL2: pass on builder type and name (#8956) 

* sets `packer_build_name` and `packer_builder_type` variables for builder provisioners and post-processors in HCL2
* allows to use the new `${source.type}` and `${source.name}` variables in HCL2
* fixes #8932 

Note that the common.PackerConfig is used everywhere and was not set for HCL2, this had some implications: 

For #8923 you can see the issue here:

dde74232f2/builder/lxd/config.go (L61-L63)

More random examples of where this could cause an issue :

0785c2f6fc/provisioner/ansible-local/provisioner.go (L380-L381)

b4efd13a4d/builder/amazon/ebs/builder.go (L232-L236)



* [All references to PackerConfig.PackerBuildName](https://sourcegraph.com/github.com/hashicorp/packer@ff6a039d5bb45e34ff761d9c52e8b98972288447/-/blob/common/packer_config.go#L7:2&tab=references)

* [All references to PackerConfig.PackerBuilderType](https://sourcegraph.com/github.com/hashicorp/packer@ff6a039d5bb45e34ff761d9c52e8b98972288447/-/blob/common/packer_config.go#L8:2&tab=references)
This commit is contained in:
Adrien Delorme 2020-04-09 11:14:37 +02:00 committed by GitHub
parent c358682411
commit 2af40c762b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 262 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
command/build_test.go
View File

@ -3,6 +3,7 @@ package command
import (
"bytes"
"fmt"
"io/ioutil"
"math"
"os"
"path/filepath"
@ -12,12 +13,14 @@ import (
"github.com/hashicorp/packer/builder/file"
"github.com/hashicorp/packer/builder/null"
"github.com/hashicorp/packer/packer"
"github.com/hashicorp/packer/post-processor/manifest"
shell_local_pp "github.com/hashicorp/packer/post-processor/shell-local"
filep "github.com/hashicorp/packer/provisioner/file"
"github.com/hashicorp/packer/provisioner/shell"
shell_local "github.com/hashicorp/packer/provisioner/shell-local"
)
func TestBuild_VarArgs(t *testing.T) {
func TestBuild(t *testing.T) {
tc := []struct {
name string
args []string
@ -25,7 +28,7 @@ func TestBuild_VarArgs(t *testing.T) {
fileCheck
}{
{
name: "json - json varfile sets an apple env var",
name: "var-args: json - json varfile sets an apple env var",
args: []string{
"-var-file=" + filepath.Join(testFixture("var-arg"), "apple.json"),
filepath.Join(testFixture("var-arg"), "fruit_builder.json"),
@ -43,7 +46,7 @@ func TestBuild_VarArgs(t *testing.T) {
fileCheck: fileCheck{expected: []string{"banana.txt"}},
},
{
name: "json - arg sets a pear env var",
name: "var-args: json - arg sets a pear env var",
args: []string{
"-var=fruit=pear",
filepath.Join(testFixture("var-arg"), "fruit_builder.json"),
@ -52,7 +55,7 @@ func TestBuild_VarArgs(t *testing.T) {
},
{
name: "json - inexistent var file errs",
name: "var-args: json - inexistent var file errs",
args: []string{
"-var-file=" + filepath.Join(testFixture("var-arg"), "potato.json"),
filepath.Join(testFixture("var-arg"), "fruit_builder.json"),
@ -62,7 +65,7 @@ func TestBuild_VarArgs(t *testing.T) {
},
{
name: "hcl - inexistent json var file errs",
name: "var-args: hcl - inexistent json var file errs",
args: []string{
"-var-file=" + filepath.Join(testFixture("var-arg"), "potato.json"),
testFixture("var-arg"),
@ -72,7 +75,7 @@ func TestBuild_VarArgs(t *testing.T) {
},
{
name: "hcl - inexistent hcl var file errs",
name: "var-args: hcl - inexistent hcl var file errs",
args: []string{
"-var-file=" + filepath.Join(testFixture("var-arg"), "potato.hcl"),
testFixture("var-arg"),
@ -82,7 +85,7 @@ func TestBuild_VarArgs(t *testing.T) {
},
{
name: "hcl - auto varfile sets a chocolate env var",
name: "var-args: hcl - auto varfile sets a chocolate env var",
args: []string{
testFixture("var-arg"),
},
@ -90,7 +93,7 @@ func TestBuild_VarArgs(t *testing.T) {
},
{
name: "hcl - hcl varfile sets a apple env var",
name: "var-args: hcl - hcl varfile sets a apple env var",
args: []string{
"-var-file=" + filepath.Join(testFixture("var-arg"), "apple.hcl"),
testFixture("var-arg"),
@ -99,7 +102,7 @@ func TestBuild_VarArgs(t *testing.T) {
},
{
name: "hcl - json varfile sets a apple env var",
name: "var-args: hcl - json varfile sets a apple env var",
args: []string{
"-var-file=" + filepath.Join(testFixture("var-arg"), "apple.json"),
testFixture("var-arg"),
@ -108,19 +111,108 @@ func TestBuild_VarArgs(t *testing.T) {
},
{
name: "hcl - arg sets a tomato env var",
name: "var-args: hcl - arg sets a tomato env var",
args: []string{
"-var=fruit=tomato",
testFixture("var-arg"),
},
fileCheck: fileCheck{expected: []string{"tomato.txt"}},
},
{
name: "build name: HCL",
args: []string{
"-parallel-builds=1", // to ensure order is kept
testFixture("build-name-and-type"),
},
fileCheck: fileCheck{
expectedContent: map[string]string{
"manifest.json": `{
"builds": [
{
"name": "test",
"builder_type": "null",
"files": null,
"artifact_id": "Null",
"packer_run_uuid": "",
"custom_data": null
},
{
"name": "potato",
"builder_type": "null",
"files": null,
"artifact_id": "Null",
"packer_run_uuid": "",
"custom_data": null
}
],
"last_run_uuid": ""
}`,
},
},
},
{
name: "build name: JSON except potato",
args: []string{
"-except=potato",
"-parallel-builds=1", // to ensure order is kept
filepath.Join(testFixture("build-name-and-type"), "all.json"),
},
fileCheck: fileCheck{
expected: []string{
"null.test.txt",
"null.potato.txt",
},
expectedContent: map[string]string{
"manifest.json": `{
"builds": [
{
"name": "test",
"builder_type": "null",
"files": null,
"artifact_id": "Null",
"packer_run_uuid": "",
"custom_data": null
}
],
"last_run_uuid": ""
}`,
},
},
},
{
name: "build name: JSON only potato",
args: []string{
"-only=potato",
"-parallel-builds=1", // to ensure order is kept
filepath.Join(testFixture("build-name-and-type"), "all.json"),
},
fileCheck: fileCheck{
expectedContent: map[string]string{
"manifest.json": `{
"builds": [
{
"name": "potato",
"builder_type": "null",
"files": null,
"artifact_id": "Null",
"packer_run_uuid": "",
"custom_data": null
}
],
"last_run_uuid": ""
}`,
},
},
},
}
for _, tt := range tc {
t.Run(tt.name, func(t *testing.T) {
defer tt.cleanup(t)
run(t, tt.args, tt.expectedCode)
defer cleanup()
tt.fileCheck.verify(t)
})
}
@ -340,10 +432,28 @@ func run(t *testing.T, args []string, expectedCode int) {
type fileCheck struct {
expected, notExpected []string
expectedContent map[string]string
}
func (fc fileCheck) cleanup(t *testing.T) {
for _, file := range fc.expectedFiles() {
t.Logf("removing %v", file)
if err := os.Remove(file); err != nil {
t.Errorf("failed to remove file %s: %v", file, err)
}
}
}
func (fc fileCheck) expectedFiles() []string {
expected := fc.expected
for file := range fc.expectedContent {
expected = append(expected, file)
}
return expected
}
func (fc fileCheck) verify(t *testing.T) {
for _, f := range fc.expected {
for _, f := range fc.expectedFiles() {
if !fileExists(f) {
t.Errorf("Expected to find %s", f)
}
@ -353,6 +463,15 @@ func (fc fileCheck) verify(t *testing.T) {
t.Errorf("Expected to not find %s", f)
}
}
for file, expectedContent := range fc.expectedContent {
content, err := ioutil.ReadFile(file)
if err != nil {
t.Fatalf("ioutil.ReadFile: %v", err)
}
if diff := cmp.Diff(expectedContent, string(content)); diff != "" {
t.Errorf("content of %s differs: %s", file, diff)
}
}
}
// fileExists returns true if the filename is found
@ -374,9 +493,11 @@ func testCoreConfigBuilder(t *testing.T) *packer.CoreConfig {
ProvisionerStore: packer.MapOfProvisioner{
"shell-local": func() (packer.Provisioner, error) { return &shell_local.Provisioner{}, nil },
"shell": func() (packer.Provisioner, error) { return &shell.Provisioner{}, nil },
"file": func() (packer.Provisioner, error) { return &filep.Provisioner{}, nil },
},
PostProcessorStore: packer.MapOfPostProcessor{
"shell-local": func() (packer.PostProcessor, error) { return &shell_local_pp.PostProcessor{}, nil },
"manifest": func() (packer.PostProcessor, error) { return &manifest.PostProcessor{}, nil },
},
}
return &packer.CoreConfig{

21
command/test-fixtures/build-name-and-type/all.json Normal file
View File

@ -0,0 +1,21 @@
{
"builders": [
{
"name": "test",
"communicator": "none",
"type": "null"
},
{
"name": "potato",
"communicator": "none",
"type": "null"
}
],
"post-processors": [
{
"type": "manifest",
"output": "manifest.json",
"strip_time": true
}
]
}

25
command/test-fixtures/build-name-and-type/all.pkr.hcl Normal file
View File

@ -0,0 +1,25 @@
source "null" "test" {
communicator = "none"
}
source "null" "potato" {
communicator = "none"
}
build {
sources = [
"sources.null.test",
"sources.null.potato",
]
provisioner "shell-local" {
inline = [
"echo '' > ${source.type}.${source.name}.txt"
]
}
post-processor "manifest" {
output = "manifest.json"
strip_time = true
}
}

2
hcl2template/parser.go
View File

@ -166,7 +166,7 @@ func (p *Parser) parse(filename string, varFiles []string, argVars map[string]st
func (p *Parser) decodeConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics {
var diags hcl.Diagnostics
body := dynblock.Expand(f.Body, cfg.EvalContext())
body := dynblock.Expand(f.Body, cfg.EvalContext(nil))
content, moreDiags := body.Content(configSchema)
diags = append(diags, moreDiags...)

4
hcl2template/types.build.post-processor.go
View File

@ -49,7 +49,7 @@ func (p *Parser) decodePostProcessor(block *hcl.Block) (*PostProcessorBlock, hcl
return postProcessor, diags
}
func (p *Parser) startPostProcessor(pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.PostProcessor, hcl.Diagnostics) {
func (p *Parser) startPostProcessor(source *SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.PostProcessor, hcl.Diagnostics) {
// ProvisionerBlock represents a detected but unparsed provisioner
var diags hcl.Diagnostics
@ -64,7 +64,7 @@ func (p *Parser) startPostProcessor(pp *PostProcessorBlock, ectx *hcl.EvalContex
}
flatProvisinerCfg, moreDiags := decodeHCL2Spec(pp.Rest, ectx, postProcessor)
diags = append(diags, moreDiags...)
err = postProcessor.Configure(flatProvisinerCfg, generatedVars)
err = postProcessor.Configure(source.builderVariables(), flatProvisinerCfg, generatedVars)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,

4
hcl2template/types.build.provisioners.go
View File

@ -46,7 +46,7 @@ func (p *Parser) decodeProvisioner(block *hcl.Block) (*ProvisionerBlock, hcl.Dia
return provisioner, diags
}
func (p *Parser) startProvisioner(pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.Provisioner, hcl.Diagnostics) {
func (p *Parser) startProvisioner(source *SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.Provisioner, hcl.Diagnostics) {
var diags hcl.Diagnostics
provisioner, err := p.ProvisionersSchemas.Start(pb.PType)
@ -69,7 +69,7 @@ func (p *Parser) startProvisioner(pb *ProvisionerBlock, ectx *hcl.EvalContext, g
// configs := make([]interface{}, 2)
// configs = append(, flatProvisionerCfg)
// configs = append(configs, generatedVars)
err = provisioner.Prepare(flatProvisionerCfg, generatedVars)
err = provisioner.Prepare(source.builderVariables(), flatProvisionerCfg, generatedVars)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,

42
hcl2template/types.packer_config.go
View File

@ -35,19 +35,32 @@ type ValidationOptions struct {
Strict bool
}
const (
inputVariablesAccessor = "var"
localsAccessor = "local"
sourcesAccessor = "source"
)
// EvalContext returns the *hcl.EvalContext that will be passed to an hcl
// decoder in order to tell what is the actual value of a var or a local and
// the list of defined functions.
func (cfg *PackerConfig) EvalContext() *hcl.EvalContext {
func (cfg *PackerConfig) EvalContext(variables map[string]cty.Value) *hcl.EvalContext {
inputVariables, _ := cfg.InputVariables.Values()
localVariables, _ := cfg.LocalVariables.Values()
ectx := &hcl.EvalContext{
Functions: Functions(cfg.Basedir),
Variables: map[string]cty.Value{
"var": cty.ObjectVal(inputVariables),
"local": cty.ObjectVal(localVariables),
inputVariablesAccessor: cty.ObjectVal(inputVariables),
localsAccessor: cty.ObjectVal(localVariables),
sourcesAccessor: cty.ObjectVal(map[string]cty.Value{
"type": cty.UnknownVal(cty.String),
"name": cty.UnknownVal(cty.String),
}),
},
}
for k, v := range variables {
ectx.Variables[k] = v
}
return ectx
}
@ -157,7 +170,7 @@ func (c *PackerConfig) evaluateLocalVariables(locals []*Local) hcl.Diagnostics {
func (c *PackerConfig) evaluateLocalVariable(local *Local) hcl.Diagnostics {
var diags hcl.Diagnostics
value, moreDiags := local.Expr.Value(c.EvalContext())
value, moreDiags := local.Expr.Value(c.EvalContext(nil))
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
return diags
@ -173,11 +186,11 @@ func (c *PackerConfig) evaluateLocalVariable(local *Local) hcl.Diagnostics {
// getCoreBuildProvisioners takes a list of provisioner block, starts according
// provisioners and sends parsed HCL2 over to it.
func (p *Parser) getCoreBuildProvisioners(blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) {
func (p *Parser) getCoreBuildProvisioners(source *SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) {
var diags hcl.Diagnostics
res := []packer.CoreBuildProvisioner{}
for _, pb := range blocks {
provisioner, moreDiags := p.startProvisioner(pb, ectx, generatedVars)
provisioner, moreDiags := p.startProvisioner(source, pb, ectx, generatedVars)
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
continue
@ -193,11 +206,11 @@ func (p *Parser) getCoreBuildProvisioners(blocks []*ProvisionerBlock, ectx *hcl.
// getCoreBuildProvisioners takes a list of post processor block, starts
// according provisioners and sends parsed HCL2 over to it.
func (p *Parser) getCoreBuildPostProcessors(blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) {
func (p *Parser) getCoreBuildPostProcessors(source *SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) {
var diags hcl.Diagnostics
res := []packer.CoreBuildPostProcessor{}
for _, ppb := range blocks {
postProcessor, moreDiags := p.startPostProcessor(ppb, ectx, generatedVars)
postProcessor, moreDiags := p.startPostProcessor(source, ppb, ectx, generatedVars)
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
continue
@ -230,12 +243,19 @@ func (p *Parser) getBuilds(cfg *PackerConfig) ([]packer.Build, hcl.Diagnostics)
})
continue
}
builder, moreDiags, generatedVars := p.startBuilder(src, cfg.EvalContext())
builder, moreDiags, generatedVars := p.startBuilder(src, cfg.EvalContext(nil))
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
continue
}
variables := map[string]cty.Value{
sourcesAccessor: cty.ObjectVal(map[string]cty.Value{
"type": cty.StringVal(src.Type),
"name": cty.StringVal(src.Name),
}),
}
// If the builder has provided a list of to-be-generated variables that
// should be made accessible to provisioners, pass that list into
// the provisioner prepare() so that the provisioner can appropriately
@ -249,12 +269,12 @@ func (p *Parser) getBuilds(cfg *PackerConfig) ([]packer.Build, hcl.Diagnostics)
}
}
provisioners, moreDiags := p.getCoreBuildProvisioners(build.ProvisionerBlocks, cfg.EvalContext(), generatedPlaceholderMap)
provisioners, moreDiags := p.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables), generatedPlaceholderMap)
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
continue
}
postProcessors, moreDiags := p.getCoreBuildPostProcessors(build.PostProcessors, cfg.EvalContext(), generatedPlaceholderMap)
postProcessors, moreDiags := p.getCoreBuildPostProcessors(src, build.PostProcessors, cfg.EvalContext(variables), generatedPlaceholderMap)
pps := [][]packer.CoreBuildPostProcessor{}
if len(postProcessors) > 0 {
pps = [][]packer.CoreBuildPostProcessor{postProcessors}

9
hcl2template/types.source.go
View File

@ -57,12 +57,19 @@ func (p *Parser) startBuilder(source *SourceBlock, ectx *hcl.EvalContext) (packe
return nil, diags, nil
}
generatedVars, warning, err := builder.Prepare(decoded)
generatedVars, warning, err := builder.Prepare(source.builderVariables(), decoded)
moreDiags = warningErrorsToDiags(source.block, warning, err)
diags = append(diags, moreDiags...)
return builder, diags, generatedVars
}
func (source *SourceBlock) builderVariables() map[string]string {
return map[string]string{
"packer_build_name": source.Name,
"packer_builder_type": source.Type,
}
}
func (source *SourceBlock) Ref() SourceRef {
return SourceRef{
Type: source.Type,

2
post-processor/manifest/artifact.go
View File

@ -12,7 +12,7 @@ type ArtifactFile struct {
type Artifact struct {
BuildName string `json:"name"`
BuilderType string `json:"builder_type"`
BuildTime int64 `json:"build_time"`
BuildTime int64 `json:"build_time,omitempty"`
ArtifactFiles []ArtifactFile `json:"files"`
ArtifactId string `json:"artifact_id"`
PackerRunUUID string `json:"packer_run_uuid"`

13
post-processor/manifest/post-processor.go
View File

@ -1,4 +1,5 @@
//go:generate mapstructure-to-hcl2 -type Config
//go:generate struct-markdown
package manifest
@ -22,8 +23,17 @@ import (
type Config struct {
common.PackerConfig `mapstructure:",squash"`
// The manifest will be written to this file. This defaults to
// `packer-manifest.json`.
OutputPath string `mapstructure:"output"`
// Write only filename without the path to the manifest file. This defaults
// to false.
StripPath bool `mapstructure:"strip_path"`
// Don't write the `build_time` field from the output.
StripTime bool `mapstructure:"strip_time"`
// Arbitrary data to add to the manifest. This is a [template
// engine](https://packer.io/docs/templates/engine.html). Therefore, you
// may use user variables and template functions in this field.
CustomData map[string]string `mapstructure:"custom_data"`
ctx interpolate.Context
}
@ -101,6 +111,9 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, source pa
artifact.BuilderType = p.config.PackerBuilderType
artifact.BuildName = p.config.PackerBuildName
artifact.BuildTime = time.Now().Unix()
if p.config.StripTime {
artifact.BuildTime = 0
}
// Since each post-processor runs in a different process we need a way to
// coordinate between various post-processors in a single packer run. We do
// this by setting a UUID per run and tracking this in the manifest file.

2
post-processor/manifest/post-processor.hcl2spec.go
View File

@ -18,6 +18,7 @@ type FlatConfig struct {
PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables"`
OutputPath *string `mapstructure:"output" cty:"output"`
StripPath *bool `mapstructure:"strip_path" cty:"strip_path"`
StripTime *bool `mapstructure:"strip_time" cty:"strip_time"`
CustomData map[string]string `mapstructure:"custom_data" cty:"custom_data"`
}
@ -42,6 +43,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
"packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
"output": &hcldec.AttrSpec{Name: "output", Type: cty.String, Required: false},
"strip_path": &hcldec.AttrSpec{Name: "strip_path", Type: cty.Bool, Required: false},
"strip_time": &hcldec.AttrSpec{Name: "strip_time", Type: cty.Bool, Required: false},
"custom_data": &hcldec.BlockAttrsSpec{TypeName: "custom_data", ElementType: cty.String, Required: false},
}
return s

8
website/pages/docs/post-processors/manifest.mdx
View File

@ -36,13 +36,7 @@ post-processors such as Docker and Artifice.
### Optional:
- `output` (string) The manifest will be written to this file. This defaults
to `packer-manifest.json`.
- `strip_path` (boolean) Write only filename without the path to the manifest
file. This defaults to false.
- `custom_data` (map of strings) Arbitrary data to add to the manifest.
This is a [template engine](/docs/templates/engine). Therefore, you
may use user variables and template functions in this field.
@include 'post-processor/manifest/Config-not-required.mdx'
- `keep_input_artifact` (boolean) - Unlike most other post-processors, the
keep_input_artifact option will have no effect for the manifest
post-processor. We will always retain the input artifact for manifest,

14
website/pages/partials/post-processor/manifest/Config-not-required.mdx Normal file
View File

@ -0,0 +1,14 @@
<!-- Code generated from the comments of the Config struct in post-processor/manifest/post-processor.go; DO NOT EDIT MANUALLY -->
- `output` (string) - The manifest will be written to this file. This defaults to
`packer-manifest.json`.
- `strip_path` (bool) - Write only filename without the path to the manifest file. This defaults
to false.
- `strip_time` (bool) - Don't write the `build_time` field from the output.
- `custom_data` (map[string]string) - Arbitrary data to add to the manifest. This is a [template
engine](https://packer.io/docs/templates/engine.html). Therefore, you
may use user variables and template functions in this field.