iSharkFly-Docs
/
packer-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HCL2 only: Make provisioners and post-processors reprepare themselves with build data (#9534) 

#9444 allows to share builder infos with provisioners and post-processor by re-starting new provisioners and post-processors once the build has started; #9490 allows to 'excpect/only' on a full build blocks (after naming a block), merged together, the tests of #9490 fail. This means we would have to reimplement the only/except logic. While this could have worked, re-using the already started provisioners and post-processor for HCL2 is better/simpler IMO as we won't have to maintain a feature parity.

Also with HCL2 and go-cty all struct ( here provisioner/post-processor) fields are set at the moment of configuring so we should not get weird 'default value' behaviour issue.

This PR reverts some parts of #9444 and implement #9444 in a different manner: I created specific HCL2Provisioner and HCL2PostProcessor structs that both re-prepare themselves right before provisioning. I think this makes the code a little simpler and lays a nice ground for improvement.
This commit is contained in:
Adrien Delorme 2020-07-07 12:10:02 +02:00 committed by GitHub
parent 93cd7e40bc b0238e6f4a
commit 741a6e4182
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 497 additions and 527 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
command/build_test.go
View File

@ -20,6 +20,22 @@ import (
shell_local "github.com/hashicorp/packer/provisioner/shell-local"
)
var (
spaghettiCarbonara = `spaghetti
carbonara
`
lasagna = `lasagna
tomato
mozza
cooking...
`
tiramisu = `whip_york
mascarpone
whipped_egg_white
dress
`
)
func TestBuild(t *testing.T) {
tc := []struct {
name string
@ -228,8 +244,72 @@ func TestBuild(t *testing.T) {
testFixture("hcl-only-except"),
},
fileCheck: fileCheck{
expected: []string{"chocolate.txt", "vanilla.txt"},
notExpected: []string{"cherry.txt"},
expected: []string{"chocolate.txt", "vanilla.txt"},
},
},
// recipes
{
name: "hcl - recipes",
args: []string{
testFixture("hcl", "recipes"),
},
fileCheck: fileCheck{
expectedContent: map[string]string{
"NULL.spaghetti_carbonara.txt": spaghettiCarbonara,
"NULL.lasagna.txt": lasagna,
"NULL.tiramisu.txt": tiramisu,
},
},
},
{
name: "hcl - recipes - except carbonara",
args: []string{
"-except", "recipes.null.spaghetti_carbonara",
testFixture("hcl", "recipes"),
},
fileCheck: fileCheck{
notExpected: []string{"NULL.spaghetti_carbonara.txt"},
expectedContent: map[string]string{
"NULL.lasagna.txt": lasagna,
"NULL.tiramisu.txt": tiramisu,
},
},
},
{
name: "hcl - recipes - only lasagna",
args: []string{
"-only", "*lasagna",
testFixture("hcl", "recipes"),
},
fileCheck: fileCheck{
notExpected: []string{
"NULL.spaghetti_carbonara.txt",
"NULL.tiramisu.txt",
},
expectedContent: map[string]string{
"NULL.lasagna.txt": lasagna,
},
},
},
{
name: "hcl - recipes - only recipes",
args: []string{
"-only", "recipes.*",
testFixture("hcl", "recipes"),
},
fileCheck: fileCheck{
notExpected: []string{
"NULL.tiramisu.txt",
},
expectedContent: map[string]string{
"NULL.spaghetti_carbonara.txt": spaghettiCarbonara,
"NULL.lasagna.txt": lasagna,
},
},
},
}
@ -341,109 +421,26 @@ func TestBuildProvisionAndPosProcessWithBuildVariablesSharing(t *testing.T) {
c := &BuildCommand{
Meta: testMetaFile(t),
}
tc := []struct {
name string
args []string
expectedFiles []string
notExpectedFiles []string
}{
{
name: "JSON: basic template",
args: []string{
filepath.Join(testFixture("build-variable-sharing"), "template.json"),
},
expectedFiles: []string{
"provisioner.Null.txt",
"post-processor.Null.txt",
},
notExpectedFiles: []string{},
},
{
name: "HCL2: basic template",
args: []string{
filepath.Join(testFixture("build-variable-sharing"), "basic_template.pkr.hcl"),
},
expectedFiles: []string{
"provisioner.Null.txt",
"post-processor.Null.txt",
},
notExpectedFiles: []string{},
},
{
name: "HCL2: basic template with build variables within HCL function",
args: []string{
filepath.Join(testFixture("build-variable-sharing"), "basic_template_with_hcl_func.pkr.hcl"),
},
expectedFiles: []string{
"provisioner.Null.txt",
"provisioner.NULL.txt",
"post-processor.Null.txt",
"post-processor.NULL.txt",
},
notExpectedFiles: []string{},
},
{
name: "HCL2: basic template with named build",
args: []string{
filepath.Join(testFixture("build-variable-sharing"), "named_build.pkr.hcl"),
},
expectedFiles: []string{
"provisioner.Null.txt",
"post-processor.Null.txt",
},
notExpectedFiles: []string{},
},
{
name: "HCL2: multiple build block sharing same sources",
args: []string{
filepath.Join(testFixture("build-variable-sharing"), "multiple_build_blocks.pkr.hcl"),
},
expectedFiles: []string{
"vanilla.chocolate.provisioner.Null.txt",
"vanilla.chocolate.post-processor.Null.txt",
"apple.chocolate.provisioner.Null.txt",
"apple.chocolate.post-processor.Null.txt",
"sugar.banana.provisioner.Null.txt",
"sugar.banana.post-processor.Null.txt",
},
notExpectedFiles: []string{},
},
{
name: "HCL2: multiple sources build with only/except set for provisioner and post-processors",
args: []string{
filepath.Join(testFixture("build-variable-sharing"), "multiple_source_build.pkr.hcl"),
},
expectedFiles: []string{
"all.Null.txt",
},
notExpectedFiles: []string{
"chocolate.Null.txt",
"banana.Null.txt",
},
},
args := []string{
filepath.Join(testFixture("build-variable-sharing"), "template.json"),
}
for _, tt := range tc {
t.Run(tt.name, func(t *testing.T) {
defer cleanup(tt.expectedFiles...)
defer cleanup(tt.notExpectedFiles...)
files := []string{
"provisioner.Null.txt",
"post-processor.Null.txt",
}
if code := c.Run(tt.args); code != 0 {
fatalCommand(t, c.Meta)
}
defer cleanup(files...)
for _, f := range tt.expectedFiles {
if !fileExists(f) {
t.Errorf("Expected to find %s", f)
}
}
if code := c.Run(args); code != 0 {
fatalCommand(t, c.Meta)
}
for _, f := range tt.notExpectedFiles {
if fileExists(f) {
t.Errorf("Not expected to find %s", f)
}
}
})
for _, f := range files {
if !fileExists(f) {
t.Errorf("Expected to find %s", f)
}
}
}

13
command/build_windows_test.go Normal file
View File

@ -0,0 +1,13 @@
package command
import "strings"
func init() {
spaghettiCarbonara = fixWindowsLineEndings(spaghettiCarbonara)
lasagna = fixWindowsLineEndings(lasagna)
tiramisu = fixWindowsLineEndings(tiramisu)
}
func fixWindowsLineEndings(s string) string {
return strings.ReplaceAll(s, "\n", " \r\n")
}

6
command/command_test.go
View File

@ -27,8 +27,10 @@ func outputCommand(t *testing.T, m Meta) (string, string) {
return out.String(), err.String()
}
func testFixture(n string) string {
return filepath.Join(fixturesDir, n)
func testFixture(n ...string) string {
paths := []string{fixturesDir}
paths = append(paths, n...)
return filepath.Join(paths...)
}
func testMeta(t *testing.T) Meta {

15
command/test-fixtures/build-variable-sharing/basic_template.pkr.hcl
View File

@ -1,15 +0,0 @@
source "null" "chocolate" {
communicator = "none"
}
build {
sources = ["null.chocolate"]
provisioner "shell-local" {
inline = ["echo hi > provisioner.${build.ID}.txt"]
}
post-processor "shell-local" {
inline = ["echo hi > post-processor.${build.ID}.txt"]
}
}

21
command/test-fixtures/build-variable-sharing/basic_template_with_hcl_func.pkr.hcl
View File

@ -1,21 +0,0 @@
source "null" "chocolate" {
communicator = "none"
}
build {
sources = ["null.chocolate"]
provisioner "shell-local" {
inline = [
"echo hi > provisioner.${build.ID}.txt",
"echo hi > provisioner.${upper(build.ID)}.txt"
]
}
post-processor "shell-local" {
inline = [
"echo hi > post-processor.${build.ID}.txt",
"echo hi > post-processor.${upper(build.ID)}.txt"
]
}
}

46
command/test-fixtures/build-variable-sharing/multiple_build_blocks.pkr.hcl
View File

@ -1,46 +0,0 @@
source "null" "chocolate" {
communicator = "none"
}
source "null" "banana" {
communicator = "none"
}
build {
name = "vanilla"
sources = ["null.chocolate"]
provisioner "shell-local" {
inline = ["echo hi > vanilla.chocolate.provisioner.${build.ID}.txt"]
}
post-processor "shell-local" {
inline = ["echo hi > vanilla.chocolate.post-processor.${build.ID}.txt"]
}
}
build {
name = "apple"
sources = ["null.chocolate"]
provisioner "shell-local" {
inline = ["echo hi > apple.chocolate.provisioner.${build.ID}.txt"]
}
post-processor "shell-local" {
inline = ["echo hi > apple.chocolate.post-processor.${build.ID}.txt"]
}
}
build {
name = "sugar"
sources = ["null.banana"]
provisioner "shell-local" {
inline = ["echo hi > sugar.banana.provisioner.${build.ID}.txt"]
}
post-processor "shell-local" {
inline = ["echo hi > sugar.banana.post-processor.${build.ID}.txt"]
}
}

32
command/test-fixtures/build-variable-sharing/multiple_source_build.pkr.hcl
View File

@ -1,32 +0,0 @@
source "null" "chocolate" {
communicator = "none"
}
source "null" "banana" {
communicator = "none"
}
build {
name = "vanilla"
sources = [
"null.chocolate",
"null.banana",
]
provisioner "shell-local" {
inline = [
"echo hi > all.${build.ID}.txt",
"echo hi > chocolate.${build.ID}.txt",
"echo hi > banana.${build.ID}.txt"
]
}
post-processor "shell-local" {
only = ["null.chocolate"]
inline = ["rm chocolate.${build.ID}.txt"]
}
post-processor "shell-local" {
except = ["null.chocolate"]
inline = ["rm banana.${build.ID}.txt"]
}
}

16
command/test-fixtures/build-variable-sharing/named_build.pkr.hcl
View File

@ -1,16 +0,0 @@
source "null" "chocolate" {
communicator = "none"
}
build {
name = "vanilla"
sources = ["null.chocolate"]
provisioner "shell-local" {
inline = ["echo hi > provisioner.${build.ID}.txt"]
}
post-processor "shell-local" {
inline = ["echo hi > post-processor.${build.ID}.txt"]
}
}

72
command/test-fixtures/hcl/recipes/build.pkr.hcl Normal file
View File

@ -0,0 +1,72 @@
build {
source "source.null.base" {
name = "tiramisu"
// pull me up !
}
provisioner "shell-local" {
name = "whipped_york"
inline = [ "echo whip_york > ${upper(build.ID)}.${source.name}.txt" ]
}
provisioner "shell-local" {
name = "mascarpone"
inline = [ "echo mascarpone >> ${upper(build.ID)}.${source.name}.txt" ]
}
post-processor "shell-local" {
name = "whipped_egg_white"
inline = [ "echo whipped_egg_white >> ${upper(build.ID)}.${source.name}.txt" ]
}
post-processor "shell-local" {
name = "dress_with_coffeed_boudoirs"
inline = [ "echo dress >> ${upper(build.ID)}.${source.name}.txt" ]
}
}
build {
name = "recipes"
source "source.null.base" {
name = "spaghetti_carbonara"
}
source "source.null.base" {
name = "lasagna"
}
provisioner "shell-local" {
name = "add_spaghetti"
inline = [ "echo spaghetti > ${upper(build.ID)}.${source.name}.txt" ]
only = ["null.spaghetti_carbonara"]
}
post-processor "shell-local" {
name = "carbonara_it"
inline = [ "echo carbonara >> ${upper(build.ID)}.${source.name}.txt" ]
except = ["null.lasagna"]
}
provisioner "shell-local" {
name = "add_lasagna"
inline = [ "echo lasagna > ${upper(build.ID)}.${source.name}.txt" ]
only = ["null.lasagna"]
}
provisioner "shell-local" {
name = "add_tomato"
inline = [ "echo tomato >> ${upper(build.ID)}.${source.name}.txt" ]
except = ["null.spaghetti_carbonara"]
}
provisioner "shell-local" {
name = "add_mozza"
inline = [ "echo mozza >> ${upper(build.ID)}.${source.name}.txt" ]
except = ["null.spaghetti_carbonara"]
}
post-processor "shell-local" {
name = "cook"
inline = [ "echo cooking... >> ${upper(build.ID)}.${source.name}.txt" ]
except = ["null.spaghetti_carbonara"]
}
}

3
command/test-fixtures/hcl/recipes/sources.pkr.hcl Normal file
View File

@ -0,0 +1,3 @@
source "null" "base" {
communicator = "none"
}

37
hcl2template/common_test.go
View File

@ -121,37 +121,6 @@ func testParse(t *testing.T, tests []parseTest) {
if tt.getBuildsWantDiags == (gotDiags == nil) {
t.Fatalf("Parser.getBuilds() unexpected diagnostics. %s", gotDiags)
}
// Validates implementation of HCL2ProvisionerPrepare and HCL2PostProcessorsPrepare
for _, build := range gotBuilds {
coreBuild, ok := build.(*packer.CoreBuild)
if !ok {
t.Fatalf("build %s should implement CoreBuild", build.Name())
}
if coreBuild.HCL2ProvisionerPrepare == nil {
t.Fatalf("build %s should have HCL2ProvisionerPrepare implementation", build.Name())
}
if coreBuild.HCL2PostProcessorsPrepare == nil {
t.Fatalf("build %s should have HCL2PostProcessorsPrepare implementation", build.Name())
}
provisioners, diags := coreBuild.HCL2ProvisionerPrepare(nil)
if diags.HasErrors() {
t.Fatalf("build %s: HCL2ProvisionerPrepare should prepare provisioners", build.Name())
}
coreBuild.Provisioners = provisioners
postProcessors, diags := coreBuild.HCL2PostProcessorsPrepare(nil)
if diags.HasErrors() {
t.Fatalf("build %s: HCL2PostProcessorsPrepare should prepare post-processors", build.Name())
}
if len(postProcessors) > 0 {
coreBuild.PostProcessors = [][]packer.CoreBuildPostProcessor{postProcessors}
} else {
coreBuild.PostProcessors = [][]packer.CoreBuildPostProcessor{}
}
}
if diff := cmp.Diff(tt.getBuildsWantBuilds, gotBuilds,
cmpopts.IgnoreUnexported(
cty.Value{},
@ -160,8 +129,9 @@ func testParse(t *testing.T, tests []parseTest) {
packer.CoreBuildProvisioner{},
packer.CoreBuildPostProcessor{},
null.Builder{},
HCL2Provisioner{},
HCL2PostProcessor{},
),
cmpopts.IgnoreFields(packer.CoreBuild{}, "HCL2ProvisionerPrepare", "HCL2PostProcessorsPrepare"),
); diff != "" {
t.Fatalf("Parser.getBuilds() wrong packer builds. %s", diff)
}
@ -208,7 +178,7 @@ var (
basicMockProvisioner = &MockProvisioner{
Config: MockConfig{
NotSquashed: "value",
NotSquashed: "value <UNKNOWN>",
NestedMockConfig: basicNestedMockConfig,
Nested: basicNestedMockConfig,
NestedSlice: []NestedMockConfig{
@ -220,6 +190,7 @@ var (
}
basicMockPostProcessor = &MockPostProcessor{
Config: MockConfig{
NotSquashed: "value <UNKNOWN>",
NestedMockConfig: basicNestedMockConfig,
Nested: basicNestedMockConfig,
NestedSlice: []NestedMockConfig{

2
hcl2template/internal/mock.go
View File

@ -74,7 +74,7 @@ var _ packer.Builder = new(MockBuilder)
func (b *MockBuilder) ConfigSpec() hcldec.ObjectSpec { return b.Config.FlatMapstructure().HCL2Spec() }
func (b *MockBuilder) Prepare(raws ...interface{}) ([]string, []string, error) {
return nil, nil, b.Config.Prepare(raws...)
return []string{"ID"}, nil, b.Config.Prepare(raws...)
}
func (b *MockBuilder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {

6
hcl2template/testdata/complete/build.pkr.hcl vendored
View File

@ -11,7 +11,7 @@ build {
provisioner "shell" {
name = "provisioner that does something"
not_squashed = var.foo
not_squashed = "${var.foo} ${upper(build.ID)}"
string = "string"
int = "${41 + 1}"
int64 = "${42 + 1}"
@ -62,7 +62,7 @@ build {
}
provisioner "file" {
not_squashed = "${var.foo}"
not_squashed = "${var.foo} ${upper(build.ID)}"
string = "string"
int = 42
int64 = 43
@ -138,6 +138,7 @@ build {
["a","b"],
["c","d"]
]
not_squashed = "${var.foo} ${upper(build.ID)}"
nested {
string = "string"
@ -185,6 +186,7 @@ build {
["a","b"],
["c","d"]
]
not_squashed = "${var.foo} ${upper(build.ID)}"
nested {
string = "string"

56
hcl2template/types.build.post-processor.go
View File

@ -6,7 +6,6 @@ import (
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/packer/packer"
"github.com/zclconf/go-cty/cty"
)
// ProvisionerBlock references a detected but unparsed post processor
@ -75,9 +74,13 @@ func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcesso
})
return nil, diags
}
flatProvisinerCfg, moreDiags := decodeHCL2Spec(pp.Rest, ectx, postProcessor)
diags = append(diags, moreDiags...)
err = postProcessor.Configure(source.builderVariables(), flatProvisinerCfg)
hclPostProcessor := &HCL2PostProcessor{
PostProcessor: postProcessor,
postProcessorBlock: pp,
evalContext: ectx,
builderVariables: source.builderVariables(),
}
err = hclPostProcessor.HCL2Prepare(nil)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
@ -87,48 +90,5 @@ func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcesso
})
return nil, diags
}
return postProcessor, diags
}
type postProcessorsPrepare struct {
cfg *PackerConfig
postProcessorBlock []*PostProcessorBlock
src SourceRef
}
// HCL2PostProcessorsPrepare is used by the CoreBuild at the runtime, after running the build and before running the post-processors,
// to interpolate any build variable by decoding and preparing it.
func (pp *postProcessorsPrepare) HCL2PostProcessorsPrepare(builderArtifact packer.Artifact) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) {
src := pp.cfg.Sources[pp.src.Ref()]
generatedData := make(map[string]interface{})
if builderArtifact != nil {
artifactStateData := builderArtifact.State("generated_data")
if artifactStateData != nil {
for k, v := range artifactStateData.(map[interface{}]interface{}) {
generatedData[k.(string)] = v
}
}
}
variables := make(Variables)
for k, v := range generatedData {
if value, ok := v.(string); ok {
variables[k] = &Variable{
DefaultValue: cty.StringVal(value),
Type: cty.String,
}
}
}
variablesVal, _ := variables.Values()
generatedVariables := map[string]cty.Value{
sourcesAccessor: cty.ObjectVal(map[string]cty.Value{
"type": cty.StringVal(src.Type),
"name": cty.StringVal(src.Name),
}),
buildAccessor: cty.ObjectVal(variablesVal),
}
return pp.cfg.getCoreBuildPostProcessors(src, pp.postProcessorBlock, pp.cfg.EvalContext(generatedVariables))
return hclPostProcessor, diags
}

53
hcl2template/types.build.provisioners.go
View File

@ -7,7 +7,6 @@ import (
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/packer/packer"
"github.com/zclconf/go-cty/cty"
)
// OnlyExcept is a struct that is meant to be embedded that contains the
@ -145,18 +144,13 @@ func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlo
})
return nil, diags
}
flatProvisionerCfg, moreDiags := decodeHCL2Spec(pb.HCL2Ref.Rest, ectx, provisioner)
diags = append(diags, moreDiags...)
if diags.HasErrors() {
return nil, diags
hclProvisioner := &HCL2Provisioner{
Provisioner: provisioner,
provisionerBlock: pb,
evalContext: ectx,
builderVariables: source.builderVariables(),
}
// manipulate generatedVars from builder to add to the interfaces being
// passed to the provisioner Prepare()
// configs := make([]interface{}, 2)
// configs = append(, flatProvisionerCfg)
// configs = append(configs, generatedVars)
err = provisioner.Prepare(source.builderVariables(), flatProvisionerCfg)
err = hclProvisioner.HCL2Prepare(nil)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
@ -166,38 +160,5 @@ func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlo
})
return nil, diags
}
return provisioner, diags
}
type provisionerPrepare struct {
cfg *PackerConfig
provisionerBlock []*ProvisionerBlock
src SourceRef
}
// HCL2ProvisionerPrepare is used by the ProvisionHook at the runtime in the provision step
// to interpolate any build variable by decoding and preparing it again.
func (pp *provisionerPrepare) HCL2ProvisionerPrepare(data map[string]interface{}) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) {
src := pp.cfg.Sources[pp.src.Ref()]
variables := make(Variables)
for k, v := range data {
if value, ok := v.(string); ok {
variables[k] = &Variable{
DefaultValue: cty.StringVal(value),
Type: cty.String,
}
}
}
variablesVal, _ := variables.Values()
generatedVariables := map[string]cty.Value{
sourcesAccessor: cty.ObjectVal(map[string]cty.Value{
"type": cty.StringVal(src.Type),
"name": cty.StringVal(src.Name),
}),
buildAccessor: cty.ObjectVal(variablesVal),
}
return pp.cfg.getCoreBuildProvisioners(src, pp.provisionerBlock, pp.cfg.EvalContext(generatedVariables))
return hclProvisioner, diags
}

80
hcl2template/types.build_test.go
View File

@ -171,19 +171,23 @@ func TestParse_build(t *testing.T) {
{
{
PType: "amazon-import",
PostProcessor: &MockPostProcessor{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
PostProcessor: &HCL2PostProcessor{
PostProcessor: &MockPostProcessor{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
},
},
},
},
{
PType: "manifest",
PostProcessor: &MockPostProcessor{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
PostProcessor: &HCL2PostProcessor{
PostProcessor: &MockPostProcessor{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
},
},
},
},
@ -199,19 +203,23 @@ func TestParse_build(t *testing.T) {
{
{
PType: "manifest",
PostProcessor: &MockPostProcessor{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
PostProcessor: &HCL2PostProcessor{
PostProcessor: &MockPostProcessor{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
},
},
},
},
{
PType: "amazon-import",
PostProcessor: &MockPostProcessor{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
PostProcessor: &HCL2PostProcessor{
PostProcessor: &MockPostProcessor{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
},
},
},
},
@ -266,19 +274,23 @@ func TestParse_build(t *testing.T) {
Provisioners: []packer.CoreBuildProvisioner{
{
PType: "shell",
Provisioner: &MockProvisioner{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
Provisioner: &HCL2Provisioner{
Provisioner: &MockProvisioner{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
},
},
},
},
{
PType: "file",
Provisioner: &MockProvisioner{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
Provisioner: &HCL2Provisioner{
Provisioner: &MockProvisioner{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
},
},
},
},
@ -292,19 +304,23 @@ func TestParse_build(t *testing.T) {
Provisioners: []packer.CoreBuildProvisioner{
{
PType: "file",
Provisioner: &MockProvisioner{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
Provisioner: &HCL2Provisioner{
Provisioner: &MockProvisioner{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
},
},
},
},
{
PType: "shell",
Provisioner: &MockProvisioner{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
Provisioner: &HCL2Provisioner{
Provisioner: &MockProvisioner{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
NestedSlice: []NestedMockConfig{},
},
},
},
},

72
hcl2template/types.hcl_post-processor.go Normal file
View File

@ -0,0 +1,72 @@
package hcl2template
import (
"context"
"fmt"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/packer/packer"
"github.com/zclconf/go-cty/cty"
)
// HCL2PostProcessor has a reference to the part of the HCL2 body where it is
// defined, allowing to completely reconfigure the PostProcessor right before
// calling PostProcess: with contextual variables.
// This permits using "${build.ID}" values for example.
type HCL2PostProcessor struct {
PostProcessor packer.PostProcessor
postProcessorBlock *PostProcessorBlock
evalContext *hcl.EvalContext
builderVariables map[string]string
}
func (p *HCL2PostProcessor) ConfigSpec() hcldec.ObjectSpec {
return p.PostProcessor.ConfigSpec()
}
func (p *HCL2PostProcessor) HCL2Prepare(buildVars map[string]interface{}) error {
var diags hcl.Diagnostics
ectx := p.evalContext
if len(buildVars) > 0 {
ectx = p.evalContext.NewChild()
buildValues := map[string]cty.Value{}
for k, v := range buildVars {
switch v := v.(type) {
case string:
buildValues[k] = cty.StringVal(v)
default:
return fmt.Errorf("unhandled builvar type: %T", v)
}
}
ectx.Variables = map[string]cty.Value{
buildAccessor: cty.ObjectVal(buildValues),
}
}
flatPostProcessorCfg, moreDiags := decodeHCL2Spec(p.postProcessorBlock.HCL2Ref.Rest, ectx, p.PostProcessor)
diags = append(diags, moreDiags...)
if diags.HasErrors() {
return diags
}
return p.PostProcessor.Configure(p.builderVariables, flatPostProcessorCfg)
}
func (p *HCL2PostProcessor) Configure(args ...interface{}) error {
return p.PostProcessor.Configure(args...)
}
func (p *HCL2PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) {
generatedData := make(map[string]interface{})
if artifactStateData, ok := artifact.State("generated_data").(map[interface{}]interface{}); ok {
for k, v := range artifactStateData {
generatedData[k.(string)] = v
}
}
err := p.HCL2Prepare(generatedData)
if err != nil {
return nil, false, false, err
}
return p.PostProcessor.PostProcess(ctx, ui, artifact)
}

65
hcl2template/types.hcl_provisioner.go Normal file
View File

@ -0,0 +1,65 @@
package hcl2template
import (
"context"
"fmt"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/packer/packer"
"github.com/zclconf/go-cty/cty"
)
// HCL2Provisioner has a reference to the part of the HCL2 body where it is
// defined, allowing to completely reconfigure the Provisioner right before
// calling Provision: with contextual variables.
// This permits using "${build.ID}" values for example.
type HCL2Provisioner struct {
Provisioner packer.Provisioner
provisionerBlock *ProvisionerBlock
evalContext *hcl.EvalContext
builderVariables map[string]string
}
func (p *HCL2Provisioner) ConfigSpec() hcldec.ObjectSpec {
return p.Provisioner.ConfigSpec()
}
func (p *HCL2Provisioner) HCL2Prepare(buildVars map[string]interface{}) error {
var diags hcl.Diagnostics
ectx := p.evalContext
if len(buildVars) > 0 {
ectx = p.evalContext.NewChild()
buildValues := map[string]cty.Value{}
for k, v := range buildVars {
switch v := v.(type) {
case string:
buildValues[k] = cty.StringVal(v)
default:
return fmt.Errorf("unhandled builvar type: %T", v)
}
}
ectx.Variables = map[string]cty.Value{
buildAccessor: cty.ObjectVal(buildValues),
}
}
flatProvisionerCfg, moreDiags := decodeHCL2Spec(p.provisionerBlock.HCL2Ref.Rest, ectx, p.Provisioner)
diags = append(diags, moreDiags...)
if diags.HasErrors() {
return diags
}
return p.Provisioner.Prepare(p.builderVariables, flatProvisionerCfg)
}
func (p *HCL2Provisioner) Prepare(args ...interface{}) error {
return p.Provisioner.Prepare(args...)
}
func (p *HCL2Provisioner) Provision(ctx context.Context, ui packer.Ui, c packer.Communicator, vars map[string]interface{}) error {
err := p.HCL2Prepare(vars)
if err != nil {
return err
}
return p.Provisioner.Provision(ctx, ui, c, vars)
}

57
hcl2template/types.packer_config.go
View File

@ -40,7 +40,6 @@ type PackerConfig struct {
except []glob.Glob
only []glob.Glob
debug bool
}
type ValidationOptions struct {
@ -69,6 +68,7 @@ func (cfg *PackerConfig) EvalContext(variables map[string]cty.Value) *hcl.EvalCo
"type": cty.UnknownVal(cty.String),
"name": cty.UnknownVal(cty.String),
}),
buildAccessor: cty.UnknownVal(cty.EmptyObject),
},
}
for k, v := range variables {
@ -211,6 +211,7 @@ func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*
if moreDiags.HasErrors() {
continue
}
// If we're pausing, we wrap the provisioner in a special pauser.
if pb.PauseBefore != 0 {
provisioner = &packer.PausedProvisioner{
@ -229,11 +230,6 @@ func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*
Provisioner: provisioner,
}
}
if cfg.debug {
provisioner = &packer.DebuggedProvisioner{
Provisioner: provisioner,
}
}
res = append(res, packer.CoreBuildProvisioner{
PType: pb.PType,
@ -293,8 +289,6 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build
res := []packer.Build{}
var diags hcl.Diagnostics
cfg.debug = opts.Debug
for _, build := range cfg.Builds {
for _, from := range build.Sources {
src, found := cfg.Sources[from.Ref()]
@ -366,45 +360,35 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build
// the provisioner prepare() so that the provisioner can appropriately
// validate user input against what will become available. Otherwise,
// only pass the default variables, using the basic placeholder data.
generatedPlaceholderMap := packer.BasicPlaceholderData()
if generatedVars != nil {
for _, k := range generatedVars {
generatedPlaceholderMap[k] = ""
}
unknownBuildValues := map[string]cty.Value{}
for _, k := range append(packer.BuilderDataCommonKeys, generatedVars...) {
unknownBuildValues[k] = cty.StringVal("<unknown>")
}
buildVariables, _ := setBuildVariables(generatedPlaceholderMap).Values()
variables := map[string]cty.Value{
sourcesAccessor: cty.ObjectVal(map[string]cty.Value{
"type": cty.StringVal(src.Type),
"name": cty.StringVal(src.Name),
}),
buildAccessor: cty.ObjectVal(buildVariables),
sourcesAccessor: cty.ObjectVal(src.ctyValues()),
buildAccessor: cty.ObjectVal(unknownBuildValues),
}
_, moreDiags = cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables))
provisioners, moreDiags := cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables))
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
continue
}
_, moreDiags = cfg.getCoreBuildPostProcessors(src, build.PostProcessors, cfg.EvalContext(variables))
postProcessors, moreDiags := cfg.getCoreBuildPostProcessors(src, build.PostProcessors, cfg.EvalContext(variables))
pps := [][]packer.CoreBuildPostProcessor{}
if len(postProcessors) > 0 {
pps = [][]packer.CoreBuildPostProcessor{postProcessors}
} // TODO(azr): remove this
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
continue
}
pcb.Builder = builder
pcb.Provisioners = provisioners
pcb.PostProcessors = pps
pcb.Prepared = true
pcb.HCL2ProvisionerPrepare = (&provisionerPrepare{
cfg: cfg,
provisionerBlock: build.ProvisionerBlocks,
src: from,
}).HCL2ProvisionerPrepare
pcb.HCL2PostProcessorsPrepare = (&postProcessorsPrepare{
cfg: cfg,
postProcessorBlock: build.PostProcessors,
src: from,
}).HCL2PostProcessorsPrepare
// Prepare just sets the "prepareCalled" flag on CoreBuild, since
// we did all the prep here.
@ -425,17 +409,6 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build
return res, diags
}
func setBuildVariables(generatedVars map[string]string) Variables {
variables := make(Variables)
for k := range generatedVars {
variables[k] = &Variable{
DefaultValue: cty.StringVal("unknown"),
Type: cty.String,
}
}
return variables
}
var PackerConsoleHelp = strings.TrimSpace(`
Packer console HCL2 Mode.
The Packer console allows you to experiment with Packer interpolations.

58
hcl2template/types.packer_config_test.go
View File

@ -110,23 +110,34 @@ func TestParser_complete(t *testing.T) {
Builder: basicMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{
{
PType: "shell",
PName: "provisioner that does something",
Provisioner: basicMockProvisioner,
PType: "shell",
PName: "provisioner that does something",
Provisioner: &HCL2Provisioner{
Provisioner: basicMockProvisioner,
},
},
{
PType: "file",
Provisioner: &HCL2Provisioner{
Provisioner: basicMockProvisioner,
},
},
{PType: "file", Provisioner: basicMockProvisioner},
},
PostProcessors: [][]packer.CoreBuildPostProcessor{
{
{
PType: "amazon-import",
PName: "something",
PostProcessor: basicMockPostProcessor,
PType: "amazon-import",
PName: "something",
PostProcessor: &HCL2PostProcessor{
PostProcessor: basicMockPostProcessor,
},
KeepInputArtifact: pTrue,
},
{
PType: "amazon-import",
PostProcessor: basicMockPostProcessor,
PType: "amazon-import",
PostProcessor: &HCL2PostProcessor{
PostProcessor: basicMockPostProcessor,
},
},
},
},
@ -146,23 +157,34 @@ func TestParser_complete(t *testing.T) {
},
Provisioners: []packer.CoreBuildProvisioner{
{
PType: "shell",
PName: "provisioner that does something",
Provisioner: basicMockProvisioner,
PType: "shell",
PName: "provisioner that does something",
Provisioner: &HCL2Provisioner{
Provisioner: basicMockProvisioner,
},
},
{
PType: "file",
Provisioner: &HCL2Provisioner{
Provisioner: basicMockProvisioner,
},
},
{PType: "file", Provisioner: basicMockProvisioner},
},
PostProcessors: [][]packer.CoreBuildPostProcessor{
{
{
PType: "amazon-import",
PName: "something",
PostProcessor: basicMockPostProcessor,
PType: "amazon-import",
PName: "something",
PostProcessor: &HCL2PostProcessor{
PostProcessor: basicMockPostProcessor,
},
KeepInputArtifact: pTrue,
},
{
PType: "amazon-import",
PostProcessor: basicMockPostProcessor,
PType: "amazon-import",
PostProcessor: &HCL2PostProcessor{
PostProcessor: basicMockPostProcessor,
},
},
},
},

19
hcl2template/types.source.go
View File

@ -7,6 +7,7 @@ import (
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/packer/packer"
"github.com/zclconf/go-cty/cty"
)
// SourceBlock references an HCL 'source' block.
@ -26,11 +27,23 @@ type SourceBlock struct {
LocalName string
}
func (b *SourceBlock) String() string {
func (b *SourceBlock) name() string {
if b.LocalName != "" {
return fmt.Sprintf("%s.%s", b.Type, b.LocalName)
return b.LocalName
}
return b.Name
}
func (b *SourceBlock) String() string {
return fmt.Sprintf("%s.%s", b.Type, b.name())
}
// EvalContext adds the values of the source to the passed eval context.
func (b *SourceBlock) ctyValues() map[string]cty.Value {
return map[string]cty.Value{
"type": cty.StringVal(b.Type),
"name": cty.StringVal(b.name()),
}
return fmt.Sprintf("%s.%s", b.Type, b.Name)
}
// decodeBuildSource reads a used source block from a build:

23
packer/build.go
View File

@ -6,7 +6,6 @@ import (
"log"
"sync"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/packer/helper/common"
)
@ -102,11 +101,6 @@ type CoreBuild struct {
// Indicates whether the build is already initialized before calling Prepare(..)
Prepared bool
// HCL2ProvisionerPrepare and HCL2PostProcessorsPrepare are used to interpolate any build variable by decoding and preparing
// the Provisioners and Post-Processors at runtime for HCL2 templates.
HCL2ProvisionerPrepare func(data map[string]interface{}) ([]CoreBuildProvisioner, hcl.Diagnostics)
HCL2PostProcessorsPrepare func(builderArtifact Artifact) ([]CoreBuildPostProcessor, hcl.Diagnostics)
debug bool
force bool
onError string
@ -274,12 +268,6 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi Ui) ([]Artifact, error)
})
}
if b.HCL2ProvisionerPrepare != nil {
hooks[HookProvision] = append(hooks[HookProvision], &ProvisionHook{
HCL2Prepare: b.HCL2ProvisionerPrepare,
})
}
if b.CleanupProvisioner.PType != "" {
hookedCleanupProvisioner := &HookedProvisioner{
b.CleanupProvisioner.Provisioner,
@ -315,17 +303,6 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi Ui) ([]Artifact, error)
}
errors := make([]error, 0)
if b.HCL2PostProcessorsPrepare != nil {
// For HCL2, decode and prepare Post-Processors to interpolate build variables.
postProcessors, diags := b.HCL2PostProcessorsPrepare(builderArtifact)
if diags.HasErrors() {
errors = append(errors, diags)
} else if len(postProcessors) > 0 {
b.PostProcessors = [][]CoreBuildPostProcessor{postProcessors}
}
}
keepOriginalArtifact := len(b.PostProcessors) == 0
// Run the post-processors

77
packer/provisioner.go
View File

@ -7,7 +7,6 @@ import (
"sync"
"time"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/packer/helper/common"
)
@ -41,8 +40,31 @@ type ProvisionHook struct {
// The provisioners to run as part of the hook. These should already
// be prepared (by calling Prepare) at some earlier stage.
Provisioners []*HookedProvisioner
}
HCL2Prepare func(data map[string]interface{}) ([]CoreBuildProvisioner, hcl.Diagnostics)
// BuilderDataCommonKeys is the list of common keys that all builder will
// return
var BuilderDataCommonKeys = []string{
"ID",
// The following correspond to communicator-agnostic functions that are }
// part of the SSH and WinRM communicator implementations. These functions
// are not part of the communicator interface, but are stored on the
// Communicator Config and return the appropriate values rather than
// depending on the actual communicator config values. E.g "Password"
// reprosents either WinRMPassword or SSHPassword, which makes this more
// useful if a template contains multiple builds.
"Host",
"Port",
"User",
"Password",
"ConnType",
"PackerRunUUID",
"PackerHTTPPort",
"PackerHTTPIP",
"PackerHTTPAddr",
"SSHPublicKey",
"SSHPrivateKey",
"WinRMPassword",
}
// Provisioners interpolate most of their fields in the prepare stage; this
@ -56,26 +78,9 @@ type ProvisionHook struct {
// data.
func BasicPlaceholderData() map[string]string {
placeholderData := map[string]string{}
msg := "Build_%s. " + common.PlaceholderMsg
placeholderData["ID"] = fmt.Sprintf(msg, "ID")
// The following correspond to communicator-agnostic functions that are
// part of the SSH and WinRM communicator implementations. These functions
// are not part of the communicator interface, but are stored on the
// Communicator Config and return the appropriate values rather than
// depending on the actual communicator config values. E.g "Password"
// reprosents either WinRMPassword or SSHPassword, which makes this more
// useful if a template contains multiple builds.
placeholderData["Host"] = fmt.Sprintf(msg, "Host")
placeholderData["Port"] = fmt.Sprintf(msg, "Port")
placeholderData["User"] = fmt.Sprintf(msg, "User")
placeholderData["Password"] = fmt.Sprintf(msg, "Password")
placeholderData["ConnType"] = fmt.Sprintf(msg, "Type")
placeholderData["PackerRunUUID"] = fmt.Sprintf(msg, "PackerRunUUID")
placeholderData["PackerHTTPPort"] = fmt.Sprintf(msg, "PackerHTTPPort")
placeholderData["PackerHTTPIP"] = fmt.Sprintf(msg, "PackerHTTPIP")
placeholderData["PackerHTTPAddr"] = fmt.Sprintf(msg, "PackerHTTPAddr")
placeholderData["SSHPublicKey"] = fmt.Sprintf(msg, "SSHPublicKey")
placeholderData["SSHPrivateKey"] = fmt.Sprintf(msg, "SSHPrivateKey")
for _, key := range BuilderDataCommonKeys {
placeholderData[key] = fmt.Sprintf("Build_%s. "+common.PlaceholderMsg, key)
}
// Backwards-compatability: WinRM Password can get through without forcing
// the generated func validation.
@ -116,7 +121,7 @@ func CastDataToMap(data interface{}) map[string]interface{} {
// Runs the provisioners in order.
func (h *ProvisionHook) Run(ctx context.Context, name string, ui Ui, comm Communicator, data interface{}) error {
// Shortcut
if len(h.Provisioners) == 0 && h.HCL2Prepare == nil {
if len(h.Provisioners) == 0 {
return nil
}
@ -126,34 +131,10 @@ func (h *ProvisionHook) Run(ctx context.Context, name string, ui Ui, comm Commun
"`communicator` config was set to \"none\". If you have any provisioners\n" +
"then a communicator is required. Please fix this to continue.")
}
cast := CastDataToMap(data)
if h.HCL2Prepare != nil {
// For HCL2, decode and prepare Provisioners now to interpolate build variables
coreP, diags := h.HCL2Prepare(cast)
if diags.HasErrors() {
return diags
}
hookedProvisioners := make([]*HookedProvisioner, len(coreP))
for i, p := range coreP {
var pConfig interface{}
if len(p.config) > 0 {
pConfig = p.config[0]
}
hookedProvisioners[i] = &HookedProvisioner{
Provisioner: p.Provisioner,
Config: pConfig,
TypeName: p.PType,
}
}
h.Provisioners = hookedProvisioners
}
for _, p := range h.Provisioners {
ts := CheckpointReporter.AddSpan(p.TypeName, "provisioner", p.Config)
cast := CastDataToMap(data)
err := p.Provisioner.Provision(ctx, ui, comm, cast)
ts.End(err)