From 774c5903f6b4f6a5c6a862afedbf51674e495b95 Mon Sep 17 00:00:00 2001 From: Sylvia Moss Date: Thu, 11 Feb 2021 10:23:15 +0100 Subject: [PATCH] Add error-cleanup-provisioner to HCL2 (#10604) --- command/hcl2_upgrade.go | 53 +++++++------ command/hcl2_upgrade_test.go | 1 + .../expected.pkr.hcl | 34 +++++++++ .../error-cleanup-provisioner/input.json | 18 +++++ hcl2template/plugin.go | 13 ++++ .../two-error-cleanup-provisioner.pkr.hcl | 15 ++++ hcl2template/testdata/complete/build.pkr.hcl | 52 +++++++++++++ hcl2template/types.build.go | 24 ++++++ hcl2template/types.build_test.go | 27 +++++++ hcl2template/types.packer_config.go | 74 ++++++++++++------- hcl2template/types.packer_config_test.go | 18 +++++ .../blocks/build/provisioner.mdx | 35 ++++++++- 12 files changed, 316 insertions(+), 48 deletions(-) create mode 100644 command/test-fixtures/hcl2_upgrade/error-cleanup-provisioner/expected.pkr.hcl create mode 100644 command/test-fixtures/hcl2_upgrade/error-cleanup-provisioner/input.json create mode 100644 hcl2template/testdata/build/two-error-cleanup-provisioner.pkr.hcl diff --git a/command/hcl2_upgrade.go b/command/hcl2_upgrade.go index c42d503f3..222d82053 100644 --- a/command/hcl2_upgrade.go +++ b/command/hcl2_upgrade.go @@ -273,27 +273,15 @@ func (c *HCL2UpgradeCommand) RunContext(_ context.Context, cla *HCL2UpgradeArgs) // Output provisioners section provisionersOut := []byte{} for _, provisioner := range tpl.Provisioners { - provisionerContent := hclwrite.NewEmptyFile() - body := provisionerContent.Body() - buildBody.AppendNewline() - block := body.AppendNewBlock("provisioner", []string{provisioner.Type}) - cfg := provisioner.Config - if len(provisioner.Except) > 0 { - cfg["except"] = provisioner.Except - } - if len(provisioner.Only) > 0 { - cfg["only"] = provisioner.Only - } - if provisioner.MaxRetries != "" { - cfg["max_retries"] = provisioner.MaxRetries - } - if provisioner.Timeout > 0 { - cfg["timeout"] = provisioner.Timeout.String() - } - jsonBodyToHCL2Body(block.Body(), cfg) + contentBytes := c.writeProvisioner("provisioner", provisioner) + provisionersOut = append(provisionersOut, transposeTemplatingCalls(contentBytes)...) + } - provisionersOut = append(provisionersOut, transposeTemplatingCalls(provisionerContent.Bytes())...) + if tpl.CleanupProvisioner != nil { + buildBody.AppendNewline() + contentBytes := c.writeProvisioner("error-cleanup-provisioner", tpl.CleanupProvisioner) + provisionersOut = append(provisionersOut, transposeTemplatingCalls(contentBytes)...) } // Output post-processors section @@ -360,8 +348,10 @@ func (c *HCL2UpgradeCommand) RunContext(_ context.Context, cla *HCL2UpgradeArgs) out.Write(fileContent.Bytes()) } - out.Write([]byte(inputVarHeader)) - out.Write(variablesOut) + if len(variablesOut) > 0 { + out.Write([]byte(inputVarHeader)) + out.Write(variablesOut) + } if len(amazonSecretsManagerMap) > 0 { out.Write([]byte(amazonSecretsManagerDataHeader)) @@ -401,6 +391,27 @@ func (c *HCL2UpgradeCommand) RunContext(_ context.Context, cla *HCL2UpgradeArgs) return 0 } +func (c *HCL2UpgradeCommand) writeProvisioner(typeName string, provisioner *template.Provisioner) []byte { + provisionerContent := hclwrite.NewEmptyFile() + body := provisionerContent.Body() + block := body.AppendNewBlock(typeName, []string{provisioner.Type}) + cfg := provisioner.Config + if len(provisioner.Except) > 0 { + cfg["except"] = provisioner.Except + } + if len(provisioner.Only) > 0 { + cfg["only"] = provisioner.Only + } + if provisioner.MaxRetries != "" { + cfg["max_retries"] = provisioner.MaxRetries + } + if provisioner.Timeout > 0 { + cfg["timeout"] = provisioner.Timeout.String() + } + jsonBodyToHCL2Body(block.Body(), cfg) + return provisionerContent.Bytes() +} + func (c *HCL2UpgradeCommand) writeAmazonAmiDatasource(builders []*template.Builder) ([]byte, error) { amazonAmiOut := []byte{} amazonAmiFilters := []map[string]interface{}{} diff --git a/command/hcl2_upgrade_test.go b/command/hcl2_upgrade_test.go index 0abfaafcd..bd8c3148e 100644 --- a/command/hcl2_upgrade_test.go +++ b/command/hcl2_upgrade_test.go @@ -22,6 +22,7 @@ func Test_hcl2_upgrade(t *testing.T) { {"complete"}, {"minimal"}, {"source-name"}, + {"error-cleanup-provisioner"}, } for _, tc := range tc { diff --git a/command/test-fixtures/hcl2_upgrade/error-cleanup-provisioner/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade/error-cleanup-provisioner/expected.pkr.hcl new file mode 100644 index 000000000..99fa3d653 --- /dev/null +++ b/command/test-fixtures/hcl2_upgrade/error-cleanup-provisioner/expected.pkr.hcl @@ -0,0 +1,34 @@ +# This file was autogenerated by the 'packer hcl2_upgrade' command. We +# recommend double checking that everything is correct before going forward. We +# also recommend treating this file as disposable. The HCL2 blocks in this +# file can be moved to other files. For example, the variable blocks could be +# moved to their own 'variables.pkr.hcl' file, etc. Those files need to be +# suffixed with '.pkr.hcl' to be visible to Packer. To use multiple files at +# once they also need to be in the same folder. 'packer inspect folder/' +# will describe to you what is in that folder. + +# Avoid mixing go templating calls ( for example ```{{ upper(`string`) }}``` ) +# and HCL2 calls (for example '${ var.string_value_example }' ). They won't be +# executed together and the outcome will be unknown. + +# source blocks are generated from your builders; a source can be referenced in +# build blocks. A build block runs provisioner and post-processors on a +# source. Read the documentation for source blocks here: +# https://www.packer.io/docs/templates/hcl_templates/blocks/source +source "null" "autogenerated_1" { + communicator = "none" +} + +# a build block invokes sources and runs provisioning steps on them. The +# documentation for build blocks can be found here: +# https://www.packer.io/docs/templates/hcl_templates/blocks/build +build { + sources = ["source.null.autogenerated_1"] + + provisioner "shell-local" { + inline = ["exit 2"] + } + error-cleanup-provisioner "shell-local" { + inline = ["echo 'rubber ducky'> ducky.txt"] + } +} diff --git a/command/test-fixtures/hcl2_upgrade/error-cleanup-provisioner/input.json b/command/test-fixtures/hcl2_upgrade/error-cleanup-provisioner/input.json new file mode 100644 index 000000000..971d9f5b4 --- /dev/null +++ b/command/test-fixtures/hcl2_upgrade/error-cleanup-provisioner/input.json @@ -0,0 +1,18 @@ +{ + "builders": [ + { + "type": "null", + "communicator": "none" + } + ], + "provisioners": [ + { + "type": "shell-local", + "inline": ["exit 2"] + } + ], + "error-cleanup-provisioner": { + "type": "shell-local", + "inline": ["echo 'rubber ducky'> ducky.txt"] + } +} \ No newline at end of file diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index 5d3a4c1fc..4b1e81859 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -162,6 +162,19 @@ func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics { provBlock.HCL2Ref.Rest = dynblock.Expand(provBlock.HCL2Ref.Rest, cfg.EvalContext(BuildContext, nil)) } + if build.ErrorCleanupProvisionerBlock != nil { + if !cfg.parser.PluginConfig.Provisioners.Has(build.ErrorCleanupProvisionerBlock.PType) { + diags = append(diags, &hcl.Diagnostic{ + Summary: fmt.Sprintf("Unknown "+buildErrorCleanupProvisionerLabel+" type %q", build.ErrorCleanupProvisionerBlock.PType), + Subject: build.ErrorCleanupProvisionerBlock.HCL2Ref.TypeRange.Ptr(), + Detail: fmt.Sprintf("known "+buildErrorCleanupProvisionerLabel+"s: %v", cfg.parser.PluginConfig.Provisioners.List()), + Severity: hcl.DiagError, + }) + } + // Allow rest of the body to have dynamic blocks + build.ErrorCleanupProvisionerBlock.HCL2Ref.Rest = dynblock.Expand(build.ErrorCleanupProvisionerBlock.HCL2Ref.Rest, cfg.EvalContext(BuildContext, nil)) + } + for _, ppList := range build.PostProcessorsLists { for _, ppBlock := range ppList { if !cfg.parser.PluginConfig.PostProcessors.Has(ppBlock.PType) { diff --git a/hcl2template/testdata/build/two-error-cleanup-provisioner.pkr.hcl b/hcl2template/testdata/build/two-error-cleanup-provisioner.pkr.hcl new file mode 100644 index 000000000..588bcf054 --- /dev/null +++ b/hcl2template/testdata/build/two-error-cleanup-provisioner.pkr.hcl @@ -0,0 +1,15 @@ +source "virtualbox-iso" "ubuntu-1204" { +} + +// starts resources to provision them. +build { + sources = [ + "source.virtualbox-iso.ubuntu-1204" + ] + + error-cleanup-provisioner "shell-local" { + } + + error-cleanup-provisioner "file" { + } +} diff --git a/hcl2template/testdata/complete/build.pkr.hcl b/hcl2template/testdata/complete/build.pkr.hcl index 3ec3a6177..9f0f77f23 100644 --- a/hcl2template/testdata/complete/build.pkr.hcl +++ b/hcl2template/testdata/complete/build.pkr.hcl @@ -129,6 +129,58 @@ build { } } + error-cleanup-provisioner "shell" { + name = "error-cleanup-provisioner that does something" + not_squashed = "${var.foo} ${upper(build.ID)}" + string = "string" + int = "${41 + 1}" + int64 = "${42 + 1}" + bool = "true" + trilean = true + duration = "${9 + 1}s" + map_string_string = { + a = "b" + c = "d" + } + slice_string = [for s in var.availability_zone_names : lower(s)] + slice_slice_string = [ + ["a","b"], + ["c","d"] + ] + + nested { + string = "string" + int = 42 + int64 = 43 + bool = true + trilean = true + duration = "10s" + map_string_string = { + a = "b" + c = "d" + } + slice_string = [for s in var.availability_zone_names : lower(s)] + slice_slice_string = [ + ["a","b"], + ["c","d"] + ] + } + + nested_slice { + tag { + key = "first_tag_key" + value = "first_tag_value" + } + dynamic "tag" { + for_each = local.standard_tags + content { + key = tag.key + value = tag.value + } + } + } + } + post-processor "amazon-import" { name = "something" string = "string" diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index 3571c1257..30e5405ca 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -1,6 +1,8 @@ package hcl2template import ( + "fmt" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" @@ -13,6 +15,8 @@ const ( buildProvisionerLabel = "provisioner" + buildErrorCleanupProvisionerLabel = "error-cleanup-provisioner" + buildPostProcessorLabel = "post-processor" buildPostProcessorsLabel = "post-processors" @@ -23,6 +27,7 @@ var buildSchema = &hcl.BodySchema{ {Type: buildFromLabel, LabelNames: []string{"type"}}, {Type: sourceLabel, LabelNames: []string{"reference"}}, {Type: buildProvisionerLabel, LabelNames: []string{"type"}}, + {Type: buildErrorCleanupProvisionerLabel, LabelNames: []string{"type"}}, {Type: buildPostProcessorLabel, LabelNames: []string{"type"}}, {Type: buildPostProcessorsLabel, LabelNames: []string{}}, }, @@ -58,6 +63,10 @@ type BuildBlock struct { // will be ran against the sources. ProvisionerBlocks []*ProvisionerBlock + // ErrorCleanupProvisionerBlock references a special provisioner block that + // will be ran only if the provision step fails. + ErrorCleanupProvisionerBlock *ProvisionerBlock + // PostProcessorLists references the lists of lists of HCL post-processors // block that will be run against the artifacts from the provisioning // steps. @@ -130,6 +139,21 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB continue } build.ProvisionerBlocks = append(build.ProvisionerBlocks, p) + case buildErrorCleanupProvisionerLabel: + if build.ErrorCleanupProvisionerBlock != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Only one " + buildErrorCleanupProvisionerLabel + " is allowed"), + Subject: block.DefRange.Ptr(), + }) + continue + } + p, moreDiags := p.decodeProvisioner(block, cfg) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + continue + } + build.ErrorCleanupProvisionerBlock = p case buildPostProcessorLabel: pp, moreDiags := p.decodePostProcessor(block) diags = append(diags, moreDiags...) diff --git a/hcl2template/types.build_test.go b/hcl2template/types.build_test.go index 3a0083584..b2833375c 100644 --- a/hcl2template/types.build_test.go +++ b/hcl2template/types.build_test.go @@ -88,6 +88,33 @@ func TestParse_build(t *testing.T) { }}, false, }, + {"two error-cleanup-provisioner", + defaultParser, + parseTestArgs{"testdata/build/two-error-cleanup-provisioner.pkr.hcl", nil, nil}, + &PackerConfig{ + CorePackerVersionString: lockedVersion, + Basedir: filepath.Join("testdata", "build"), + Sources: map[SourceRef]SourceBlock{ + refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, + }, + }, + true, true, + []packersdk.Build{&packer.CoreBuild{ + Builder: emptyMockBuilder, + CleanupProvisioner: packer.CoreBuildProvisioner{ + PType: "shell-local", + Provisioner: &HCL2Provisioner{ + Provisioner: &MockProvisioner{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, + NestedSlice: []NestedMockConfig{}, + }, + }, + }, + }, + }}, + false, + }, {"untyped post-processor", defaultParser, parseTestArgs{"testdata/build/post-processor_untyped.pkr.hcl", nil, nil}, diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 1e458d1a6..1b9762982 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -323,40 +323,51 @@ func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceUseBlock, blocks if pb.OnlyExcept.Skip(source.String()) { continue } - provisioner, moreDiags := cfg.startProvisioner(source, pb, ectx) + + coreBuildProv, moreDiags := cfg.getCoreBuildProvisioner(source, pb, ectx) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue } - - // If we're pausing, we wrap the provisioner in a special pauser. - if pb.PauseBefore != 0 { - provisioner = &packer.PausedProvisioner{ - PauseBefore: pb.PauseBefore, - Provisioner: provisioner, - } - } else if pb.Timeout != 0 { - provisioner = &packer.TimeoutProvisioner{ - Timeout: pb.Timeout, - Provisioner: provisioner, - } - } - if pb.MaxRetries != 0 { - provisioner = &packer.RetriedProvisioner{ - MaxRetries: pb.MaxRetries, - Provisioner: provisioner, - } - } - - res = append(res, packer.CoreBuildProvisioner{ - PType: pb.PType, - PName: pb.PName, - Provisioner: provisioner, - }) + res = append(res, coreBuildProv) } return res, diags } +func (cfg *PackerConfig) getCoreBuildProvisioner(source SourceUseBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext) (packer.CoreBuildProvisioner, hcl.Diagnostics) { + var diags hcl.Diagnostics + provisioner, moreDiags := cfg.startProvisioner(source, pb, ectx) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + return packer.CoreBuildProvisioner{}, diags + } + + // If we're pausing, we wrap the provisioner in a special pauser. + if pb.PauseBefore != 0 { + provisioner = &packer.PausedProvisioner{ + PauseBefore: pb.PauseBefore, + Provisioner: provisioner, + } + } else if pb.Timeout != 0 { + provisioner = &packer.TimeoutProvisioner{ + Timeout: pb.Timeout, + Provisioner: provisioner, + } + } + if pb.MaxRetries != 0 { + provisioner = &packer.RetriedProvisioner{ + MaxRetries: pb.MaxRetries, + Provisioner: provisioner, + } + } + + return packer.CoreBuildProvisioner{ + PType: pb.PType, + PName: pb.PName, + Provisioner: provisioner, + }, diags +} + // getCoreBuildProvisioners takes a list of post processor block, starts // according provisioners and sends parsed HCL2 over to it. func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceUseBlock, blocksList [][]*PostProcessorBlock, ectx *hcl.EvalContext) ([][]packer.CoreBuildPostProcessor, hcl.Diagnostics) { @@ -507,6 +518,17 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packersdk.Bu continue } + if build.ErrorCleanupProvisionerBlock != nil { + if !build.ErrorCleanupProvisionerBlock.OnlyExcept.Skip(srcUsage.String()) { + errorCleanupProv, moreDiags := cfg.getCoreBuildProvisioner(srcUsage, build.ErrorCleanupProvisionerBlock, cfg.EvalContext(BuildContext, variables)) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + continue + } + pcb.CleanupProvisioner = errorCleanupProv + } + } + pcb.Builder = builder pcb.Provisioners = provisioners pcb.PostProcessors = pps diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index 887cc8b4b..87c7825bf 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -157,6 +157,10 @@ func TestParser_complete(t *testing.T) { }, {PType: "file"}, }, + ErrorCleanupProvisionerBlock: &ProvisionerBlock{ + PType: "shell", + PName: "error-cleanup-provisioner that does something", + }, PostProcessorsLists: [][]*PostProcessorBlock{ { { @@ -215,6 +219,13 @@ func TestParser_complete(t *testing.T) { }, }, }, + CleanupProvisioner: packer.CoreBuildProvisioner{ + PType: "shell", + PName: "error-cleanup-provisioner that does something", + Provisioner: &HCL2Provisioner{ + Provisioner: basicMockProvisioner, + }, + }, PostProcessors: [][]packer.CoreBuildPostProcessor{ { { @@ -309,6 +320,13 @@ func TestParser_complete(t *testing.T) { }, }, }, + CleanupProvisioner: packer.CoreBuildProvisioner{ + PType: "shell", + PName: "error-cleanup-provisioner that does something", + Provisioner: &HCL2Provisioner{ + Provisioner: basicMockProvisioner, + }, + }, PostProcessors: [][]packer.CoreBuildPostProcessor{ { { diff --git a/website/content/docs/templates/hcl_templates/blocks/build/provisioner.mdx b/website/content/docs/templates/hcl_templates/blocks/build/provisioner.mdx index eab09a689..8c37560a3 100644 --- a/website/content/docs/templates/hcl_templates/blocks/build/provisioner.mdx +++ b/website/content/docs/templates/hcl_templates/blocks/build/provisioner.mdx @@ -30,7 +30,7 @@ machine image after booting. Provisioners prepare the system for use. The list of available provisioners can be found in the [provisioners](/docs/provisioners) section. -# Run on Specific Builds +## Run on Specific Builds You can use the `only` or `except` configurations to run a provisioner only with specific builds. These two configurations do what you expect: `only` will @@ -76,6 +76,39 @@ build { The values within `only` or `except` are _build names_, not builder types. +## On Error Provisioner + +You can optionally create a single specialized provisioner called an +`error-cleanup-provisioner`. This provisioner will not run unless the normal +provisioning run fails. If the normal provisioning run does fail, this special +error provisioner will run _before the instance is shut down_. This allows you +to make last minute changes and clean up behaviors that Packer may not be able +to clean up on its own. + +For examples, users may use this provisioner to make sure that the instance is +properly unsubscribed from any services that it connected to during the build +run. + +Toy usage example for the error cleanup script: + +```hcl +source "null" "example" { + communicator = "none" +} + +build { + sources = ["source.null.example"] + + provisioner "shell-local" { + inline = ["exit 2"] + } + + error-cleanup-provisioner "shell-local" { + inline = ["echo 'rubber ducky'> ducky.txt"] + } +} +``` + ## Pausing Before Running With certain provisioners it is sometimes desirable to pause for some period of