diff --git a/command/hcl2_upgrade.go b/command/hcl2_upgrade.go index 1eafeb93e..24e0f3f37 100644 --- a/command/hcl2_upgrade.go +++ b/command/hcl2_upgrade.go @@ -74,9 +74,12 @@ const ( # All generated input variables will be of 'string' type as this is how Packer JSON # views them; you can change their type later on. Read the variables type # constraints documentation -# https://www.packer.io/docs/templates/hcl_templates/variables#type-constraints for more info. -` - +# https://www.packer.io/docs/templates/hcl_templates/variables#type-constraints for more info.` + localsVarHeader = ` +# All locals variables are generated from variables that uses expressions +# that are not allowed in HCL2 variables. +# Read the documentation for locals blocks here: +# https://www.packer.io/docs/templates/hcl_templates/blocks/locals` packerBlockHeader = ` # See https://www.packer.io/docs/templates/hcl_templates/blocks/packer for more info ` @@ -93,15 +96,28 @@ const ( # https://www.packer.io/docs/templates/hcl_templates/blocks/build build { ` + amazonAmiDataHeader = ` # The amazon-ami data block is generated from your amazon builder source_ami_filter; a data # from this block can be referenced in source and locals blocks. # Read the documentation for data blocks here: -# https://www.packer.io/docs/templates/hcl_templates/blocks/data` +# https://www.packer.io/docs/templates/hcl_templates/blocks/data +# Read the documentation for the Amazon AMI Data Source here: +# https://www.packer.io/docs/datasources/amazon/ami` + + amazonSecretsManagerDataHeader = ` +# The amazon-secretsmanager data block is generated from your aws_secretsmanager template function; a data +# from this block can be referenced in source and locals blocks. +# Read the documentation for data blocks here: +# https://www.packer.io/docs/templates/hcl_templates/blocks/data +# Read the documentation for the Amazon Secrets Manager Data Source here: +# https://www.packer.io/docs/datasources/amazon/secretsmanager` ) +var amazonSecretsManagerMap = map[string]map[string]interface{}{} +var localsVariableMap = map[string]string{} + func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2UpgradeArgs) int { - out := &bytes.Buffer{} var output io.Writer if err := os.MkdirAll(filepath.Dir(cla.OutputFile), 0); err != nil { c.Ui.Error(fmt.Sprintf("Failed to create output directory: %v", err)) @@ -131,20 +147,16 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra } tpl := core.Template - // Packer section - if tpl.MinVersion != "" { - out.Write([]byte(packerBlockHeader)) - fileContent := hclwrite.NewEmptyFile() - body := fileContent.Body() - packerBody := body.AppendNewBlock("packer", nil).Body() - packerBody.SetAttributeValue("required_version", cty.StringVal(fmt.Sprintf(">= %s", tpl.MinVersion))) - out.Write(fileContent.Bytes()) - } + // OutPut Locals and Local blocks + localsContent := hclwrite.NewEmptyFile() + localsBody := localsContent.Body() + localsBody.AppendNewline() + localBody := localsBody.AppendNewBlock("locals", nil).Body() - out.Write([]byte(inputVarHeader)) + localsOut := []byte{} // Output variables section - + variablesOut := []byte{} variables := []*template.Variable{} { // sort variables to avoid map's randomness @@ -160,24 +172,40 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra for _, variable := range variables { variablesContent := hclwrite.NewEmptyFile() variablesBody := variablesContent.Body() - + variablesBody.AppendNewline() variableBody := variablesBody.AppendNewBlock("variable", []string{variable.Key}).Body() variableBody.SetAttributeRaw("type", hclwrite.Tokens{&hclwrite.Token{Bytes: []byte("string")}}) if variable.Default != "" || !variable.Required { variableBody.SetAttributeValue("default", hcl2shim.HCL2ValueFromConfigValue(variable.Default)) } + sensitive := false if isSensitiveVariable(variable.Key, tpl.SensitiveVariables) { + sensitive = true variableBody.SetAttributeValue("sensitive", cty.BoolVal(true)) } - variablesBody.AppendNewline() - out.Write(transposeTemplatingCalls(variablesContent.Bytes())) + isLocal, out := variableTransposeTemplatingCalls(variablesContent.Bytes()) + if isLocal { + if sensitive { + // Create Local block because this is sensitive + localContent := hclwrite.NewEmptyFile() + body := localContent.Body() + body.AppendNewline() + localBody := body.AppendNewBlock("local", []string{variable.Key}).Body() + localBody.SetAttributeValue("sensitive", cty.BoolVal(true)) + localBody.SetAttributeValue("expression", hcl2shim.HCL2ValueFromConfigValue(variable.Default)) + localsOut = append(localsOut, transposeTemplatingCalls(localContent.Bytes())...) + localsVariableMap[variable.Key] = "local" + continue + } + localBody.SetAttributeValue(variable.Key, hcl2shim.HCL2ValueFromConfigValue(variable.Default)) + localsVariableMap[variable.Key] = "locals" + continue + } + variablesOut = append(variablesOut, out...) } - fmt.Fprintln(out, `# "timestamp" template function replacement`) - fmt.Fprintln(out, `locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }`) - - // Output sources section + localsOut = append(localsOut, transposeTemplatingCalls(localsContent.Bytes())...) builders := []*template.Builder{} { @@ -187,7 +215,9 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra } } - if err := c.writeAmazonAmiDatasource(builders, out); err != nil { + // Output amazon-ami data source section + amazonAmiOut, err := c.writeAmazonAmiDatasource(builders) + if err != nil { return 1 } @@ -195,8 +225,8 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra return builders[i].Type+builders[i].Name < builders[j].Type+builders[j].Name }) - out.Write([]byte(sourcesHeader)) - + // Output sources section + sourcesOut := []byte{} for i, builderCfg := range builders { sourcesContent := hclwrite.NewEmptyFile() body := sourcesContent.Body() @@ -213,12 +243,10 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra jsonBodyToHCL2Body(sourceBody, builderCfg.Config) - _, _ = out.Write(transposeTemplatingCalls(sourcesContent.Bytes())) + sourcesOut = append(sourcesOut, transposeTemplatingCalls(sourcesContent.Bytes())...) } // Output build section - out.Write([]byte(buildHeader)) - buildContent := hclwrite.NewEmptyFile() buildBody := buildContent.Body() if tpl.Description != "" { @@ -232,8 +260,10 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra } buildBody.SetAttributeValue("sources", hcl2shim.HCL2ValueFromConfigValue(sourceNames)) buildBody.AppendNewline() - _, _ = buildContent.WriteTo(out) + buildOut := buildContent.Bytes() + // Output provisioners section + provisionersOut := []byte{} for _, provisioner := range tpl.Provisioners { provisionerContent := hclwrite.NewEmptyFile() body := provisionerContent.Body() @@ -255,8 +285,11 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra } jsonBodyToHCL2Body(block.Body(), cfg) - out.Write(transposeTemplatingCalls(provisionerContent.Bytes())) + provisionersOut = append(provisionersOut, transposeTemplatingCalls(provisionerContent.Bytes())...) } + + // Output post-processors section + postProcessorsOut := []byte{} for _, pps := range tpl.PostProcessors { postProcessorContent := hclwrite.NewEmptyFile() body := postProcessorContent.Body() @@ -286,9 +319,69 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra jsonBodyToHCL2Body(ppBody, cfg) } - _, _ = out.Write(transposeTemplatingCalls(postProcessorContent.Bytes())) + postProcessorsOut = append(postProcessorsOut, transposeTemplatingCalls(postProcessorContent.Bytes())...) } + // Output amazon-secretsmanager data source section + keys := make([]string, 0, len(amazonSecretsManagerMap)) + for k := range amazonSecretsManagerMap { + keys = append(keys, k) + } + sort.Strings(keys) + + amazonSecretsDataOut := []byte{} + for _, dataSourceName := range keys { + datasourceContent := hclwrite.NewEmptyFile() + body := datasourceContent.Body() + body.AppendNewline() + datasourceBody := body.AppendNewBlock("data", []string{"amazon-secretsmanager", dataSourceName}).Body() + jsonBodyToHCL2Body(datasourceBody, amazonSecretsManagerMap[dataSourceName]) + amazonSecretsDataOut = append(amazonSecretsDataOut, datasourceContent.Bytes()...) + } + + // Write file + out := &bytes.Buffer{} + + // Packer section + if tpl.MinVersion != "" { + out.Write([]byte(packerBlockHeader)) + fileContent := hclwrite.NewEmptyFile() + body := fileContent.Body() + packerBody := body.AppendNewBlock("packer", nil).Body() + packerBody.SetAttributeValue("required_version", cty.StringVal(fmt.Sprintf(">= %s", tpl.MinVersion))) + out.Write(fileContent.Bytes()) + } + + out.Write([]byte(inputVarHeader)) + out.Write(variablesOut) + + if len(amazonSecretsManagerMap) > 0 { + out.Write([]byte(amazonSecretsManagerDataHeader)) + out.Write(amazonSecretsDataOut) + } + + if len(amazonAmiOut) > 0 { + out.Write([]byte(amazonAmiDataHeader)) + out.Write(amazonAmiOut) + } + + _, _ = out.Write([]byte("\n")) + fmt.Fprintln(out, `# "timestamp" template function replacement`) + fmt.Fprintln(out, `locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }`) + + if len(localsOut) > 0 { + out.Write([]byte(localsVarHeader)) + out.Write(localsOut) + } + + out.Write([]byte(sourcesHeader)) + out.Write(sourcesOut) + + out.Write([]byte(buildHeader)) + out.Write(buildOut) + out.Write(provisionersOut) + out.Write(postProcessorsOut) + _, _ = out.Write([]byte("}\n")) _, _ = output.Write(hclwrite.Format(out.Bytes())) @@ -298,9 +391,9 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra return 0 } -func (c *HCL2UpgradeCommand) writeAmazonAmiDatasource(builders []*template.Builder, out *bytes.Buffer) error { +func (c *HCL2UpgradeCommand) writeAmazonAmiDatasource(builders []*template.Builder) ([]byte, error) { + amazonAmiOut := []byte{} amazonAmiFilters := []map[string]interface{}{} - first := true i := 1 for _, builder := range builders { if strings.HasPrefix(builder.Type, "amazon-") { @@ -308,7 +401,7 @@ func (c *HCL2UpgradeCommand) writeAmazonAmiDatasource(builders []*template.Build sourceAmiFilterCfg := map[string]interface{}{} if err := mapstructure.Decode(sourceAmiFilter, &sourceAmiFilterCfg); err != nil { c.Ui.Error(fmt.Sprintf("Failed to write amazon-ami data source: %v", err)) - return err + return nil, err } duplicate := false @@ -336,21 +429,17 @@ func (c *HCL2UpgradeCommand) writeAmazonAmiDatasource(builders []*template.Build builder.Config["source_ami"] = sourceAmiDataRef i++ - if first { - out.Write([]byte(amazonAmiDataHeader)) - first = false - } datasourceContent := hclwrite.NewEmptyFile() body := datasourceContent.Body() body.AppendNewline() sourceBody := body.AppendNewBlock("data", []string{"amazon-ami", dataSourceName}).Body() jsonBodyToHCL2Body(sourceBody, sourceAmiFilterCfg) - _, _ = out.Write(transposeTemplatingCalls(datasourceContent.Bytes())) + amazonAmiOut = append(amazonAmiOut, transposeTemplatingCalls(datasourceContent.Bytes())...) } } } - return nil + return amazonAmiOut, nil } type UnhandleableArgumentError struct { @@ -377,14 +466,71 @@ func transposeTemplatingCalls(s []byte) []byte { return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...) } - funcMap := texttemplate.FuncMap{ - "timestamp": func() string { + funcMap := templateCommonFunctionMap() + + tpl, err := texttemplate.New("hcl2_upgrade"). + Funcs(funcMap). + Parse(string(s)) + + if err != nil { + return fallbackReturn(err) + } + + str := &bytes.Buffer{} + v := struct { + HTTPIP string + HTTPPort string + }{ + HTTPIP: "{{ .HTTPIP }}", + HTTPPort: "{{ .HTTPPort }}", + } + if err := tpl.Execute(str, v); err != nil { + return fallbackReturn(err) + } + + return str.Bytes() +} + +func templateCommonFunctionMap() texttemplate.FuncMap { + return texttemplate.FuncMap{ + "aws_secretsmanager": func(a ...string) string { + if len(a) == 2 { + for key, config := range amazonSecretsManagerMap { + nameOk := config["name"] == a[0] + keyOk := config["key"] == a[1] + if nameOk && keyOk { + return fmt.Sprintf("${data.amazon-secretsmanager.%s.value}", key) + } + } + id := fmt.Sprintf("autogenerated_%d", len(amazonSecretsManagerMap)+1) + amazonSecretsManagerMap[id] = map[string]interface{}{ + "name": a[0], + "key": a[1], + } + return fmt.Sprintf("${data.amazon-secretsmanager.%s.value}", id) + } + for key, config := range amazonSecretsManagerMap { + nameOk := config["name"] == a[0] + if nameOk { + return fmt.Sprintf("${data.amazon-secretsmanager.%s.value}", key) + } + } + id := fmt.Sprintf("autogenerated_%d", len(amazonSecretsManagerMap)+1) + amazonSecretsManagerMap[id] = map[string]interface{}{ + "name": a[0], + } + return fmt.Sprintf("${data.amazon-secretsmanager.%s.value}", id) + }, "timestamp": func() string { return "${local.timestamp}" }, "isotime": func() string { return "${local.timestamp}" }, "user": func(in string) string { + if _, ok := localsVariableMap[in]; ok { + // variable is now a local + return fmt.Sprintf("${local.%s}", in) + } return fmt.Sprintf("${var.%s}", in) }, "env": func(in string) string { @@ -459,13 +605,34 @@ func transposeTemplatingCalls(s []byte) []byte { return fmt.Sprintf("${build.type}") }, } +} + +// variableTransposeTemplatingCalls executes parts of blocks as go template files and replaces +// their result with their hcl2 variant for variables block only. If something goes wrong the template +// containing the go template string is returned. +// In variableTransposeTemplatingCalls the definition of aws_secretsmanager function will create a data source +// with the same name as the variable. +func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) { + fallbackReturn := func(err error) []byte { + if strings.Contains(err.Error(), "unhandled") { + return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...) + } + + return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...) + } + + funcMap := templateCommonFunctionMap() + funcMap["aws_secretsmanager"] = func(a ...string) string { + isLocal = true + return "" + } tpl, err := texttemplate.New("hcl2_upgrade"). Funcs(funcMap). Parse(string(s)) if err != nil { - return fallbackReturn(err) + return isLocal, fallbackReturn(err) } str := &bytes.Buffer{} @@ -477,10 +644,10 @@ func transposeTemplatingCalls(s []byte) []byte { HTTPPort: "{{ .HTTPPort }}", } if err := tpl.Execute(str, v); err != nil { - return fallbackReturn(err) + return isLocal, fallbackReturn(err) } - return str.Bytes() + return isLocal, str.Bytes() } func jsonBodyToHCL2Body(out *hclwrite.Body, kvs map[string]interface{}) { diff --git a/command/test-fixtures/hcl2_upgrade_basic/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade_basic/expected.pkr.hcl index 59961b1f6..aaa0bdd7b 100644 --- a/command/test-fixtures/hcl2_upgrade_basic/expected.pkr.hcl +++ b/command/test-fixtures/hcl2_upgrade_basic/expected.pkr.hcl @@ -47,13 +47,36 @@ variable "secret_account" { sensitive = true } -# "timestamp" template function replacement -locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } +# The amazon-secretsmanager data block is generated from your aws_secretsmanager template function; a data +# from this block can be referenced in source and locals blocks. +# Read the documentation for data blocks here: +# https://www.packer.io/docs/templates/hcl_templates/blocks/data +# Read the documentation for the Amazon Secrets Manager Data Source here: +# https://www.packer.io/docs/datasources/amazon/secretsmanager +data "amazon-secretsmanager" "autogenerated_1" { + name = "sample/app/password" +} + +data "amazon-secretsmanager" "autogenerated_2" { + key = "api_key" + name = "sample/app/passwords" +} + +data "amazon-secretsmanager" "autogenerated_3" { + name = "some_secret" +} + +data "amazon-secretsmanager" "autogenerated_4" { + key = "with_key" + name = "some_secret" +} # The amazon-ami data block is generated from your amazon builder source_ami_filter; a data # from this block can be referenced in source and locals blocks. # Read the documentation for data blocks here: # https://www.packer.io/docs/templates/hcl_templates/blocks/data +# Read the documentation for the Amazon AMI Data Source here: +# https://www.packer.io/docs/datasources/amazon/ami data "amazon-ami" "autogenerated_1" { filters = { name = "ubuntu/images/*/ubuntu-xenial-16.04-amd64-server-*" @@ -64,6 +87,22 @@ data "amazon-ami" "autogenerated_1" { owners = ["099720109477"] } +# "timestamp" template function replacement +locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } + +# All locals variables are generated from variables that uses expressions +# that are not allowed in HCL2 variables. +# Read the documentation for locals blocks here: +# https://www.packer.io/docs/templates/hcl_templates/blocks/locals +local "password" { + sensitive = true + expression = "${data.amazon-secretsmanager.autogenerated_1.value}" +} + +locals { + password_key = "MY_KEY_${data.amazon-secretsmanager.autogenerated_2.value}" +} + # source blocks are generated from your builders; a source can be referenced in # build blocks. A build block runs provisioner and post-processors on a # source. Read the documentation for source blocks here: @@ -135,6 +174,12 @@ build { inline = ["echo ${var.secret_account}", "echo ${build.ID}", "echo ${build.SSHPublicKey} | head -c 14", "echo ${path.root} is not ${path.cwd}", "echo ${packer.version}", "echo ${uuidv4()}"] max_retries = "5" } + provisioner "shell" { + inline = ["echo ${local.password}", "echo ${data.amazon-secretsmanager.autogenerated_1.value}", "echo ${local.password_key}", "echo ${data.amazon-secretsmanager.autogenerated_2.value}"] + } + provisioner "shell" { + inline = ["echo ${data.amazon-secretsmanager.autogenerated_3.value}", "echo ${data.amazon-secretsmanager.autogenerated_4.value}"] + } # template: hcl2_upgrade:2:38: executing "hcl2_upgrade" at : error calling clean_resource_name: unhandled "clean_resource_name" call: # there is no way to automatically upgrade the "clean_resource_name" call. diff --git a/command/test-fixtures/hcl2_upgrade_basic/input.json b/command/test-fixtures/hcl2_upgrade_basic/input.json index 16208ac9a..1b4eb5dec 100644 --- a/command/test-fixtures/hcl2_upgrade_basic/input.json +++ b/command/test-fixtures/hcl2_upgrade_basic/input.json @@ -5,13 +5,16 @@ "aws_region": null, "aws_secondary_region": "{{ env `AWS_DEFAULT_REGION` }}", "aws_secret_key": "", - "aws_access_key": "" + "aws_access_key": "", + "password": "{{ aws_secretsmanager `sample/app/password` }}", + "password_key": "MY_KEY_{{ aws_secretsmanager `sample/app/passwords` `api_key` }}" }, "sensitive-variables": [ "aws_secret_key", "aws_access_key", "secret_account", - "potato" + "potato", + "password" ], "builders": [ { @@ -128,6 +131,22 @@ "echo {{ uuid }}" ] }, + { + "type": "shell", + "inline": [ + "echo {{ user `password` }}", + "echo {{ aws_secretsmanager `sample/app/password` }}", + "echo {{ user `password_key` }}", + "echo {{ aws_secretsmanager `sample/app/passwords` `api_key` }}" + ] + }, + { + "type": "shell", + "inline": [ + "echo {{ aws_secretsmanager `some_secret` }}", + "echo {{ aws_secretsmanager `some_secret` `with_key` }}" + ] + }, { "type": "shell", "inline": [