Add Amazon AMI data source to hcl_upgrade command (#10491)

command/hcl2_upgrade.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"reflect"
 	"sort"
 	"strings"
 	texttemplate "text/template"
@@ -14,6 +15,7 @@ import (
 	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/hashicorp/packer-plugin-sdk/template"
 	hcl2shim "github.com/hashicorp/packer/hcl2template/shim"
+	"github.com/mitchellh/mapstructure"
 	"github.com/posener/complete"
 	"github.com/zclconf/go-cty/cty"
 )
@@ -91,10 +93,14 @@ const (
 # https://www.packer.io/docs/templates/hcl_templates/blocks/build
 build {
 `
+	amazonAmiDataHeader = `
+# The amazon-ami data block is generated from your amazon builder source_ami_filter; a data
+# from this block can be referenced in source and locals blocks.
+# Read the documentation for data blocks here:
+# https://www.packer.io/docs/templates/hcl_templates/blocks/data`
 )
 
 func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2UpgradeArgs) int {
-
 	out := &bytes.Buffer{}
 	var output io.Writer
 	if err := os.MkdirAll(filepath.Dir(cla.OutputFile), 0); err != nil {
@@ -179,11 +185,16 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra
 		for _, builder := range tpl.Builders {
 			builders = append(builders, builder)
 		}
-		sort.Slice(builders, func(i, j int) bool {
-			return builders[i].Type+builders[i].Name < builders[j].Type+builders[j].Name
-		})
 	}
 
+	if err := c.writeAmazonAmiDatasource(builders, out); err != nil {
+		return 1
+	}
+
+	sort.Slice(builders, func(i, j int) bool {
+		return builders[i].Type+builders[i].Name < builders[j].Type+builders[j].Name
+	})
+
 	out.Write([]byte(sourcesHeader))
 
 	for i, builderCfg := range builders {
@@ -287,6 +298,61 @@ func (c *HCL2UpgradeCommand) RunContext(buildCtx context.Context, cla *HCL2Upgra
 	return 0
 }
 
+func (c *HCL2UpgradeCommand) writeAmazonAmiDatasource(builders []*template.Builder, out *bytes.Buffer) error {
+	amazonAmiFilters := []map[string]interface{}{}
+	first := true
+	i := 1
+	for _, builder := range builders {
+		if strings.HasPrefix(builder.Type, "amazon-") {
+			if sourceAmiFilter, ok := builder.Config["source_ami_filter"]; ok {
+				sourceAmiFilterCfg := map[string]interface{}{}
+				if err := mapstructure.Decode(sourceAmiFilter, &sourceAmiFilterCfg); err != nil {
+					c.Ui.Error(fmt.Sprintf("Failed to write amazon-ami data source: %v", err))
+					return err
+				}
+
+				duplicate := false
+				dataSourceName := fmt.Sprintf("autogenerated_%d", i)
+				for j, filter := range amazonAmiFilters {
+					if reflect.DeepEqual(filter, sourceAmiFilter) {
+						duplicate = true
+						dataSourceName = fmt.Sprintf("autogenerated_%d", j+1)
+						continue
+					}
+				}
+
+				// This is a hack...
+				// Use templating so that it could be correctly transformed later into a data resource
+				sourceAmiDataRef := fmt.Sprintf("{{ data `amazon-ami.%s.id` }}", dataSourceName)
+
+				if duplicate {
+					delete(builder.Config, "source_ami_filter")
+					builder.Config["source_ami"] = sourceAmiDataRef
+					continue
+				}
+
+				amazonAmiFilters = append(amazonAmiFilters, sourceAmiFilterCfg)
+				delete(builder.Config, "source_ami_filter")
+				builder.Config["source_ami"] = sourceAmiDataRef
+				i++
+
+				if first {
+					out.Write([]byte(amazonAmiDataHeader))
+					first = false
+				}
+				datasourceContent := hclwrite.NewEmptyFile()
+				body := datasourceContent.Body()
+				body.AppendNewline()
+				sourceBody := body.AppendNewBlock("data", []string{"amazon-ami", dataSourceName}).Body()
+				jsonBodyToHCL2Body(sourceBody, sourceAmiFilterCfg)
+				_, _ = out.Write(transposeTemplatingCalls(datasourceContent.Bytes()))
+			}
+		}
+	}
+
+	return nil
+}
+
 type UnhandleableArgumentError struct {
 	Call           string
 	Correspondance string
@@ -327,6 +393,9 @@ func transposeTemplatingCalls(s []byte) []byte {
 		"build": func(a string) string {
 			return fmt.Sprintf("${build.%s}", a)
 		},
+		"data": func(a string) string {
+			return fmt.Sprintf("${data.%s}", a)
+		},
 		"template_dir": func() string {
 			return fmt.Sprintf("${path.root}")
 		},

command/test-fixtures/hcl2_upgrade_basic/expected.pkr.hcl

@@ -50,6 +50,20 @@ variable "secret_account" {
 # "timestamp" template function replacement
 locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
 
+# The amazon-ami data block is generated from your amazon builder source_ami_filter; a data
+# from this block can be referenced in source and locals blocks.
+# Read the documentation for data blocks here:
+# https://www.packer.io/docs/templates/hcl_templates/blocks/data
+data "amazon-ami" "autogenerated_1" {
+  filters = {
+    name                = "ubuntu/images/*/ubuntu-xenial-16.04-amd64-server-*"
+    root-device-type    = "ebs"
+    virtualization-type = "hvm"
+  }
+  most_recent = true
+  owners      = ["099720109477"]
+}
+
 # source blocks are generated from your builders; a source can be referenced in
 # build blocks. A build block runs provisioner and post-processors on a
 # source. Read the documentation for source blocks here:
@@ -65,17 +79,37 @@ source "amazon-ebs" "autogenerated_1" {
     volume_size           = 48
     volume_type           = "gp2"
   }
-  region     = "${var.aws_region}"
+  region              = "${var.aws_region}"
-  secret_key = "${var.aws_secret_key}"
+  secret_key          = "${var.aws_secret_key}"
-  source_ami_filter {
+  source_ami          = "${data.amazon-ami.autogenerated_1.id}"
-    filters = {
+  spot_instance_types = ["t2.small", "t2.medium", "t2.large"]
-      name                = "ubuntu/images/*/ubuntu-xenial-16.04-amd64-server-*"
+  spot_price          = "0.0075"
-      root-device-type    = "ebs"
+  ssh_interface       = "session_manager"
-      virtualization-type = "hvm"
+  ssh_username        = "ubuntu"
+  temporary_iam_instance_profile_policy_document {
+    Statement {
+      Action   = ["*"]
+      Effect   = "Allow"
+      Resource = ["*"]
     }
-    most_recent = true
+    Version = "2012-10-17"
-    owners      = ["099720109477"]
   }
+}
+
+source "amazon-ebs" "named_builder" {
+  access_key      = "${var.aws_access_key}"
+  ami_description = "Ubuntu 16.04 LTS - expand root partition"
+  ami_name        = "ubuntu-16-04-test-${local.timestamp}"
+  encrypt_boot    = true
+  launch_block_device_mappings {
+    delete_on_termination = true
+    device_name           = "/dev/sda1"
+    volume_size           = 48
+    volume_type           = "gp2"
+  }
+  region              = "${var.aws_region}"
+  secret_key          = "${var.aws_secret_key}"
+  source_ami          = "${data.amazon-ami.autogenerated_1.id}"
   spot_instance_types = ["t2.small", "t2.medium", "t2.large"]
   spot_price          = "0.0075"
   ssh_interface       = "session_manager"
@@ -94,7 +128,7 @@ source "amazon-ebs" "autogenerated_1" {
 # documentation for build blocks can be found here:
 # https://www.packer.io/docs/templates/hcl_templates/blocks/build
 build {
-  sources = ["source.amazon-ebs.autogenerated_1"]
+  sources = ["source.amazon-ebs.autogenerated_1", "source.amazon-ebs.named_builder"]
 
   provisioner "shell" {
     except      = ["amazon-ebs"]

command/test-fixtures/hcl2_upgrade_basic/input.json

@@ -61,6 +61,55 @@
                 ]
             },
             "ssh_interface": "session_manager"
+        },
+        {
+            "type": "amazon-ebs",
+            "name": "named_builder",
+            "region": "{{ user `aws_region` }}",
+            "secret_key": "{{ user `aws_secret_key` }}",
+            "access_key": "{{ user `aws_access_key` }}",
+            "ami_name": "ubuntu-16-04-test-{{ timestamp }}",
+            "ami_description": "Ubuntu 16.04 LTS - expand root partition",
+            "source_ami_filter": {
+                "filters": {
+                    "virtualization-type": "hvm",
+                    "name": "ubuntu/images/*/ubuntu-xenial-16.04-amd64-server-*",
+                    "root-device-type": "ebs"
+                },
+                "owners": [
+                    "099720109477"
+                ],
+                "most_recent": true
+            },
+            "launch_block_device_mappings": [
+                {
+                    "delete_on_termination": true,
+                    "device_name": "/dev/sda1",
+                    "volume_type": "gp2",
+                    "volume_size": 48
+                }
+            ],
+            "spot_price": "0.0075",
+            "spot_instance_types": [
+                "t2.small",
+                "t2.medium",
+                "t2.large"
+            ],
+            "encrypt_boot": true,
+            "ssh_username": "ubuntu",
+            "temporary_iam_instance_profile_policy_document": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "*"
+                        ],
+                        "Resource": ["*"]
+                    }
+                ]
+            },
+            "ssh_interface": "session_manager"
         }
     ],
     "provisioners": [