builder/amazon: Add suppport for vpc_filter and subnet_filter

First step of adding support for discovering VPC's and Subnets using
filters.

builder/amazon/common/build_filter.go

@@ -0,0 +1,17 @@
+package common
+
+import (
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+// Build a slice of EC2 (AMI/Subnet/VPC) filter options from the filters provided.
+func buildEc2Filters(input map[*string]*string) []*ec2.Filter {
+	var filters []*ec2.Filter
+	for k, v := range input {
+		filters = append(filters, &ec2.Filter{
+			Name:   k,
+			Values: []*string{v},
+		})
+	}
+	return filters
+}

builder/amazon/common/run_config.go

@@ -29,32 +29,52 @@ func (d *AmiFilterOptions) NoOwner() bool {
 	return len(d.Owners) == 0
 }
 
+type SubnetFilterOptions struct {
+	Filters  map[*string]*string
+	MostFree bool `mapstructure:"most_free"`
+	Random   bool `mapstructure:"random"`
+}
+
+func (d *SubnetFilterOptions) Empty() bool {
+	return len(d.Filters) == 0
+}
+
+type VpcFilterOptions struct {
+	Filters map[*string]*string
+}
+
+func (d *VpcFilterOptions) Empty() bool {
+	return len(d.Filters) == 0
+}
+
 // RunConfig contains configuration for running an instance from a source
 // AMI and details on how to access that launched image.
 type RunConfig struct {
-	AssociatePublicIpAddress          bool              `mapstructure:"associate_public_ip_address"`
+	AssociatePublicIpAddress          bool                `mapstructure:"associate_public_ip_address"`
-	AvailabilityZone                  string            `mapstructure:"availability_zone"`
+	AvailabilityZone                  string              `mapstructure:"availability_zone"`
-	DisableStopInstance               bool              `mapstructure:"disable_stop_instance"`
+	DisableStopInstance               bool                `mapstructure:"disable_stop_instance"`
-	EbsOptimized                      bool              `mapstructure:"ebs_optimized"`
+	EbsOptimized                      bool                `mapstructure:"ebs_optimized"`
-	EnableT2Unlimited                 bool              `mapstructure:"enable_t2_unlimited"`
+	EnableT2Unlimited                 bool                `mapstructure:"enable_t2_unlimited"`
-	IamInstanceProfile                string            `mapstructure:"iam_instance_profile"`
+	IamInstanceProfile                string              `mapstructure:"iam_instance_profile"`
-	InstanceInitiatedShutdownBehavior string            `mapstructure:"shutdown_behavior"`
+	InstanceInitiatedShutdownBehavior string              `mapstructure:"shutdown_behavior"`
-	InstanceType                      string            `mapstructure:"instance_type"`
+	InstanceType                      string              `mapstructure:"instance_type"`
-	RunTags                           map[string]string `mapstructure:"run_tags"`
+	RunTags                           map[string]string   `mapstructure:"run_tags"`
-	SecurityGroupId                   string            `mapstructure:"security_group_id"`
+	SecurityGroupId                   string              `mapstructure:"security_group_id"`
-	SecurityGroupIds                  []string          `mapstructure:"security_group_ids"`
+	SecurityGroupIds                  []string            `mapstructure:"security_group_ids"`
-	SourceAmi                         string            `mapstructure:"source_ami"`
+	SourceAmi                         string              `mapstructure:"source_ami"`
-	SourceAmiFilter                   AmiFilterOptions  `mapstructure:"source_ami_filter"`
+	SourceAmiFilter                   AmiFilterOptions    `mapstructure:"source_ami_filter"`
-	SpotPrice                         string            `mapstructure:"spot_price"`
+	SpotPrice                         string              `mapstructure:"spot_price"`
-	SpotPriceAutoProduct              string            `mapstructure:"spot_price_auto_product"`
+	SpotPriceAutoProduct              string              `mapstructure:"spot_price_auto_product"`
-	SpotTags                          map[string]string `mapstructure:"spot_tags"`
+	SpotTags                          map[string]string   `mapstructure:"spot_tags"`
-	SubnetId                          string            `mapstructure:"subnet_id"`
+	SubnetFilter                      SubnetFilterOptions `mapstructure:"subnet_filter"`
-	TemporaryKeyPairName              string            `mapstructure:"temporary_key_pair_name"`
+	SubnetId                          string              `mapstructure:"subnet_id"`
-	TemporarySGSourceCidr             string            `mapstructure:"temporary_security_group_source_cidr"`
+	TemporaryKeyPairName              string              `mapstructure:"temporary_key_pair_name"`
-	UserData                          string            `mapstructure:"user_data"`
+	TemporarySGSourceCidr             string              `mapstructure:"temporary_security_group_source_cidr"`
-	UserDataFile                      string            `mapstructure:"user_data_file"`
+	UserData                          string              `mapstructure:"user_data"`
-	VpcId                             string            `mapstructure:"vpc_id"`
+	UserDataFile                      string              `mapstructure:"user_data_file"`
-	WindowsPasswordTimeout            time.Duration     `mapstructure:"windows_password_timeout"`
+	VpcFilter                         VpcFilterOptions    `mapstructure:"vpc_filter"`
+	VpcId                             string              `mapstructure:"vpc_id"`
+	WindowsPasswordTimeout            time.Duration       `mapstructure:"windows_password_timeout"`
 
 	// Communicator settings
 	Comm           communicator.Config `mapstructure:",squash"`

builder/amazon/common/step_network_info.go

@@ -0,0 +1,131 @@
+package common
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"math/rand"
+	"sort"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+// StepNetworkInfo queries AWS for information about
+// VPC's, Subnets, and Security Groups that is used
+// throughout the AMI creation process.
+//
+// Produces:
+//   vpc_id string - the VPC ID
+//   subnet_id string - the Subnet ID
+//   az string - the AZ name
+//   sg_ids []string - the SG IDs
+type StepNetworkInfo struct {
+	VpcId            string
+	VpcFilter        VpcFilterOptions
+	SubnetId         string
+	SubnetFilter     SubnetFilterOptions
+	AvailabilityZone string
+	// TODO Security groups + filter
+}
+
+type subnetsSort []*ec2.Subnet
+
+func (a subnetsSort) Len() int      { return len(a) }
+func (a subnetsSort) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a subnetsSort) Less(i, j int) bool {
+	return *a[i].AvailableIpAddressCount < *a[j].AvailableIpAddressCount
+}
+
+// Returns the most recent AMI out of a slice of images.
+func mostFreeSubnet(subnets []*ec2.Subnet) *ec2.Subnet {
+	sortedSubnets := subnets
+	sort.Sort(subnetsSort(sortedSubnets))
+	return sortedSubnets[len(sortedSubnets)-1]
+}
+
+func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
+	ec2conn := state.Get("ec2").(*ec2.EC2)
+	ui := state.Get("ui").(packer.Ui)
+
+	// VPC
+	if s.VpcId == "" && !s.VpcFilter.Empty() {
+		params := &ec2.DescribeVpcsInput{}
+
+		params.Filters = buildEc2Filters(s.VpcFilter.Filters)
+		log.Printf("Using VPC Filters %v", params)
+
+		vpcResp, err := ec2conn.DescribeVpcs(params)
+		if err != nil {
+			err := fmt.Errorf("Error querying VPCs: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		if len(vpcResp.Vpcs) != 1 {
+			err := fmt.Errorf("No or more than one VPC was found matching filters: %v", params)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		s.VpcId = *vpcResp.Vpcs[0].VpcId
+		ui.Message(fmt.Sprintf("Found VPC ID: %s", s.VpcId))
+	}
+
+	// Subnet
+	if s.SubnetId == "" && !s.SubnetFilter.Empty() {
+		params := &ec2.DescribeSubnetsInput{}
+
+		vpcId := "vpc-id"
+		s.SubnetFilter.Filters[&vpcId] = &s.VpcId
+		if s.AvailabilityZone != "" {
+			az := "availability-zone"
+			s.SubnetFilter.Filters[&az] = &s.AvailabilityZone
+		}
+		params.Filters = buildEc2Filters(s.SubnetFilter.Filters)
+		log.Printf("Using Subnet Filters %v", params)
+
+		subnetsResp, err := ec2conn.DescribeSubnets(params)
+		if err != nil {
+			err := fmt.Errorf("Error querying Subnets: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		if len(subnetsResp.Subnets) == 0 {
+			err := fmt.Errorf("No Subnets was found matching filters: %v", params)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		if len(subnetsResp.Subnets) > 1 && !s.SubnetFilter.Random && !s.SubnetFilter.MostFree {
+			err := fmt.Errorf("Your query returned more than one result. Please try a more specific search, or set random or most_free to true.")
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		var subnet *ec2.Subnet
+		switch {
+		case s.SubnetFilter.MostFree:
+			subnet = mostFreeSubnet(subnetsResp.Subnets)
+		case s.SubnetFilter.Random:
+			subnet = subnetsResp.Subnets[rand.Intn(len(subnetsResp.Subnets))]
+		default:
+			subnet = subnetsResp.Subnets[0]
+		}
+		s.SubnetId = *subnet.SubnetId
+		ui.Message(fmt.Sprintf("Found Subnet ID: %s", s.SubnetId))
+	}
+
+	state.Put("vpc_id", s.VpcId)
+	state.Put("subnet_id", s.SubnetId)
+	return multistep.ActionContinue
+}
+
+func (s *StepNetworkInfo) Cleanup(multistep.StateBag) {}

builder/amazon/common/step_run_source_instance.go

@@ -154,6 +154,13 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa
 		runOpts.KeyName = &keyName
 	}
 
+	// TODO always get subnet_id from state.
+	if s.SubnetId == "" {
+		if subnetId, ok := state.GetOk("subnet_id"); ok {
+			s.SubnetId = subnetId.(string)
+		}
+	}
+
 	if s.SubnetId != "" && s.AssociatePublicIpAddress {
 		runOpts.NetworkInterfaces = []*ec2.InstanceNetworkInterfaceSpecification{
 			{

builder/amazon/common/step_security_group.go

@@ -60,6 +60,13 @@ func (s *StepSecurityGroup) Run(_ context.Context, state multistep.StateBag) mul
 		Description: aws.String("Temporary group for Packer"),
 	}
 
+	// TODO always get vpc_id from state.
+	if s.VpcId == "" {
+		if vpcId, ok := state.GetOk("vpc_id"); ok {
+			s.VpcId = vpcId.(string)
+		}
+	}
+
 	if s.VpcId != "" {
 		group.VpcId = &s.VpcId
 	}

builder/amazon/common/step_source_ami_info.go

@@ -24,18 +24,6 @@ type StepSourceAMIInfo struct {
 	AmiFilters               AmiFilterOptions
 }
 
-// Build a slice of AMI filter options from the filters provided.
-func buildAmiFilters(input map[*string]*string) []*ec2.Filter {
-	var filters []*ec2.Filter
-	for k, v := range input {
-		filters = append(filters, &ec2.Filter{
-			Name:   k,
-			Values: []*string{v},
-		})
-	}
-	return filters
-}
-
 type imageSort []*ec2.Image
 
 func (a imageSort) Len() int      { return len(a) }
@@ -65,7 +53,7 @@ func (s *StepSourceAMIInfo) Run(_ context.Context, state multistep.StateBag) mul
 
 	// We have filters to apply
 	if len(s.AmiFilters.Filters) > 0 {
-		params.Filters = buildAmiFilters(s.AmiFilters.Filters)
+		params.Filters = buildEc2Filters(s.AmiFilters.Filters)
 	}
 	if len(s.AmiFilters.Owners) > 0 {
 		params.Owners = s.AmiFilters.Owners

builder/amazon/ebs/builder.go

@@ -93,7 +93,12 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	}
 	ec2conn := ec2.New(session)
 
+	// TODO Translate this into VpcFilter/SubnetFilter and move the describe into apropriate step.
 	// If the subnet is specified but not the VpcId or AZ, try to determine them automatically
+
+	/*
+	 * If subnet => vpc, az
+	 */
 	if b.config.SubnetId != "" && (b.config.AvailabilityZone == "" || b.config.VpcId == "") {
 		log.Printf("[INFO] Finding AZ and VpcId for the given subnet '%s'", b.config.SubnetId)
 		resp, err := ec2conn.DescribeSubnets(&ec2.DescribeSubnetsInput{SubnetIds: []*string{&b.config.SubnetId}})
@@ -177,6 +182,13 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			EnableAMIENASupport:      b.config.AMIENASupport,
 			AmiFilters:               b.config.SourceAmiFilter,
 		},
+		&awscommon.StepNetworkInfo{
+			VpcId:            b.config.VpcId,
+			VpcFilter:        b.config.VpcFilter,
+			SubnetId:         b.config.SubnetId,
+			SubnetFilter:     b.config.SubnetFilter,
+			AvailabilityZone: b.config.AvailabilityZone,
+		},
 		&awscommon.StepKeyPair{
 			Debug:                b.config.PackerDebug,
 			SSHAgentAuth:         b.config.Comm.SSHAgentAuth,
@@ -186,9 +198,11 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			PrivateKeyFile:       b.config.RunConfig.Comm.SSHPrivateKey,
 		},
 		&awscommon.StepSecurityGroup{
+			// TODO remove
 			SecurityGroupIds: b.config.SecurityGroupIds,
 			CommConfig:       &b.config.RunConfig.Comm,
-			VpcId:            b.config.VpcId,
+			// TODO remove
+			VpcId: b.config.VpcId,
 			TemporarySGSourceCidr: b.config.TemporarySGSourceCidr,
 		},
 		&awscommon.StepCleanupVolumes{