builder/amazon: Add suppport for vpc_filter and subnet_filter

First step of adding support for discovering VPC's and Subnets using
filters.

builder/amazon/common/build_filter.go

@@ -0,0 +1,17 @@
+package common
+
+import (
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+// Build a slice of EC2 (AMI/Subnet/VPC) filter options from the filters provided.
+func buildEc2Filters(input map[*string]*string) []*ec2.Filter {
+	var filters []*ec2.Filter
+	for k, v := range input {
+		filters = append(filters, &ec2.Filter{
+			Name:   k,
+			Values: []*string{v},
+		})
+	}
+	return filters
+}

builder/amazon/common/run_config.go

@@ -29,6 +29,24 @@ func (d *AmiFilterOptions) NoOwner() bool {
 	return len(d.Owners) == 0
 }
 
+type SubnetFilterOptions struct {
+	Filters  map[*string]*string
+	MostFree bool `mapstructure:"most_free"`
+	Random   bool `mapstructure:"random"`
+}
+
+func (d *SubnetFilterOptions) Empty() bool {
+	return len(d.Filters) == 0
+}
+
+type VpcFilterOptions struct {
+	Filters map[*string]*string
+}
+
+func (d *VpcFilterOptions) Empty() bool {
+	return len(d.Filters) == 0
+}
+
 // RunConfig contains configuration for running an instance from a source
 // AMI and details on how to access that launched image.
 type RunConfig struct {
@@ -48,11 +66,13 @@ type RunConfig struct {
 	SpotPrice                         string              `mapstructure:"spot_price"`
 	SpotPriceAutoProduct              string              `mapstructure:"spot_price_auto_product"`
 	SpotTags                          map[string]string   `mapstructure:"spot_tags"`
+	SubnetFilter                      SubnetFilterOptions `mapstructure:"subnet_filter"`
 	SubnetId                          string              `mapstructure:"subnet_id"`
 	TemporaryKeyPairName              string              `mapstructure:"temporary_key_pair_name"`
 	TemporarySGSourceCidr             string              `mapstructure:"temporary_security_group_source_cidr"`
 	UserData                          string              `mapstructure:"user_data"`
 	UserDataFile                      string              `mapstructure:"user_data_file"`
+	VpcFilter                         VpcFilterOptions    `mapstructure:"vpc_filter"`
 	VpcId                             string              `mapstructure:"vpc_id"`
 	WindowsPasswordTimeout            time.Duration       `mapstructure:"windows_password_timeout"`
 

builder/amazon/common/step_network_info.go

@@ -0,0 +1,131 @@
+package common
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"math/rand"
+	"sort"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+// StepNetworkInfo queries AWS for information about
+// VPC's, Subnets, and Security Groups that is used
+// throughout the AMI creation process.
+//
+// Produces:
+//   vpc_id string - the VPC ID
+//   subnet_id string - the Subnet ID
+//   az string - the AZ name
+//   sg_ids []string - the SG IDs
+type StepNetworkInfo struct {
+	VpcId            string
+	VpcFilter        VpcFilterOptions
+	SubnetId         string
+	SubnetFilter     SubnetFilterOptions
+	AvailabilityZone string
+	// TODO Security groups + filter
+}
+
+type subnetsSort []*ec2.Subnet
+
+func (a subnetsSort) Len() int      { return len(a) }
+func (a subnetsSort) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a subnetsSort) Less(i, j int) bool {
+	return *a[i].AvailableIpAddressCount < *a[j].AvailableIpAddressCount
+}
+
+// Returns the most recent AMI out of a slice of images.
+func mostFreeSubnet(subnets []*ec2.Subnet) *ec2.Subnet {
+	sortedSubnets := subnets
+	sort.Sort(subnetsSort(sortedSubnets))
+	return sortedSubnets[len(sortedSubnets)-1]
+}
+
+func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
+	ec2conn := state.Get("ec2").(*ec2.EC2)
+	ui := state.Get("ui").(packer.Ui)
+
+	// VPC
+	if s.VpcId == "" && !s.VpcFilter.Empty() {
+		params := &ec2.DescribeVpcsInput{}
+
+		params.Filters = buildEc2Filters(s.VpcFilter.Filters)
+		log.Printf("Using VPC Filters %v", params)
+
+		vpcResp, err := ec2conn.DescribeVpcs(params)
+		if err != nil {
+			err := fmt.Errorf("Error querying VPCs: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		if len(vpcResp.Vpcs) != 1 {
+			err := fmt.Errorf("No or more than one VPC was found matching filters: %v", params)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		s.VpcId = *vpcResp.Vpcs[0].VpcId
+		ui.Message(fmt.Sprintf("Found VPC ID: %s", s.VpcId))
+	}
+
+	// Subnet
+	if s.SubnetId == "" && !s.SubnetFilter.Empty() {
+		params := &ec2.DescribeSubnetsInput{}
+
+		vpcId := "vpc-id"
+		s.SubnetFilter.Filters[&vpcId] = &s.VpcId
+		if s.AvailabilityZone != "" {
+			az := "availability-zone"
+			s.SubnetFilter.Filters[&az] = &s.AvailabilityZone
+		}
+		params.Filters = buildEc2Filters(s.SubnetFilter.Filters)
+		log.Printf("Using Subnet Filters %v", params)
+
+		subnetsResp, err := ec2conn.DescribeSubnets(params)
+		if err != nil {
+			err := fmt.Errorf("Error querying Subnets: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		if len(subnetsResp.Subnets) == 0 {
+			err := fmt.Errorf("No Subnets was found matching filters: %v", params)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		if len(subnetsResp.Subnets) > 1 && !s.SubnetFilter.Random && !s.SubnetFilter.MostFree {
+			err := fmt.Errorf("Your query returned more than one result. Please try a more specific search, or set random or most_free to true.")
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		var subnet *ec2.Subnet
+		switch {
+		case s.SubnetFilter.MostFree:
+			subnet = mostFreeSubnet(subnetsResp.Subnets)
+		case s.SubnetFilter.Random:
+			subnet = subnetsResp.Subnets[rand.Intn(len(subnetsResp.Subnets))]
+		default:
+			subnet = subnetsResp.Subnets[0]
+		}
+		s.SubnetId = *subnet.SubnetId
+		ui.Message(fmt.Sprintf("Found Subnet ID: %s", s.SubnetId))
+	}
+
+	state.Put("vpc_id", s.VpcId)
+	state.Put("subnet_id", s.SubnetId)
+	return multistep.ActionContinue
+}
+
+func (s *StepNetworkInfo) Cleanup(multistep.StateBag) {}

builder/amazon/common/step_run_source_instance.go

@@ -154,6 +154,13 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa
 		runOpts.KeyName = &keyName
 	}
 
+	// TODO always get subnet_id from state.
+	if s.SubnetId == "" {
+		if subnetId, ok := state.GetOk("subnet_id"); ok {
+			s.SubnetId = subnetId.(string)
+		}
+	}
+
 	if s.SubnetId != "" && s.AssociatePublicIpAddress {
 		runOpts.NetworkInterfaces = []*ec2.InstanceNetworkInterfaceSpecification{
 			{

builder/amazon/common/step_security_group.go

@@ -60,6 +60,13 @@ func (s *StepSecurityGroup) Run(_ context.Context, state multistep.StateBag) mul
 		Description: aws.String("Temporary group for Packer"),
 	}
 
+	// TODO always get vpc_id from state.
+	if s.VpcId == "" {
+		if vpcId, ok := state.GetOk("vpc_id"); ok {
+			s.VpcId = vpcId.(string)
+		}
+	}
+
 	if s.VpcId != "" {
 		group.VpcId = &s.VpcId
 	}

builder/amazon/common/step_source_ami_info.go

@@ -24,18 +24,6 @@ type StepSourceAMIInfo struct {
 	AmiFilters               AmiFilterOptions
 }
 
-// Build a slice of AMI filter options from the filters provided.
-func buildAmiFilters(input map[*string]*string) []*ec2.Filter {
-	var filters []*ec2.Filter
-	for k, v := range input {
-		filters = append(filters, &ec2.Filter{
-			Name:   k,
-			Values: []*string{v},
-		})
-	}
-	return filters
-}
-
 type imageSort []*ec2.Image
 
 func (a imageSort) Len() int      { return len(a) }
@@ -65,7 +53,7 @@ func (s *StepSourceAMIInfo) Run(_ context.Context, state multistep.StateBag) mul
 
 	// We have filters to apply
 	if len(s.AmiFilters.Filters) > 0 {
-		params.Filters = buildAmiFilters(s.AmiFilters.Filters)
+		params.Filters = buildEc2Filters(s.AmiFilters.Filters)
 	}
 	if len(s.AmiFilters.Owners) > 0 {
 		params.Owners = s.AmiFilters.Owners

builder/amazon/ebs/builder.go

@@ -93,7 +93,12 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	}
 	ec2conn := ec2.New(session)
 
+	// TODO Translate this into VpcFilter/SubnetFilter and move the describe into apropriate step.
 	// If the subnet is specified but not the VpcId or AZ, try to determine them automatically
+
+	/*
+	 * If subnet => vpc, az
+	 */
 	if b.config.SubnetId != "" && (b.config.AvailabilityZone == "" || b.config.VpcId == "") {
 		log.Printf("[INFO] Finding AZ and VpcId for the given subnet '%s'", b.config.SubnetId)
 		resp, err := ec2conn.DescribeSubnets(&ec2.DescribeSubnetsInput{SubnetIds: []*string{&b.config.SubnetId}})
@@ -177,6 +182,13 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			EnableAMIENASupport:      b.config.AMIENASupport,
 			AmiFilters:               b.config.SourceAmiFilter,
 		},
+		&awscommon.StepNetworkInfo{
+			VpcId:            b.config.VpcId,
+			VpcFilter:        b.config.VpcFilter,
+			SubnetId:         b.config.SubnetId,
+			SubnetFilter:     b.config.SubnetFilter,
+			AvailabilityZone: b.config.AvailabilityZone,
+		},
 		&awscommon.StepKeyPair{
 			Debug:                b.config.PackerDebug,
 			SSHAgentAuth:         b.config.Comm.SSHAgentAuth,
@@ -186,8 +198,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			PrivateKeyFile:       b.config.RunConfig.Comm.SSHPrivateKey,
 		},
 		&awscommon.StepSecurityGroup{
+			// TODO remove
 			SecurityGroupIds: b.config.SecurityGroupIds,
 			CommConfig:       &b.config.RunConfig.Comm,
+			// TODO remove
 			VpcId: b.config.VpcId,
 			TemporarySGSourceCidr: b.config.TemporarySGSourceCidr,
 		},