From 984086275719fa638e22a2ca8ce50fdfc77d0229 Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Tue, 12 Jun 2018 12:05:16 +0200 Subject: [PATCH] builder/amazon: Add suppport for vpc_filter and subnet_filter First step of adding support for discovering VPC's and Subnets using filters. --- builder/amazon/common/build_filter.go | 17 +++ builder/amazon/common/run_config.go | 66 ++++++--- builder/amazon/common/step_network_info.go | 131 ++++++++++++++++++ .../amazon/common/step_run_source_instance.go | 7 + builder/amazon/common/step_security_group.go | 7 + builder/amazon/common/step_source_ami_info.go | 14 +- builder/amazon/ebs/builder.go | 16 ++- 7 files changed, 221 insertions(+), 37 deletions(-) create mode 100644 builder/amazon/common/build_filter.go create mode 100644 builder/amazon/common/step_network_info.go diff --git a/builder/amazon/common/build_filter.go b/builder/amazon/common/build_filter.go new file mode 100644 index 000000000..8f621efc8 --- /dev/null +++ b/builder/amazon/common/build_filter.go @@ -0,0 +1,17 @@ +package common + +import ( + "github.com/aws/aws-sdk-go/service/ec2" +) + +// Build a slice of EC2 (AMI/Subnet/VPC) filter options from the filters provided. +func buildEc2Filters(input map[*string]*string) []*ec2.Filter { + var filters []*ec2.Filter + for k, v := range input { + filters = append(filters, &ec2.Filter{ + Name: k, + Values: []*string{v}, + }) + } + return filters +} diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index dbafd0f38..5eb2cfa42 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -29,32 +29,52 @@ func (d *AmiFilterOptions) NoOwner() bool { return len(d.Owners) == 0 } +type SubnetFilterOptions struct { + Filters map[*string]*string + MostFree bool `mapstructure:"most_free"` + Random bool `mapstructure:"random"` +} + +func (d *SubnetFilterOptions) Empty() bool { + return len(d.Filters) == 0 +} + +type VpcFilterOptions struct { + Filters map[*string]*string +} + +func (d *VpcFilterOptions) Empty() bool { + return len(d.Filters) == 0 +} + // RunConfig contains configuration for running an instance from a source // AMI and details on how to access that launched image. type RunConfig struct { - AssociatePublicIpAddress bool `mapstructure:"associate_public_ip_address"` - AvailabilityZone string `mapstructure:"availability_zone"` - DisableStopInstance bool `mapstructure:"disable_stop_instance"` - EbsOptimized bool `mapstructure:"ebs_optimized"` - EnableT2Unlimited bool `mapstructure:"enable_t2_unlimited"` - IamInstanceProfile string `mapstructure:"iam_instance_profile"` - InstanceInitiatedShutdownBehavior string `mapstructure:"shutdown_behavior"` - InstanceType string `mapstructure:"instance_type"` - RunTags map[string]string `mapstructure:"run_tags"` - SecurityGroupId string `mapstructure:"security_group_id"` - SecurityGroupIds []string `mapstructure:"security_group_ids"` - SourceAmi string `mapstructure:"source_ami"` - SourceAmiFilter AmiFilterOptions `mapstructure:"source_ami_filter"` - SpotPrice string `mapstructure:"spot_price"` - SpotPriceAutoProduct string `mapstructure:"spot_price_auto_product"` - SpotTags map[string]string `mapstructure:"spot_tags"` - SubnetId string `mapstructure:"subnet_id"` - TemporaryKeyPairName string `mapstructure:"temporary_key_pair_name"` - TemporarySGSourceCidr string `mapstructure:"temporary_security_group_source_cidr"` - UserData string `mapstructure:"user_data"` - UserDataFile string `mapstructure:"user_data_file"` - VpcId string `mapstructure:"vpc_id"` - WindowsPasswordTimeout time.Duration `mapstructure:"windows_password_timeout"` + AssociatePublicIpAddress bool `mapstructure:"associate_public_ip_address"` + AvailabilityZone string `mapstructure:"availability_zone"` + DisableStopInstance bool `mapstructure:"disable_stop_instance"` + EbsOptimized bool `mapstructure:"ebs_optimized"` + EnableT2Unlimited bool `mapstructure:"enable_t2_unlimited"` + IamInstanceProfile string `mapstructure:"iam_instance_profile"` + InstanceInitiatedShutdownBehavior string `mapstructure:"shutdown_behavior"` + InstanceType string `mapstructure:"instance_type"` + RunTags map[string]string `mapstructure:"run_tags"` + SecurityGroupId string `mapstructure:"security_group_id"` + SecurityGroupIds []string `mapstructure:"security_group_ids"` + SourceAmi string `mapstructure:"source_ami"` + SourceAmiFilter AmiFilterOptions `mapstructure:"source_ami_filter"` + SpotPrice string `mapstructure:"spot_price"` + SpotPriceAutoProduct string `mapstructure:"spot_price_auto_product"` + SpotTags map[string]string `mapstructure:"spot_tags"` + SubnetFilter SubnetFilterOptions `mapstructure:"subnet_filter"` + SubnetId string `mapstructure:"subnet_id"` + TemporaryKeyPairName string `mapstructure:"temporary_key_pair_name"` + TemporarySGSourceCidr string `mapstructure:"temporary_security_group_source_cidr"` + UserData string `mapstructure:"user_data"` + UserDataFile string `mapstructure:"user_data_file"` + VpcFilter VpcFilterOptions `mapstructure:"vpc_filter"` + VpcId string `mapstructure:"vpc_id"` + WindowsPasswordTimeout time.Duration `mapstructure:"windows_password_timeout"` // Communicator settings Comm communicator.Config `mapstructure:",squash"` diff --git a/builder/amazon/common/step_network_info.go b/builder/amazon/common/step_network_info.go new file mode 100644 index 000000000..3f75c62b2 --- /dev/null +++ b/builder/amazon/common/step_network_info.go @@ -0,0 +1,131 @@ +package common + +import ( + "context" + "fmt" + "log" + "math/rand" + "sort" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/hashicorp/packer/helper/multistep" + "github.com/hashicorp/packer/packer" +) + +// StepNetworkInfo queries AWS for information about +// VPC's, Subnets, and Security Groups that is used +// throughout the AMI creation process. +// +// Produces: +// vpc_id string - the VPC ID +// subnet_id string - the Subnet ID +// az string - the AZ name +// sg_ids []string - the SG IDs +type StepNetworkInfo struct { + VpcId string + VpcFilter VpcFilterOptions + SubnetId string + SubnetFilter SubnetFilterOptions + AvailabilityZone string + // TODO Security groups + filter +} + +type subnetsSort []*ec2.Subnet + +func (a subnetsSort) Len() int { return len(a) } +func (a subnetsSort) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a subnetsSort) Less(i, j int) bool { + return *a[i].AvailableIpAddressCount < *a[j].AvailableIpAddressCount +} + +// Returns the most recent AMI out of a slice of images. +func mostFreeSubnet(subnets []*ec2.Subnet) *ec2.Subnet { + sortedSubnets := subnets + sort.Sort(subnetsSort(sortedSubnets)) + return sortedSubnets[len(sortedSubnets)-1] +} + +func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { + ec2conn := state.Get("ec2").(*ec2.EC2) + ui := state.Get("ui").(packer.Ui) + + // VPC + if s.VpcId == "" && !s.VpcFilter.Empty() { + params := &ec2.DescribeVpcsInput{} + + params.Filters = buildEc2Filters(s.VpcFilter.Filters) + log.Printf("Using VPC Filters %v", params) + + vpcResp, err := ec2conn.DescribeVpcs(params) + if err != nil { + err := fmt.Errorf("Error querying VPCs: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if len(vpcResp.Vpcs) != 1 { + err := fmt.Errorf("No or more than one VPC was found matching filters: %v", params) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + s.VpcId = *vpcResp.Vpcs[0].VpcId + ui.Message(fmt.Sprintf("Found VPC ID: %s", s.VpcId)) + } + + // Subnet + if s.SubnetId == "" && !s.SubnetFilter.Empty() { + params := &ec2.DescribeSubnetsInput{} + + vpcId := "vpc-id" + s.SubnetFilter.Filters[&vpcId] = &s.VpcId + if s.AvailabilityZone != "" { + az := "availability-zone" + s.SubnetFilter.Filters[&az] = &s.AvailabilityZone + } + params.Filters = buildEc2Filters(s.SubnetFilter.Filters) + log.Printf("Using Subnet Filters %v", params) + + subnetsResp, err := ec2conn.DescribeSubnets(params) + if err != nil { + err := fmt.Errorf("Error querying Subnets: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if len(subnetsResp.Subnets) == 0 { + err := fmt.Errorf("No Subnets was found matching filters: %v", params) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if len(subnetsResp.Subnets) > 1 && !s.SubnetFilter.Random && !s.SubnetFilter.MostFree { + err := fmt.Errorf("Your query returned more than one result. Please try a more specific search, or set random or most_free to true.") + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + var subnet *ec2.Subnet + switch { + case s.SubnetFilter.MostFree: + subnet = mostFreeSubnet(subnetsResp.Subnets) + case s.SubnetFilter.Random: + subnet = subnetsResp.Subnets[rand.Intn(len(subnetsResp.Subnets))] + default: + subnet = subnetsResp.Subnets[0] + } + s.SubnetId = *subnet.SubnetId + ui.Message(fmt.Sprintf("Found Subnet ID: %s", s.SubnetId)) + } + + state.Put("vpc_id", s.VpcId) + state.Put("subnet_id", s.SubnetId) + return multistep.ActionContinue +} + +func (s *StepNetworkInfo) Cleanup(multistep.StateBag) {} diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 7c31fa1f8..e4ad43157 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -154,6 +154,13 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa runOpts.KeyName = &keyName } + // TODO always get subnet_id from state. + if s.SubnetId == "" { + if subnetId, ok := state.GetOk("subnet_id"); ok { + s.SubnetId = subnetId.(string) + } + } + if s.SubnetId != "" && s.AssociatePublicIpAddress { runOpts.NetworkInterfaces = []*ec2.InstanceNetworkInterfaceSpecification{ { diff --git a/builder/amazon/common/step_security_group.go b/builder/amazon/common/step_security_group.go index 031842cd3..6ef3f4dbb 100644 --- a/builder/amazon/common/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -60,6 +60,13 @@ func (s *StepSecurityGroup) Run(_ context.Context, state multistep.StateBag) mul Description: aws.String("Temporary group for Packer"), } + // TODO always get vpc_id from state. + if s.VpcId == "" { + if vpcId, ok := state.GetOk("vpc_id"); ok { + s.VpcId = vpcId.(string) + } + } + if s.VpcId != "" { group.VpcId = &s.VpcId } diff --git a/builder/amazon/common/step_source_ami_info.go b/builder/amazon/common/step_source_ami_info.go index 5e57c2fbf..ce322130e 100644 --- a/builder/amazon/common/step_source_ami_info.go +++ b/builder/amazon/common/step_source_ami_info.go @@ -24,18 +24,6 @@ type StepSourceAMIInfo struct { AmiFilters AmiFilterOptions } -// Build a slice of AMI filter options from the filters provided. -func buildAmiFilters(input map[*string]*string) []*ec2.Filter { - var filters []*ec2.Filter - for k, v := range input { - filters = append(filters, &ec2.Filter{ - Name: k, - Values: []*string{v}, - }) - } - return filters -} - type imageSort []*ec2.Image func (a imageSort) Len() int { return len(a) } @@ -65,7 +53,7 @@ func (s *StepSourceAMIInfo) Run(_ context.Context, state multistep.StateBag) mul // We have filters to apply if len(s.AmiFilters.Filters) > 0 { - params.Filters = buildAmiFilters(s.AmiFilters.Filters) + params.Filters = buildEc2Filters(s.AmiFilters.Filters) } if len(s.AmiFilters.Owners) > 0 { params.Owners = s.AmiFilters.Owners diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index f2c0bcbec..d846f617d 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -93,7 +93,12 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } ec2conn := ec2.New(session) + // TODO Translate this into VpcFilter/SubnetFilter and move the describe into apropriate step. // If the subnet is specified but not the VpcId or AZ, try to determine them automatically + + /* + * If subnet => vpc, az + */ if b.config.SubnetId != "" && (b.config.AvailabilityZone == "" || b.config.VpcId == "") { log.Printf("[INFO] Finding AZ and VpcId for the given subnet '%s'", b.config.SubnetId) resp, err := ec2conn.DescribeSubnets(&ec2.DescribeSubnetsInput{SubnetIds: []*string{&b.config.SubnetId}}) @@ -177,6 +182,13 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe EnableAMIENASupport: b.config.AMIENASupport, AmiFilters: b.config.SourceAmiFilter, }, + &awscommon.StepNetworkInfo{ + VpcId: b.config.VpcId, + VpcFilter: b.config.VpcFilter, + SubnetId: b.config.SubnetId, + SubnetFilter: b.config.SubnetFilter, + AvailabilityZone: b.config.AvailabilityZone, + }, &awscommon.StepKeyPair{ Debug: b.config.PackerDebug, SSHAgentAuth: b.config.Comm.SSHAgentAuth, @@ -186,9 +198,11 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe PrivateKeyFile: b.config.RunConfig.Comm.SSHPrivateKey, }, &awscommon.StepSecurityGroup{ + // TODO remove SecurityGroupIds: b.config.SecurityGroupIds, CommConfig: &b.config.RunConfig.Comm, - VpcId: b.config.VpcId, + // TODO remove + VpcId: b.config.VpcId, TemporarySGSourceCidr: b.config.TemporarySGSourceCidr, }, &awscommon.StepCleanupVolumes{