From 3c7944624a02a26e9a9148e11c03fc58a7f31626 Mon Sep 17 00:00:00 2001 From: Sylvia Moss Date: Wed, 20 Jan 2021 11:05:03 +0100 Subject: [PATCH] (3) Add amazon-ami data source (#10467) --- builder/amazon/chroot/builder.go | 4 +- builder/amazon/common/access_config.go | 3 +- builder/amazon/common/access_config_test.go | 2 +- builder/amazon/common/ami_filter.go | 77 +++++++++ builder/amazon/common/run_config.go | 24 --- builder/amazon/common/run_config.hcl2spec.go | 8 +- builder/amazon/common/run_config_test.go | 7 +- builder/amazon/common/step_source_ami_info.go | 34 +--- builder/amazon/ebs/acceptance/aws.go | 12 +- builder/amazon/ebs/builder.go | 2 +- builder/amazon/ebssurrogate/builder.go | 2 +- builder/amazon/ebsvolume/builder.go | 2 +- builder/amazon/instance/builder.go | 2 +- cmd/packer-plugin-amazon/main.go | 2 + .../main.go | 8 +- command/plugin.go | 5 +- datasource/amazon/ami/data.go | 90 ++++++++++ datasource/amazon/ami/data.hcl2spec.go | 101 +++++++++++ datasource/amazon/ami/data_acc_test.go | 64 +++++++ datasource/amazon/ami/data_test.go | 45 +++++ .../test-fixtures/configure-source-ssh.ps1 | 161 ++++++++++++++++++ .../amazon-import/post-processor.go | 2 +- .../content/docs/datasources/amazon-ami.mdx | 46 +++++ website/content/docs/datasources/index.mdx | 14 ++ .../common/AmiFilterOptions-not-required.mdx | 15 +- website/data/docs-navigation.js | 1 + 26 files changed, 643 insertions(+), 90 deletions(-) create mode 100644 builder/amazon/common/ami_filter.go rename cmd/{packer-plugin-check => packer-plugins-check}/main.go (96%) create mode 100644 datasource/amazon/ami/data.go create mode 100644 datasource/amazon/ami/data.hcl2spec.go create mode 100644 datasource/amazon/ami/data_acc_test.go create mode 100644 datasource/amazon/ami/data_test.go create mode 100644 datasource/amazon/ami/test-fixtures/configure-source-ssh.ps1 create mode 100644 website/content/docs/datasources/amazon-ami.mdx create mode 100644 website/content/docs/datasources/index.mdx diff --git a/builder/amazon/chroot/builder.go b/builder/amazon/chroot/builder.go index 450963978..5b9951c8a 100644 --- a/builder/amazon/chroot/builder.go +++ b/builder/amazon/chroot/builder.go @@ -285,9 +285,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) { var warns []string errs = packersdk.MultiErrorAppend(errs, b.config.RootVolumeTag.CopyOn(&b.config.RootVolumeTags)...) - errs = packersdk.MultiErrorAppend(errs, b.config.SourceAmiFilter.Prepare()...) - - errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) + errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) errs = packersdk.MultiErrorAppend(errs, b.config.AMIConfig.Prepare(&b.config.AccessConfig, &b.config.ctx)...) diff --git a/builder/amazon/common/access_config.go b/builder/amazon/common/access_config.go index 6dcb23eb8..03bc1bce9 100644 --- a/builder/amazon/common/access_config.go +++ b/builder/amazon/common/access_config.go @@ -17,7 +17,6 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" awsbase "github.com/hashicorp/aws-sdk-go-base" cleanhttp "github.com/hashicorp/go-cleanhttp" - "github.com/hashicorp/packer-plugin-sdk/template/interpolate" "github.com/hashicorp/packer/builder/amazon/common/awserrors" vaultapi "github.com/hashicorp/vault/api" ) @@ -370,7 +369,7 @@ func (c *AccessConfig) GetCredsFromVault() error { return nil } -func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { +func (c *AccessConfig) Prepare() []error { var errs []error if c.SkipMetadataApiCheck { diff --git a/builder/amazon/common/access_config_test.go b/builder/amazon/common/access_config_test.go index 61449cbde..0e2d4b4fa 100644 --- a/builder/amazon/common/access_config_test.go +++ b/builder/amazon/common/access_config_test.go @@ -47,7 +47,7 @@ func TestAccessConfigPrepare_RegionRestricted(t *testing.T) { Region: aws.String("us-gov-west-1"), })) - if err := c.Prepare(nil); err != nil { + if err := c.Prepare(); err != nil { t.Fatalf("shouldn't have err: %s", err) } diff --git a/builder/amazon/common/ami_filter.go b/builder/amazon/common/ami_filter.go new file mode 100644 index 000000000..2fb9e2d54 --- /dev/null +++ b/builder/amazon/common/ami_filter.go @@ -0,0 +1,77 @@ +//go:generate struct-markdown +package common + +import ( + "fmt" + "log" + + "github.com/aws/aws-sdk-go/service/ec2" +) + +type AmiFilterOptions struct { + // Filters used to select an AMI. Any filter described in the docs for + // [DescribeImages](http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html) + // is valid. + Filters map[string]string `mapstructure:"filters"` + // Filters the images by their owner. You + // may specify one or more AWS account IDs, "self" (which will use the + // account whose credentials you are using to run Packer), or an AWS owner + // alias: for example, `amazon`, `aws-marketplace`, or `microsoft`. This + // option is required for security reasons. + Owners []string `mapstructure:"owners"` + // Selects the newest created image when true. + // This is most useful for selecting a daily distro build. + MostRecent bool `mapstructure:"most_recent"` +} + +func (d *AmiFilterOptions) GetOwners() []*string { + res := make([]*string, 0, len(d.Owners)) + for _, owner := range d.Owners { + i := owner + res = append(res, &i) + } + return res +} + +func (d *AmiFilterOptions) Empty() bool { + return len(d.Owners) == 0 && len(d.Filters) == 0 +} + +func (d *AmiFilterOptions) NoOwner() bool { + return len(d.Owners) == 0 +} + +func (d *AmiFilterOptions) GetFilteredImage(params *ec2.DescribeImagesInput, ec2conn *ec2.EC2) (*ec2.Image, error) { + // We have filters to apply + if len(d.Filters) > 0 { + params.Filters = buildEc2Filters(d.Filters) + } + if len(d.Owners) > 0 { + params.Owners = d.GetOwners() + } + + log.Printf("Using AMI Filters %v", params) + imageResp, err := ec2conn.DescribeImages(params) + if err != nil { + err := fmt.Errorf("Error querying AMI: %s", err) + return nil, err + } + + if len(imageResp.Images) == 0 { + err := fmt.Errorf("No AMI was found matching filters: %v", params) + return nil, err + } + + if len(imageResp.Images) > 1 && !d.MostRecent { + err := fmt.Errorf("Your query returned more than one result. Please try a more specific search, or set most_recent to true.") + return nil, err + } + + var image *ec2.Image + if d.MostRecent { + image = mostRecentAmi(imageResp.Images) + } else { + image = imageResp.Images[0] + } + return image, nil +} diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index c64ce1337..f67c0d5da 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -19,29 +19,6 @@ import ( var reShutdownBehavior = regexp.MustCompile("^(stop|terminate)$") -type AmiFilterOptions struct { - config.KeyValueFilter `mapstructure:",squash"` - Owners []string - MostRecent bool `mapstructure:"most_recent"` -} - -func (d *AmiFilterOptions) GetOwners() []*string { - res := make([]*string, 0, len(d.Owners)) - for _, owner := range d.Owners { - i := owner - res = append(res, &i) - } - return res -} - -func (d *AmiFilterOptions) Empty() bool { - return len(d.Owners) == 0 && d.KeyValueFilter.Empty() -} - -func (d *AmiFilterOptions) NoOwner() bool { - return len(d.Owners) == 0 -} - type SubnetFilterOptions struct { config.NameValueFilter `mapstructure:",squash"` MostFree bool `mapstructure:"most_free"` @@ -515,7 +492,6 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { errs = append(errs, c.SpotTag.CopyOn(&c.SpotTags)...) for _, preparer := range []interface{ Prepare() []error }{ - &c.SourceAmiFilter, &c.SecurityGroupFilter, &c.SubnetFilter, &c.VpcFilter, diff --git a/builder/amazon/common/run_config.hcl2spec.go b/builder/amazon/common/run_config.hcl2spec.go index 7fa7c062c..c800283fb 100644 --- a/builder/amazon/common/run_config.hcl2spec.go +++ b/builder/amazon/common/run_config.hcl2spec.go @@ -11,10 +11,9 @@ import ( // FlatAmiFilterOptions is an auto-generated flat version of AmiFilterOptions. // Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. type FlatAmiFilterOptions struct { - Filters map[string]string `cty:"filters" hcl:"filters"` - Filter []config.FlatKeyValue `cty:"filter" hcl:"filter"` - Owners []string `cty:"owners" hcl:"owners"` - MostRecent *bool `mapstructure:"most_recent" cty:"most_recent" hcl:"most_recent"` + Filters map[string]string `mapstructure:"filters" cty:"filters" hcl:"filters"` + Owners []string `mapstructure:"owners" cty:"owners" hcl:"owners"` + MostRecent *bool `mapstructure:"most_recent" cty:"most_recent" hcl:"most_recent"` } // FlatMapstructure returns a new FlatAmiFilterOptions. @@ -30,7 +29,6 @@ func (*AmiFilterOptions) FlatMapstructure() interface{ HCL2Spec() map[string]hcl func (*FlatAmiFilterOptions) HCL2Spec() map[string]hcldec.Spec { s := map[string]hcldec.Spec{ "filters": &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false}, - "filter": &hcldec.BlockListSpec{TypeName: "filter", Nested: hcldec.ObjectSpec((*config.FlatKeyValue)(nil).HCL2Spec())}, "owners": &hcldec.AttrSpec{Name: "owners", Type: cty.List(cty.String), Required: false}, "most_recent": &hcldec.AttrSpec{Name: "most_recent", Type: cty.Bool, Required: false}, } diff --git a/builder/amazon/common/run_config_test.go b/builder/amazon/common/run_config_test.go index e679fb2b8..0497714e6 100644 --- a/builder/amazon/common/run_config_test.go +++ b/builder/amazon/common/run_config_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/hashicorp/packer-plugin-sdk/communicator" - "github.com/hashicorp/packer-plugin-sdk/template/config" ) func init() { @@ -86,10 +85,8 @@ func TestRunConfigPrepare_SourceAmiFilterGood(t *testing.T) { filter_key := "name" filter_value := "foo" goodFilter := AmiFilterOptions{ - Owners: []string{owner}, - KeyValueFilter: config.KeyValueFilter{ - Filters: map[string]string{filter_key: filter_value}, - }, + Owners: []string{owner}, + Filters: map[string]string{filter_key: filter_value}, } c.SourceAmiFilter = goodFilter if err := c.Prepare(nil); len(err) != 0 { diff --git a/builder/amazon/common/step_source_ami_info.go b/builder/amazon/common/step_source_ami_info.go index 6e3cafb60..200460ca9 100644 --- a/builder/amazon/common/step_source_ami_info.go +++ b/builder/amazon/common/step_source_ami_info.go @@ -3,7 +3,6 @@ package common import ( "context" "fmt" - "log" "sort" "time" @@ -53,44 +52,13 @@ func (s *StepSourceAMIInfo) Run(ctx context.Context, state multistep.StateBag) m params.ImageIds = []*string{&s.SourceAmi} } - // We have filters to apply - if len(s.AmiFilters.Filters) > 0 { - params.Filters = buildEc2Filters(s.AmiFilters.Filters) - } - if len(s.AmiFilters.Owners) > 0 { - params.Owners = s.AmiFilters.GetOwners() - } - - log.Printf("Using AMI Filters %v", params) - imageResp, err := ec2conn.DescribeImages(params) + image, err := s.AmiFilters.GetFilteredImage(params, ec2conn) if err != nil { - err := fmt.Errorf("Error querying AMI: %s", err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - if len(imageResp.Images) == 0 { - err := fmt.Errorf("No AMI was found matching filters: %v", params) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - if len(imageResp.Images) > 1 && !s.AmiFilters.MostRecent { - err := fmt.Errorf("Your query returned more than one result. Please try a more specific search, or set most_recent to true.") - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - var image *ec2.Image - if s.AmiFilters.MostRecent { - image = mostRecentAmi(imageResp.Images) - } else { - image = imageResp.Images[0] - } - ui.Message(fmt.Sprintf("Found Image ID: %s", *image.ImageId)) // Enhanced Networking can only be enabled on HVM AMIs. diff --git a/builder/amazon/ebs/acceptance/aws.go b/builder/amazon/ebs/acceptance/aws.go index 210bce310..bbccf3857 100644 --- a/builder/amazon/ebs/acceptance/aws.go +++ b/builder/amazon/ebs/acceptance/aws.go @@ -34,11 +34,13 @@ func (a *AWSHelper) CleanUpAmi() error { return fmt.Errorf("AWSAMICleanUp: Unable to find Image %s: %s", a.AMIName, err.Error()) } - _, err = regionconn.DeregisterImage(&ec2.DeregisterImageInput{ - ImageId: resp.Images[0].ImageId, - }) - if err != nil { - return fmt.Errorf("AWSAMICleanUp: Unable to Deregister Image %s", err.Error()) + if resp != nil && len(resp.Images) > 0 { + _, err = regionconn.DeregisterImage(&ec2.DeregisterImageInput{ + ImageId: resp.Images[0].ImageId, + }) + if err != nil { + return fmt.Errorf("AWSAMICleanUp: Unable to Deregister Image %s", err.Error()) + } } return nil } diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index d7c387ace..37804b215 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -118,7 +118,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) { errs = packersdk.MultiErrorAppend(errs, b.config.VolumeRunTag.CopyOn(&b.config.VolumeRunTags)...) - errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) + errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) errs = packersdk.MultiErrorAppend(errs, b.config.AMIConfig.Prepare(&b.config.AccessConfig, &b.config.ctx)...) errs = packersdk.MultiErrorAppend(errs, b.config.AMIMappings.Prepare(&b.config.ctx)...) diff --git a/builder/amazon/ebssurrogate/builder.go b/builder/amazon/ebssurrogate/builder.go index 4c5387442..0283fc43a 100644 --- a/builder/amazon/ebssurrogate/builder.go +++ b/builder/amazon/ebssurrogate/builder.go @@ -116,7 +116,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) { var warns []string errs = packersdk.MultiErrorAppend(errs, b.config.VolumeRunTag.CopyOn(&b.config.VolumeRunTags)...) - errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) + errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) errs = packersdk.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...) errs = packersdk.MultiErrorAppend(errs, b.config.AMIConfig.Prepare(&b.config.AccessConfig, &b.config.ctx)...) diff --git a/builder/amazon/ebsvolume/builder.go b/builder/amazon/ebsvolume/builder.go index 0115d9e54..f87228ad5 100644 --- a/builder/amazon/ebsvolume/builder.go +++ b/builder/amazon/ebsvolume/builder.go @@ -124,7 +124,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) { var errs *packersdk.MultiError var warns []string errs = packersdk.MultiErrorAppend(errs, b.config.VolumeRunTag.CopyOn(&b.config.VolumeRunTags)...) - errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) + errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) errs = packersdk.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...) errs = packersdk.MultiErrorAppend(errs, b.config.launchBlockDevices.Prepare(&b.config.ctx)...) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 56d33876a..373051585 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -178,7 +178,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) { // Accumulate any errors var errs *packersdk.MultiError var warns []string - errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) + errs = packersdk.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) errs = packersdk.MultiErrorAppend(errs, b.config.AMIMappings.Prepare(&b.config.ctx)...) errs = packersdk.MultiErrorAppend(errs, b.config.LaunchMappings.Prepare(&b.config.ctx)...) errs = packersdk.MultiErrorAppend(errs, diff --git a/cmd/packer-plugin-amazon/main.go b/cmd/packer-plugin-amazon/main.go index 245c56047..01d77d70d 100644 --- a/cmd/packer-plugin-amazon/main.go +++ b/cmd/packer-plugin-amazon/main.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/packer/builder/amazon/ebssurrogate" "github.com/hashicorp/packer/builder/amazon/ebsvolume" "github.com/hashicorp/packer/builder/osc/chroot" + amazonami "github.com/hashicorp/packer/datasource/amazon/ami" amazonimport "github.com/hashicorp/packer/post-processor/amazon-import" ) @@ -19,6 +20,7 @@ func main() { pps.RegisterBuilder("ebssurrogate", new(ebssurrogate.Builder)) pps.RegisterBuilder("ebsvolume", new(ebsvolume.Builder)) pps.RegisterPostProcessor("import", new(amazonimport.PostProcessor)) + pps.RegisterDatasource("ami", new(amazonami.Datasource)) err := pps.Run() if err != nil { fmt.Fprintln(os.Stderr, err.Error()) diff --git a/cmd/packer-plugin-check/main.go b/cmd/packer-plugins-check/main.go similarity index 96% rename from cmd/packer-plugin-check/main.go rename to cmd/packer-plugins-check/main.go index e7a57291e..885be184e 100644 --- a/cmd/packer-plugin-check/main.go +++ b/cmd/packer-plugins-check/main.go @@ -15,7 +15,7 @@ import ( "github.com/hashicorp/packer/packer/plugin" ) -const packerPluginCheck = "packer-plugin-check" +const packerPluginsCheck = "packer-plugins-check" var ( docs = flag.Bool("docs", false, "flag to indicate that documentation files should be checked.") @@ -24,15 +24,15 @@ var ( // Usage is a replacement usage function for the flags package. func Usage() { - fmt.Fprintf(os.Stderr, "Usage of "+packerPluginCheck+":\n") - fmt.Fprintf(os.Stderr, "\t"+packerPluginCheck+" [flags]\n") + fmt.Fprintf(os.Stderr, "Usage of "+packerPluginsCheck+":\n") + fmt.Fprintf(os.Stderr, "\t"+packerPluginsCheck+" [flags]\n") fmt.Fprintf(os.Stderr, "Flags:\n") flag.PrintDefaults() } func main() { log.SetFlags(0) - log.SetPrefix(packerPluginCheck + ": ") + log.SetPrefix(packerPluginsCheck + ": ") flag.Usage = Usage flag.Parse() diff --git a/command/plugin.go b/command/plugin.go index 49ee45148..cfc505700 100644 --- a/command/plugin.go +++ b/command/plugin.go @@ -65,6 +65,7 @@ import ( vsphereclonebuilder "github.com/hashicorp/packer/builder/vsphere/clone" vsphereisobuilder "github.com/hashicorp/packer/builder/vsphere/iso" yandexbuilder "github.com/hashicorp/packer/builder/yandex" + amazonamidatasource "github.com/hashicorp/packer/datasource/amazon/ami" alicloudimportpostprocessor "github.com/hashicorp/packer/post-processor/alicloud-import" amazonimportpostprocessor "github.com/hashicorp/packer/post-processor/amazon-import" artificepostprocessor "github.com/hashicorp/packer/post-processor/artifice" @@ -212,7 +213,9 @@ var PostProcessors = map[string]packersdk.PostProcessor{ "yandex-import": new(yandeximportpostprocessor.PostProcessor), } -var Datasources = map[string]packersdk.Datasource{} +var Datasources = map[string]packersdk.Datasource{ + "amazon-ami": new(amazonamidatasource.Datasource), +} var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner|datasource)-(.+)") diff --git a/datasource/amazon/ami/data.go b/datasource/amazon/ami/data.go new file mode 100644 index 000000000..77a10e1f5 --- /dev/null +++ b/datasource/amazon/ami/data.go @@ -0,0 +1,90 @@ +//go:generate mapstructure-to-hcl2 -type DatasourceOutput,Config +package ami + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer-plugin-sdk/hcl2helper" + packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer-plugin-sdk/template/config" + awscommon "github.com/hashicorp/packer/builder/amazon/common" + "github.com/zclconf/go-cty/cty" +) + +type Datasource struct { + config Config +} + +type Config struct { + awscommon.AccessConfig `mapstructure:",squash"` + awscommon.AmiFilterOptions `mapstructure:",squash"` +} + +func (d *Datasource) ConfigSpec() hcldec.ObjectSpec { + return d.config.FlatMapstructure().HCL2Spec() +} + +func (d *Datasource) Configure(raws ...interface{}) error { + err := config.Decode(&d.config, nil, raws...) + if err != nil { + return err + } + + var errs *packersdk.MultiError + errs = packersdk.MultiErrorAppend(errs, d.config.AccessConfig.Prepare()...) + + if d.config.Empty() { + errs = packersdk.MultiErrorAppend(errs, fmt.Errorf("The `filters` must be specified")) + } + if d.config.NoOwner() { + errs = packersdk.MultiErrorAppend(errs, fmt.Errorf("For security reasons, you must declare an owner.")) + } + + if errs != nil && len(errs.Errors) > 0 { + return errs + } + return nil +} + +type DatasourceOutput struct { + ID string `mapstructure:"id"` + Name string `mapstructure:"name"` + CreationDate string `mapstructure:"creation_date"` + Owner string `mapstructure:"owner"` + OwnerName string `mapstructure:"owner_name"` + Tags map[string]string `mapstructure:"tags"` +} + +func (d *Datasource) OutputSpec() hcldec.ObjectSpec { + return (&DatasourceOutput{}).FlatMapstructure().HCL2Spec() +} + +func (d *Datasource) Execute() (cty.Value, error) { + session, err := d.config.Session() + if err != nil { + return cty.NullVal(cty.EmptyObject), err + } + + image, err := d.config.AmiFilterOptions.GetFilteredImage(&ec2.DescribeImagesInput{}, ec2.New(session)) + if err != nil { + return cty.NullVal(cty.EmptyObject), err + } + + imageTags := make(map[string]string, len(image.Tags)) + for _, tag := range image.Tags { + imageTags[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value) + } + + output := DatasourceOutput{ + ID: aws.StringValue(image.ImageId), + Name: aws.StringValue(image.Name), + CreationDate: aws.StringValue(image.CreationDate), + Owner: aws.StringValue(image.OwnerId), + OwnerName: aws.StringValue(image.ImageOwnerAlias), + Tags: imageTags, + } + return hcl2helper.HCL2ValueFromConfig(output, d.OutputSpec()), nil +} diff --git a/datasource/amazon/ami/data.hcl2spec.go b/datasource/amazon/ami/data.hcl2spec.go new file mode 100644 index 000000000..5560d2a66 --- /dev/null +++ b/datasource/amazon/ami/data.hcl2spec.go @@ -0,0 +1,101 @@ +// Code generated by "mapstructure-to-hcl2 -type DatasourceOutput,Config"; DO NOT EDIT. + +package ami + +import ( + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer/builder/amazon/common" + "github.com/zclconf/go-cty/cty" +) + +// FlatConfig is an auto-generated flat version of Config. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatConfig struct { + AccessKey *string `mapstructure:"access_key" required:"true" cty:"access_key" hcl:"access_key"` + AssumeRole *common.FlatAssumeRoleConfig `mapstructure:"assume_role" required:"false" cty:"assume_role" hcl:"assume_role"` + CustomEndpointEc2 *string `mapstructure:"custom_endpoint_ec2" required:"false" cty:"custom_endpoint_ec2" hcl:"custom_endpoint_ec2"` + CredsFilename *string `mapstructure:"shared_credentials_file" required:"false" cty:"shared_credentials_file" hcl:"shared_credentials_file"` + DecodeAuthZMessages *bool `mapstructure:"decode_authorization_messages" required:"false" cty:"decode_authorization_messages" hcl:"decode_authorization_messages"` + InsecureSkipTLSVerify *bool `mapstructure:"insecure_skip_tls_verify" required:"false" cty:"insecure_skip_tls_verify" hcl:"insecure_skip_tls_verify"` + MaxRetries *int `mapstructure:"max_retries" required:"false" cty:"max_retries" hcl:"max_retries"` + MFACode *string `mapstructure:"mfa_code" required:"false" cty:"mfa_code" hcl:"mfa_code"` + ProfileName *string `mapstructure:"profile" required:"false" cty:"profile" hcl:"profile"` + RawRegion *string `mapstructure:"region" required:"true" cty:"region" hcl:"region"` + SecretKey *string `mapstructure:"secret_key" required:"true" cty:"secret_key" hcl:"secret_key"` + SkipMetadataApiCheck *bool `mapstructure:"skip_metadata_api_check" cty:"skip_metadata_api_check" hcl:"skip_metadata_api_check"` + SkipCredsValidation *bool `mapstructure:"skip_credential_validation" cty:"skip_credential_validation" hcl:"skip_credential_validation"` + Token *string `mapstructure:"token" required:"false" cty:"token" hcl:"token"` + VaultAWSEngine *common.FlatVaultAWSEngineOptions `mapstructure:"vault_aws_engine" required:"false" cty:"vault_aws_engine" hcl:"vault_aws_engine"` + PollingConfig *common.FlatAWSPollingConfig `mapstructure:"aws_polling" required:"false" cty:"aws_polling" hcl:"aws_polling"` + Filters map[string]string `mapstructure:"filters" cty:"filters" hcl:"filters"` + Owners []string `mapstructure:"owners" cty:"owners" hcl:"owners"` + MostRecent *bool `mapstructure:"most_recent" cty:"most_recent" hcl:"most_recent"` +} + +// FlatMapstructure returns a new FlatConfig. +// FlatConfig is an auto-generated flat version of Config. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatConfig) +} + +// HCL2Spec returns the hcl spec of a Config. +// This spec is used by HCL to read the fields of Config. +// The decoded values from this spec will then be applied to a FlatConfig. +func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "access_key": &hcldec.AttrSpec{Name: "access_key", Type: cty.String, Required: false}, + "assume_role": &hcldec.BlockSpec{TypeName: "assume_role", Nested: hcldec.ObjectSpec((*common.FlatAssumeRoleConfig)(nil).HCL2Spec())}, + "custom_endpoint_ec2": &hcldec.AttrSpec{Name: "custom_endpoint_ec2", Type: cty.String, Required: false}, + "shared_credentials_file": &hcldec.AttrSpec{Name: "shared_credentials_file", Type: cty.String, Required: false}, + "decode_authorization_messages": &hcldec.AttrSpec{Name: "decode_authorization_messages", Type: cty.Bool, Required: false}, + "insecure_skip_tls_verify": &hcldec.AttrSpec{Name: "insecure_skip_tls_verify", Type: cty.Bool, Required: false}, + "max_retries": &hcldec.AttrSpec{Name: "max_retries", Type: cty.Number, Required: false}, + "mfa_code": &hcldec.AttrSpec{Name: "mfa_code", Type: cty.String, Required: false}, + "profile": &hcldec.AttrSpec{Name: "profile", Type: cty.String, Required: false}, + "region": &hcldec.AttrSpec{Name: "region", Type: cty.String, Required: false}, + "secret_key": &hcldec.AttrSpec{Name: "secret_key", Type: cty.String, Required: false}, + "skip_metadata_api_check": &hcldec.AttrSpec{Name: "skip_metadata_api_check", Type: cty.Bool, Required: false}, + "skip_credential_validation": &hcldec.AttrSpec{Name: "skip_credential_validation", Type: cty.Bool, Required: false}, + "token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false}, + "vault_aws_engine": &hcldec.BlockSpec{TypeName: "vault_aws_engine", Nested: hcldec.ObjectSpec((*common.FlatVaultAWSEngineOptions)(nil).HCL2Spec())}, + "aws_polling": &hcldec.BlockSpec{TypeName: "aws_polling", Nested: hcldec.ObjectSpec((*common.FlatAWSPollingConfig)(nil).HCL2Spec())}, + "filters": &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false}, + "owners": &hcldec.AttrSpec{Name: "owners", Type: cty.List(cty.String), Required: false}, + "most_recent": &hcldec.AttrSpec{Name: "most_recent", Type: cty.Bool, Required: false}, + } + return s +} + +// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatDatasourceOutput struct { + ID *string `mapstructure:"id" cty:"id" hcl:"id"` + Name *string `mapstructure:"name" cty:"name" hcl:"name"` + CreationDate *string `mapstructure:"creation_date" cty:"creation_date" hcl:"creation_date"` + Owner *string `mapstructure:"owner" cty:"owner" hcl:"owner"` + OwnerName *string `mapstructure:"owner_name" cty:"owner_name" hcl:"owner_name"` + Tags map[string]string `mapstructure:"tags" cty:"tags" hcl:"tags"` +} + +// FlatMapstructure returns a new FlatDatasourceOutput. +// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*DatasourceOutput) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatDatasourceOutput) +} + +// HCL2Spec returns the hcl spec of a DatasourceOutput. +// This spec is used by HCL to read the fields of DatasourceOutput. +// The decoded values from this spec will then be applied to a FlatDatasourceOutput. +func (*FlatDatasourceOutput) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "id": &hcldec.AttrSpec{Name: "id", Type: cty.String, Required: false}, + "name": &hcldec.AttrSpec{Name: "name", Type: cty.String, Required: false}, + "creation_date": &hcldec.AttrSpec{Name: "creation_date", Type: cty.String, Required: false}, + "owner": &hcldec.AttrSpec{Name: "owner", Type: cty.String, Required: false}, + "owner_name": &hcldec.AttrSpec{Name: "owner_name", Type: cty.String, Required: false}, + "tags": &hcldec.AttrSpec{Name: "tags", Type: cty.Map(cty.String), Required: false}, + } + return s +} diff --git a/datasource/amazon/ami/data_acc_test.go b/datasource/amazon/ami/data_acc_test.go new file mode 100644 index 000000000..ced44bd3d --- /dev/null +++ b/datasource/amazon/ami/data_acc_test.go @@ -0,0 +1,64 @@ +package ami + +import ( + "fmt" + "os/exec" + "testing" + + "github.com/hashicorp/packer-plugin-sdk/acctest" + amazonacc "github.com/hashicorp/packer/builder/amazon/ebs/acceptance" +) + +func TestAmazonAmi(t *testing.T) { + testCase := &acctest.DatasourceTestCase{ + Name: "amazon_ami_datasource_basic_test", + Teardown: func() error { + helper := amazonacc.AWSHelper{ + Region: "us-west-2", + AMIName: "packer-amazon-ami-test", + } + return helper.CleanUpAmi() + }, + Template: testDatasourceBasic, + Type: "amazon-ami", + Check: func(buildCommand *exec.Cmd, logfile string) error { + if buildCommand.ProcessState != nil { + if buildCommand.ProcessState.ExitCode() != 0 { + return fmt.Errorf("Bad exit code. Logfile: %s", logfile) + } + } + return nil + }, + } + acctest.TestDatasource(t, testCase) +} + +const testDatasourceBasic = ` +data "amazon-ami" "test" { + filters = { + virtualization-type = "hvm" + name = "Windows_Server-2016-English-Full-Base-*" + root-device-type = "ebs" + } + most_recent = true + owners = ["801119661308"] +} + +source "amazon-ebs" "basic-example" { + user_data_file = "./test-fixtures/configure-source-ssh.ps1" + region = "us-west-2" + source_ami = data.amazon-ami.test.id + instance_type = "t2.small" + ssh_agent_auth = false + ami_name = "packer-amazon-ami-test" + communicator = "ssh" + ssh_timeout = "10m" + ssh_username = "Administrator" +} + +build { + sources = [ + "source.amazon-ebs.basic-example" + ] +} +` diff --git a/datasource/amazon/ami/data_test.go b/datasource/amazon/ami/data_test.go new file mode 100644 index 000000000..ff11431b2 --- /dev/null +++ b/datasource/amazon/ami/data_test.go @@ -0,0 +1,45 @@ +package ami + +import ( + "testing" + + awscommon "github.com/hashicorp/packer/builder/amazon/common" +) + +func TestDatasourceConfigure_FilterBlank(t *testing.T) { + datasource := Datasource{ + config: Config{ + AmiFilterOptions: awscommon.AmiFilterOptions{}, + }, + } + if err := datasource.Configure(nil); err == nil { + t.Fatalf("Should error if filters map is empty or not specified") + } +} + +func TestRunConfigPrepare_SourceAmiFilterOwnersBlank(t *testing.T) { + datasource := Datasource{ + config: Config{ + AmiFilterOptions: awscommon.AmiFilterOptions{ + Filters: map[string]string{"foo": "bar"}, + }, + }, + } + if err := datasource.Configure(nil); err == nil { + t.Fatalf("Should error if Owners is not specified)") + } +} + +func TestRunConfigPrepare_SourceAmiFilterGood(t *testing.T) { + datasource := Datasource{ + config: Config{ + AmiFilterOptions: awscommon.AmiFilterOptions{ + Owners: []string{"1234567"}, + Filters: map[string]string{"foo": "bar"}, + }, + }, + } + if err := datasource.Configure(nil); err != nil { + t.Fatalf("err: %s", err) + } +} diff --git a/datasource/amazon/ami/test-fixtures/configure-source-ssh.ps1 b/datasource/amazon/ami/test-fixtures/configure-source-ssh.ps1 new file mode 100644 index 000000000..7c12dd742 --- /dev/null +++ b/datasource/amazon/ami/test-fixtures/configure-source-ssh.ps1 @@ -0,0 +1,161 @@ + +# Version and download URL +$openSSHVersion = "8.1.0.0p1-Beta" +$openSSHURL = "https://github.com/PowerShell/Win32-OpenSSH/releases/download/v$openSSHVersion/OpenSSH-Win64.zip" + +Set-ExecutionPolicy Unrestricted + +# GitHub became TLS 1.2 only on Feb 22, 2018 +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12; + +# Function to unzip an archive to a given destination +Add-Type -AssemblyName System.IO.Compression.FileSystem +Function Unzip +{ + [CmdletBinding()] + param( + [Parameter(Mandatory=$true, Position=0)] + [string] $ZipFile, + [Parameter(Mandatory=$true, Position=1)] + [string] $OutPath + ) + + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $outPath) +} + +# Set various known paths +$openSSHZip = Join-Path $env:TEMP 'OpenSSH.zip' +$openSSHInstallDir = Join-Path $env:ProgramFiles 'OpenSSH' +$openSSHInstallScript = Join-Path $openSSHInstallDir 'install-sshd.ps1' +$openSSHDownloadKeyScript = Join-Path $openSSHInstallDir 'download-key-pair.ps1' +$openSSHDaemon = Join-Path $openSSHInstallDir 'sshd.exe' +$openSSHDaemonConfig = [io.path]::combine($env:ProgramData, 'ssh', 'sshd_config') + +# Download and unpack the binary distribution of OpenSSH +Invoke-WebRequest -Uri $openSSHURL ` + -OutFile $openSSHZip ` + -ErrorAction Stop + +Unzip -ZipFile $openSSHZip ` + -OutPath "$env:TEMP" ` + -ErrorAction Stop + +Remove-Item $openSSHZip ` + -ErrorAction SilentlyContinue + +# Move into Program Files +Move-Item -Path (Join-Path $env:TEMP 'OpenSSH-Win64') ` + -Destination $openSSHInstallDir ` + -ErrorAction Stop + +# Run the install script, terminate if it fails +& Powershell.exe -ExecutionPolicy Bypass -File $openSSHInstallScript +if ($LASTEXITCODE -ne 0) { + throw("Failed to install OpenSSH Server") +} + +# Add a firewall rule to allow inbound SSH connections to sshd.exe +New-NetFirewallRule -Name sshd ` + -DisplayName "OpenSSH Server (sshd)" ` + -Group "Remote Access" ` + -Description "Allow access via TCP port 22 to the OpenSSH Daemon" ` + -Enabled True ` + -Direction Inbound ` + -Protocol TCP ` + -LocalPort 22 ` + -Program "$openSSHDaemon" ` + -Action Allow ` + -ErrorAction Stop + +# Ensure sshd automatically starts on boot +Set-Service sshd -StartupType Automatic ` + -ErrorAction Stop + +# Set the default login shell for SSH connections to Powershell +New-Item -Path HKLM:\SOFTWARE\OpenSSH -Force +New-ItemProperty -Path HKLM:\SOFTWARE\OpenSSH ` + -Name DefaultShell ` + -Value "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" ` + -ErrorAction Stop + +$keyDownloadScript = @' +# Download the instance key pair and authorize Administrator logins using it +$openSSHAdminUser = 'c:\ProgramData\ssh' +$openSSHAuthorizedKeys = Join-Path $openSSHAdminUser 'authorized_keys' + +If (-Not (Test-Path $openSSHAdminUser)) { + New-Item -Path $openSSHAdminUser -Type Directory +} + +$keyUrl = "http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key" +$keyReq = [System.Net.WebRequest]::Create($keyUrl) +$keyResp = $keyReq.GetResponse() +$keyRespStream = $keyResp.GetResponseStream() + $streamReader = New-Object System.IO.StreamReader $keyRespStream +$keyMaterial = $streamReader.ReadToEnd() + +$keyMaterial | Out-File -Append -FilePath $openSSHAuthorizedKeys -Encoding ASCII + +# Ensure access control on authorized_keys meets the requirements +$acl = Get-ACL -Path $openSSHAuthorizedKeys +$acl.SetAccessRuleProtection($True, $True) +Set-Acl -Path $openSSHAuthorizedKeys -AclObject $acl + +$acl = Get-ACL -Path $openSSHAuthorizedKeys +$ar = New-Object System.Security.AccessControl.FileSystemAccessRule( ` + "NT Authority\Authenticated Users", "ReadAndExecute", "Allow") +$acl.RemoveAccessRule($ar) +$ar = New-Object System.Security.AccessControl.FileSystemAccessRule( ` + "BUILTIN\Administrators", "FullControl", "Allow") +$acl.RemoveAccessRule($ar) +$ar = New-Object System.Security.AccessControl.FileSystemAccessRule( ` + "BUILTIN\Users", "FullControl", "Allow") +$acl.RemoveAccessRule($ar) +Set-Acl -Path $openSSHAuthorizedKeys -AclObject $acl + +Disable-ScheduledTask -TaskName "Download Key Pair" + +$sshdConfigContent = @" +# Modified sshd_config, created by Packer provisioner + +PasswordAuthentication yes +PubKeyAuthentication yes +PidFile __PROGRAMDATA__/ssh/logs/sshd.pid +AuthorizedKeysFile __PROGRAMDATA__/ssh/authorized_keys +AllowUsers Administrator + +Subsystem sftp sftp-server.exe +"@ + +Set-Content -Path C:\ProgramData\ssh\sshd_config ` + -Value $sshdConfigContent + +'@ +$keyDownloadScript | Out-File $openSSHDownloadKeyScript + +# Create Task - Ensure the name matches the verbatim version above +$taskName = "Download Key Pair" +$principal = New-ScheduledTaskPrincipal ` + -UserID "NT AUTHORITY\SYSTEM" ` + -LogonType ServiceAccount ` + -RunLevel Highest +$action = New-ScheduledTaskAction -Execute 'Powershell.exe' ` + -Argument "-NoProfile -File ""$openSSHDownloadKeyScript""" +$trigger = New-ScheduledTaskTrigger -AtStartup +Register-ScheduledTask -Action $action ` + -Trigger $trigger ` + -Principal $principal ` + -TaskName $taskName ` + -Description $taskName +Disable-ScheduledTask -TaskName $taskName + +# Run the install script, terminate if it fails +& Powershell.exe -ExecutionPolicy Bypass -File $openSSHDownloadKeyScript +if ($LASTEXITCODE -ne 0) { + throw("Failed to download key pair") +} + +# Restart to ensure public key authentication works and SSH comes up +Restart-Computer + +true \ No newline at end of file diff --git a/post-processor/amazon-import/post-processor.go b/post-processor/amazon-import/post-processor.go index 49af07ba3..30a8e1c90 100644 --- a/post-processor/amazon-import/post-processor.go +++ b/post-processor/amazon-import/post-processor.go @@ -90,7 +90,7 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } // Check we have AWS access variables defined somewhere - errs = packersdk.MultiErrorAppend(errs, p.config.AccessConfig.Prepare(&p.config.ctx)...) + errs = packersdk.MultiErrorAppend(errs, p.config.AccessConfig.Prepare()...) // define all our required parameters templates := map[string]*string{ diff --git a/website/content/docs/datasources/amazon-ami.mdx b/website/content/docs/datasources/amazon-ami.mdx new file mode 100644 index 000000000..bf2ebe217 --- /dev/null +++ b/website/content/docs/datasources/amazon-ami.mdx @@ -0,0 +1,46 @@ +--- +description: | + The Amazon AMI data source provides information from an AMI that will be fetched based + on the filter options provided in the configuration. + +page_title: Amazon AMI - Data Source +sidebar_title: Amazon AMI +--- + +# Amazon AMI Data Source + +Type: `amazon-ami` + +The Amazon AMI data source will filter and fetch an Amazon AMI, and output all the AMI information that will +be then available to use in the [Amazon builders](/docs/builders/amazon/instance). + +-> **Note:** Data sources is a feature exclusively to HCL2 templates. + +Basic example of usage: + +```hcl +data "amazon-ami" "basic-example" { + filters = { + virtualization-type = "hvm" + name = "ubuntu/images/*ubuntu-xenial-16.04-amd64-server-*" + root-device-type = "ebs" + } + owners = ["099720109477"] + most_recent = true +} +``` +This selects the most recent Ubuntu 16.04 HVM EBS AMI from Canonical. Note that the data source will fail unless +*exactly* one AMI is returned. In the above example, `most_recent` will cause this to succeed by selecting the newest image. + +## Configuration Reference + +@include 'builder/amazon/common/AmiFilterOptions-not-required.mdx' + +## Output Data + +- `id` - The ID of the AMI. +- `name` - The name of the AMI. +- `creation_date` - The date of creation of the AMI. +- `owner` - The AWS account ID of the owner. +- `owner_name` - The owner alias. +- `tags` - The key/value combination of the tags assigned to the AMI. diff --git a/website/content/docs/datasources/index.mdx b/website/content/docs/datasources/index.mdx new file mode 100644 index 000000000..5a1570da9 --- /dev/null +++ b/website/content/docs/datasources/index.mdx @@ -0,0 +1,14 @@ +--- +description: | + Data sources allow data to be fetched for use in Packer configuration. Use of data sources + allows a build to use information defined outside of Packer. +page_title: Data Sources +sidebar_title: Data Sources +--- + +# Data Sources + +Data sources allow data to be fetched for use in Packer configuration. Use of data sources +allows a build to use information defined outside of Packer. + +-> **Note:** Data sources is a feature exclusively to HCL2 templates included in Packer `v1.7.0`. diff --git a/website/content/partials/builder/amazon/common/AmiFilterOptions-not-required.mdx b/website/content/partials/builder/amazon/common/AmiFilterOptions-not-required.mdx index 15064b6bd..bcd0f9bd2 100644 --- a/website/content/partials/builder/amazon/common/AmiFilterOptions-not-required.mdx +++ b/website/content/partials/builder/amazon/common/AmiFilterOptions-not-required.mdx @@ -1,3 +1,14 @@ - + -- `most_recent` (bool) - Most Recent +- `filters` (map[string]string) - Filters used to select an AMI. Any filter described in the docs for + [DescribeImages](http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html) + is valid. + +- `owners` ([]string) - Filters the images by their owner. You + may specify one or more AWS account IDs, "self" (which will use the + account whose credentials you are using to run Packer), or an AWS owner + alias: for example, `amazon`, `aws-marketplace`, or `microsoft`. This + option is required for security reasons. + +- `most_recent` (bool) - Selects the newest created image when true. + This is most useful for selecting a daily distro build. diff --git a/website/data/docs-navigation.js b/website/data/docs-navigation.js index b0b2cf52d..dc5018331 100644 --- a/website/data/docs-navigation.js +++ b/website/data/docs-navigation.js @@ -247,6 +247,7 @@ export default [ 'community-supported', ], }, + { category: 'datasources', content: ['amazon-ami'] }, { category: 'provisioners', content: [