From 27128dc62f4f3a17804b3948c3759d47cae7b001 Mon Sep 17 00:00:00 2001 From: js-g Date: Wed, 14 Oct 2020 17:09:09 -0400 Subject: [PATCH 1/4] feat (builder/oracle-oci): allow filtering on base image --- builder/oracle/oci/config.go | 32 +++++++++++---- builder/oracle/oci/config.hcl2spec.go | 35 +++++++++++++++- builder/oracle/oci/driver_oci.go | 59 ++++++++++++++++++--------- 3 files changed, 99 insertions(+), 27 deletions(-) diff --git a/builder/oracle/oci/config.go b/builder/oracle/oci/config.go index 0b3eb9bff..748beb9d3 100644 --- a/builder/oracle/oci/config.go +++ b/builder/oracle/oci/config.go @@ -1,4 +1,4 @@ -//go:generate mapstructure-to-hcl2 -type Config,CreateVNICDetails +//go:generate mapstructure-to-hcl2 -type Config,CreateVNICDetails,ListImagesRequest package oci @@ -35,6 +35,15 @@ type CreateVNICDetails struct { SubnetId *string `mapstructure:"subnet_id" required:"false"` } +type ListImagesRequest struct { + // fields that can be specified under "base_image_filter" + CompartmentId *string `mapstructure:"compartment_id"` + DisplayName *string `mapstructure:"display_name"` + OperatingSystem *string `mapstructure:"operating_system"` + OperatingSystemVersion *string `mapstructure:"operating_system_version"` + Shape *string `mapstructure:"shape"` +} + type Config struct { common.PackerConfig `mapstructure:",squash"` Comm communicator.Config `mapstructure:",squash"` @@ -69,12 +78,13 @@ type Config struct { CompartmentID string `mapstructure:"compartment_ocid"` // Image - BaseImageID string `mapstructure:"base_image_ocid"` - ImageName string `mapstructure:"image_name"` - ImageCompartmentID string `mapstructure:"image_compartment_ocid"` + BaseImageID string `mapstructure:"base_image_ocid"` + BaseImageFilter ListImagesRequest `mapstructure:"base_image_filter"` + ImageName string `mapstructure:"image_name"` + ImageCompartmentID string `mapstructure:"image_compartment_ocid"` // Instance - InstanceName string `mapstructure:"instance_name"` + InstanceName *string `mapstructure:"instance_name"` InstanceTags map[string]string `mapstructure:"instance_tags"` InstanceDefinedTags map[string]map[string]interface{} `mapstructure:"instance_defined_tags"` Shape string `mapstructure:"shape"` @@ -269,9 +279,17 @@ func (c *Config) Prepare(raws ...interface{}) error { errs, errors.New("'subnet_ocid' must be specified")) } - if c.BaseImageID == "" { + if (c.BaseImageID == "") && (c.BaseImageFilter == ListImagesRequest{}) { errs = packer.MultiErrorAppend( - errs, errors.New("'base_image_ocid' must be specified")) + errs, errors.New("'base_image_ocid' or 'base_image_filter' must be specified")) + } + + if c.BaseImageFilter.CompartmentId == nil { + c.BaseImageFilter.CompartmentId = &c.CompartmentID + } + + if c.BaseImageFilter.Shape == nil { + c.BaseImageFilter.Shape = &c.Shape } // Validate tag lengths. TODO (hlowndes) maximum number of tags allowed. diff --git a/builder/oracle/oci/config.hcl2spec.go b/builder/oracle/oci/config.hcl2spec.go index 647244f12..b7dcf8ba4 100644 --- a/builder/oracle/oci/config.hcl2spec.go +++ b/builder/oracle/oci/config.hcl2spec.go @@ -1,4 +1,4 @@ -// Code generated by "mapstructure-to-hcl2 -type Config,CreateVNICDetails"; DO NOT EDIT. +// Code generated by "mapstructure-to-hcl2 -type Config,CreateVNICDetails,ListImagesRequest"; DO NOT EDIT. package oci import ( @@ -76,6 +76,7 @@ type FlatConfig struct { AvailabilityDomain *string `mapstructure:"availability_domain" cty:"availability_domain" hcl:"availability_domain"` CompartmentID *string `mapstructure:"compartment_ocid" cty:"compartment_ocid" hcl:"compartment_ocid"` BaseImageID *string `mapstructure:"base_image_ocid" cty:"base_image_ocid" hcl:"base_image_ocid"` + BaseImageFilter *FlatListImagesRequest `mapstructure:"base_image_filter" cty:"base_image_filter" hcl:"base_image_filter"` ImageName *string `mapstructure:"image_name" cty:"image_name" hcl:"image_name"` ImageCompartmentID *string `mapstructure:"image_compartment_ocid" cty:"image_compartment_ocid" hcl:"image_compartment_ocid"` InstanceName *string `mapstructure:"instance_name" cty:"instance_name" hcl:"instance_name"` @@ -171,6 +172,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { "availability_domain": &hcldec.AttrSpec{Name: "availability_domain", Type: cty.String, Required: false}, "compartment_ocid": &hcldec.AttrSpec{Name: "compartment_ocid", Type: cty.String, Required: false}, "base_image_ocid": &hcldec.AttrSpec{Name: "base_image_ocid", Type: cty.String, Required: false}, + "base_image_filter": &hcldec.BlockSpec{TypeName: "base_image_filter", Nested: hcldec.ObjectSpec((*FlatListImagesRequest)(nil).HCL2Spec())}, "image_name": &hcldec.AttrSpec{Name: "image_name", Type: cty.String, Required: false}, "image_compartment_ocid": &hcldec.AttrSpec{Name: "image_compartment_ocid", Type: cty.String, Required: false}, "instance_name": &hcldec.AttrSpec{Name: "instance_name", Type: cty.String, Required: false}, @@ -227,3 +229,34 @@ func (*FlatCreateVNICDetails) HCL2Spec() map[string]hcldec.Spec { } return s } + +// FlatListImagesRequest is an auto-generated flat version of ListImagesRequest. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatListImagesRequest struct { + CompartmentId *string `mapstructure:"compartment_id" cty:"compartment_id" hcl:"compartment_id"` + DisplayName *string `mapstructure:"display_name" cty:"display_name" hcl:"display_name"` + OperatingSystem *string `mapstructure:"operating_system" cty:"operating_system" hcl:"operating_system"` + OperatingSystemVersion *string `mapstructure:"operating_system_version" cty:"operating_system_version" hcl:"operating_system_version"` + Shape *string `mapstructure:"shape" cty:"shape" hcl:"shape"` +} + +// FlatMapstructure returns a new FlatListImagesRequest. +// FlatListImagesRequest is an auto-generated flat version of ListImagesRequest. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*ListImagesRequest) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatListImagesRequest) +} + +// HCL2Spec returns the hcl spec of a ListImagesRequest. +// This spec is used by HCL to read the fields of ListImagesRequest. +// The decoded values from this spec will then be applied to a FlatListImagesRequest. +func (*FlatListImagesRequest) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "compartment_id": &hcldec.AttrSpec{Name: "compartment_id", Type: cty.String, Required: false}, + "display_name": &hcldec.AttrSpec{Name: "display_name", Type: cty.String, Required: false}, + "operating_system": &hcldec.AttrSpec{Name: "operating_system", Type: cty.String, Required: false}, + "operating_system_version": &hcldec.AttrSpec{Name: "operating_system_version", Type: cty.String, Required: false}, + "shape": &hcldec.AttrSpec{Name: "shape", Type: cty.String, Required: false}, + } + return s +} diff --git a/builder/oracle/oci/driver_oci.go b/builder/oracle/oci/driver_oci.go index 3901a0bac..b3a261ca7 100644 --- a/builder/oracle/oci/driver_oci.go +++ b/builder/oracle/oci/driver_oci.go @@ -51,22 +51,7 @@ func (d *driverOCI) CreateInstance(ctx context.Context, publicKey string) (strin metadata["user_data"] = d.cfg.UserData } - instanceDetails := core.LaunchInstanceDetails{ - AvailabilityDomain: &d.cfg.AvailabilityDomain, - CompartmentId: &d.cfg.CompartmentID, - DefinedTags: d.cfg.InstanceDefinedTags, - FreeformTags: d.cfg.InstanceTags, - Shape: &d.cfg.Shape, - SubnetId: &d.cfg.SubnetID, - Metadata: metadata, - } - - // When empty, the default display name is used. - if d.cfg.InstanceName != "" { - instanceDetails.DisplayName = &d.cfg.InstanceName - } - - // Pass VNIC details, if specified, to the instance + // Create VNIC details for instance CreateVnicDetails := core.CreateVnicDetails{ AssignPublicIp: d.cfg.CreateVnicDetails.AssignPublicIp, DisplayName: d.cfg.CreateVnicDetails.DisplayName, @@ -79,14 +64,50 @@ func (d *driverOCI) CreateInstance(ctx context.Context, publicKey string) (strin FreeformTags: d.cfg.CreateVnicDetails.FreeformTags, } - instanceDetails.CreateVnicDetails = &CreateVnicDetails + // Determine base image ID + var imageId *string + if d.cfg.BaseImageID != "" { + imageId = &d.cfg.BaseImageID + } else { + response, err := d.computeClient.ListImages(ctx, core.ListImagesRequest{ + CompartmentId: d.cfg.BaseImageFilter.CompartmentId, + DisplayName: d.cfg.BaseImageFilter.DisplayName, + OperatingSystem: d.cfg.BaseImageFilter.OperatingSystem, + OperatingSystemVersion: d.cfg.BaseImageFilter.OperatingSystemVersion, + Shape: d.cfg.BaseImageFilter.Shape, + LifecycleState: "AVAILABLE", + SortBy: "TIMECREATED", + SortOrder: "DESC", + }) + if err != nil { + return "", err + } + if len(response.Items) == 0 { + return "", errors.New("base_image_filter returned no images") + } + imageId = response.Items[0].Id + } // Create Source details which will be used to Launch Instance - instanceDetails.SourceDetails = core.InstanceSourceViaImageDetails{ - ImageId: &d.cfg.BaseImageID, + InstanceSourceDetails := core.InstanceSourceViaImageDetails{ + ImageId: imageId, BootVolumeSizeInGBs: &d.cfg.BootVolumeSizeInGBs, } + // Build instance details + instanceDetails := core.LaunchInstanceDetails{ + AvailabilityDomain: &d.cfg.AvailabilityDomain, + CompartmentId: &d.cfg.CompartmentID, + CreateVnicDetails: &CreateVnicDetails, + DefinedTags: d.cfg.InstanceDefinedTags, + DisplayName: d.cfg.InstanceName, + FreeformTags: d.cfg.InstanceTags, + Shape: &d.cfg.Shape, + SourceDetails: InstanceSourceDetails, + SubnetId: &d.cfg.SubnetID, + Metadata: metadata, + } + instance, err := d.computeClient.LaunchInstance(context.TODO(), core.LaunchInstanceRequest{LaunchInstanceDetails: instanceDetails}) if err != nil { From 0c34d6ca129fa6b44d95230bdc503c2532aacab8 Mon Sep 17 00:00:00 2001 From: js-g Date: Thu, 15 Oct 2020 15:28:38 -0400 Subject: [PATCH 2/4] feat (builder/oracle-oci): add regex search on base image name --- builder/oracle/oci/config.go | 1 + builder/oracle/oci/config.hcl2spec.go | 2 ++ builder/oracle/oci/driver_oci.go | 22 +++++++++++++++++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/builder/oracle/oci/config.go b/builder/oracle/oci/config.go index 748beb9d3..cc47c893e 100644 --- a/builder/oracle/oci/config.go +++ b/builder/oracle/oci/config.go @@ -39,6 +39,7 @@ type ListImagesRequest struct { // fields that can be specified under "base_image_filter" CompartmentId *string `mapstructure:"compartment_id"` DisplayName *string `mapstructure:"display_name"` + DisplayNameSearch *string `mapstructure:"display_name_search"` OperatingSystem *string `mapstructure:"operating_system"` OperatingSystemVersion *string `mapstructure:"operating_system_version"` Shape *string `mapstructure:"shape"` diff --git a/builder/oracle/oci/config.hcl2spec.go b/builder/oracle/oci/config.hcl2spec.go index b7dcf8ba4..678fc8376 100644 --- a/builder/oracle/oci/config.hcl2spec.go +++ b/builder/oracle/oci/config.hcl2spec.go @@ -235,6 +235,7 @@ func (*FlatCreateVNICDetails) HCL2Spec() map[string]hcldec.Spec { type FlatListImagesRequest struct { CompartmentId *string `mapstructure:"compartment_id" cty:"compartment_id" hcl:"compartment_id"` DisplayName *string `mapstructure:"display_name" cty:"display_name" hcl:"display_name"` + DisplayNameSearch *string `mapstructure:"display_name_search" cty:"display_name_search" hcl:"display_name_search"` OperatingSystem *string `mapstructure:"operating_system" cty:"operating_system" hcl:"operating_system"` OperatingSystemVersion *string `mapstructure:"operating_system_version" cty:"operating_system_version" hcl:"operating_system_version"` Shape *string `mapstructure:"shape" cty:"shape" hcl:"shape"` @@ -254,6 +255,7 @@ func (*FlatListImagesRequest) HCL2Spec() map[string]hcldec.Spec { s := map[string]hcldec.Spec{ "compartment_id": &hcldec.AttrSpec{Name: "compartment_id", Type: cty.String, Required: false}, "display_name": &hcldec.AttrSpec{Name: "display_name", Type: cty.String, Required: false}, + "display_name_search": &hcldec.AttrSpec{Name: "display_name_search", Type: cty.String, Required: false}, "operating_system": &hcldec.AttrSpec{Name: "operating_system", Type: cty.String, Required: false}, "operating_system_version": &hcldec.AttrSpec{Name: "operating_system_version", Type: cty.String, Required: false}, "shape": &hcldec.AttrSpec{Name: "shape", Type: cty.String, Required: false}, diff --git a/builder/oracle/oci/driver_oci.go b/builder/oracle/oci/driver_oci.go index b3a261ca7..0178741ea 100644 --- a/builder/oracle/oci/driver_oci.go +++ b/builder/oracle/oci/driver_oci.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "regexp" "time" core "github.com/oracle/oci-go-sdk/core" @@ -69,6 +70,7 @@ func (d *driverOCI) CreateInstance(ctx context.Context, publicKey string) (strin if d.cfg.BaseImageID != "" { imageId = &d.cfg.BaseImageID } else { + // Pull images and determine which image ID to use, if BaseImageId not specified response, err := d.computeClient.ListImages(ctx, core.ListImagesRequest{ CompartmentId: d.cfg.BaseImageFilter.CompartmentId, DisplayName: d.cfg.BaseImageFilter.DisplayName, @@ -85,7 +87,25 @@ func (d *driverOCI) CreateInstance(ctx context.Context, publicKey string) (strin if len(response.Items) == 0 { return "", errors.New("base_image_filter returned no images") } - imageId = response.Items[0].Id + if d.cfg.BaseImageFilter.DisplayNameSearch != nil { + // Return most recent image that matches regex + imageNameRegex, err := regexp.Compile(*d.cfg.BaseImageFilter.DisplayNameSearch) + if err != nil { + return "", err + } + for _, image := range response.Items { + if imageNameRegex.MatchString(*image.DisplayName) { + imageId = image.Id + break + } + } + if imageId == nil { + return "", errors.New("No image matched display_name_search criteria") + } + } else { + // If no regex provided, simply return most recent image pulled + imageId = response.Items[0].Id + } } // Create Source details which will be used to Launch Instance From b0cc71d35ea410ee70a7d440148ca28ff4f5b54d Mon Sep 17 00:00:00 2001 From: js-g Date: Thu, 15 Oct 2020 17:08:03 -0400 Subject: [PATCH 3/4] docs (builder/oracle-oci): add documentaion for base_image_filter --- website/pages/docs/builders/oracle/oci.mdx | 38 +++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/website/pages/docs/builders/oracle/oci.mdx b/website/pages/docs/builders/oracle/oci.mdx index 9e9b7a82b..c96c7a172 100644 --- a/website/pages/docs/builders/oracle/oci.mdx +++ b/website/pages/docs/builders/oracle/oci.mdx @@ -70,6 +70,24 @@ can also be supplied to override the typical auto-generated key: [ListImages](https://docs.us-phoenix-1.oraclecloud.com/api/#/en/iaas/latest/Image/ListImages) operation available in the Core Services API. +- `base_image_filter` (map of strings) - As an alternative to providing `base_image_ocid`, + the user can supply search criteria, and Packer will use the the most recent image that meets + all search criteria. If no image meets all search criteria, Packer returns an error. The + following fields, if specified, must match exactly: + - `compartment_id` - The OCID of the compartment to find the image. If not specified, will use `compartment_ocid` + used for the instance. + - `display_name` - The full name of the image, e.g., `Oracle-Linux-7.8-2020.05.26-0` + - `operating_system` - The operating system used on the image, e.g., `Oracle Linux` + - `operating_system_version` - The version of the operating system on the image, e.g., `7.8` + - `shape` - A shape that the image supports. If not specified, will use `shape` used for the instance + + Additionally, the following field takes a regular expression: + - `display_name_search` - a regular expression for the display name, e.g., `^Oracle-Linux`. This + is ignored if `display_name` is also specified under `base_image_filter`. If no images match + the expression, Packer returns an error. If multiple images match, the most recent is used. + + `base_image_filter` is ignored if `base_image_ocid` is also specified. + - `compartment_ocid` (string) - The OCID of the [compartment](https://docs.us-phoenix-1.oraclecloud.com/Content/GSG/Tasks/choosingcompartments.htm) that the instance will run in. @@ -274,7 +292,7 @@ to the instance. Depending on network (VCN and subnet) setup, this may be required for Packer to successfully SSH into the instance. NSGs are a property of the virtual network interface card (VNIC) attached to the instance, and are listed in `nsg_ids` under `create_vnic_details`. -``` +```json { "name": "base-image-{{isotime \"20060102030405\"}}", "type": "oracle-oci", @@ -303,3 +321,21 @@ are listed in `nsg_ids` under `create_vnic_details`. } } ``` + +## Base Image Filter Example +Note that `base_image_filter` gets passed as a string, then interpreted as a +regular expression. This means that all back-slashes must be doubled, e.g., +use `\\w+` to mean `\w+`, and `\\\\` to create the regular expression equivalent +of `\\` (which will search for a literal back-slash). +```json +{ + "name": "base-image-{{isotime \"20060102030405\"}}", + "type": "oracle-oci", + "availability_domain": "aaaa:PHX-AD-1", + "base_image_filter": { + "operating_system": "Oracle Linux", + "operating_system_version": "7.8", + "display_name_search": "^Oracle-Linux-7\\.8-2020\\.\\d+" + } + ... +} From 2f26e6d823de9a6f59dec254b7240890662185de Mon Sep 17 00:00:00 2001 From: js-g Date: Mon, 19 Oct 2020 23:28:28 -0400 Subject: [PATCH 4/4] tests (builder/oracle-oci): test base image filter in config --- builder/oracle/oci/config_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/builder/oracle/oci/config_test.go b/builder/oracle/oci/config_test.go index 1026d8aed..2dc9d2162 100644 --- a/builder/oracle/oci/config_test.go +++ b/builder/oracle/oci/config_test.go @@ -88,6 +88,36 @@ func TestConfig(t *testing.T) { } }) + t.Run("BaseImageFilterWithoutOCID", func(t *testing.T) { + raw := testConfig(cfgFile) + raw["base_image_ocid"] = "" + raw["base_image_filter"] = map[string]interface{}{ + "display_name": "hello_world", + } + + var c Config + errs := c.Prepare(raw) + + if errs != nil { + t.Fatalf("Unexpected error in configuration %+v", errs) + } + }) + + t.Run("BaseImageFilterDefault", func(t *testing.T) { + raw := testConfig(cfgFile) + + var c Config + errs := c.Prepare(raw) + if errs != nil { + t.Fatalf("Unexpected error in configuration %+v", errs) + } + + if *c.BaseImageFilter.Shape != raw["shape"] { + t.Fatalf("Default base_image_filter shape %v does not equal config shape %v", + *c.BaseImageFilter.Shape, raw["shape"]) + } + }) + t.Run("NoAccessConfig", func(t *testing.T) { raw := testConfig(cfgFile) delete(raw, "access_cfg_file")