From 524b822e11e2ae4049492dbea8836d6572f8544a Mon Sep 17 00:00:00 2001 From: Gennady Lipenkov Date: Tue, 10 Sep 2019 17:52:42 +0300 Subject: [PATCH] Support GPU instances, allow set source image by name Signed-off-by: Gennady Lipenkov --- builder/yandex/builder.go | 2 +- builder/yandex/config.go | 26 +++++++++++--- builder/yandex/config_test.go | 19 ++++++++++ builder/yandex/driver.go | 1 + builder/yandex/driver_yc.go | 12 +++++++ builder/yandex/ssh.go | 10 ++++++ builder/yandex/step_create_instance.go | 17 ++++++--- builder/yandex/step_instance_info.go | 2 +- website/source/docs/builders/yandex.html.md | 40 +++++++++++---------- 9 files changed, 100 insertions(+), 29 deletions(-) create mode 100644 builder/yandex/ssh.go diff --git a/builder/yandex/builder.go b/builder/yandex/builder.go index 5020fec2b..38ea7524d 100644 --- a/builder/yandex/builder.go +++ b/builder/yandex/builder.go @@ -61,7 +61,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack &stepInstanceInfo{}, &communicator.StepConnect{ Config: &b.config.Communicator, - Host: communicator.CommHost(b.config.Communicator.SSHHost, "instance_ip"), + Host: commHost, SSHConfig: b.config.Communicator.SSHConfigFunc(), }, &common.StepProvision{}, diff --git a/builder/yandex/config.go b/builder/yandex/config.go index 813f161d4..bc6520c8c 100644 --- a/builder/yandex/config.go +++ b/builder/yandex/config.go @@ -18,6 +18,8 @@ import ( ) const defaultEndpoint = "api.cloud.yandex.net:443" +const defaultGpuPlatformID = "gpu-standard-v1" +const defaultPlatformID = "standard-v1" const defaultZone = "ru-central1-a" var reImageFamily = regexp.MustCompile(`^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) @@ -40,6 +42,7 @@ type Config struct { ImageName string `mapstructure:"image_name"` ImageProductIDs []string `mapstructure:"image_product_ids"` InstanceCores int `mapstructure:"instance_cores"` + InstanceGpus int `mapstructure:"instance_gpus"` InstanceMemory int `mapstructure:"instance_mem_gb"` InstanceName string `mapstructure:"instance_name"` Labels map[string]string `mapstructure:"labels"` @@ -51,6 +54,7 @@ type Config struct { SourceImageFamily string `mapstructure:"source_image_family"` SourceImageFolderID string `mapstructure:"source_image_folder_id"` SourceImageID string `mapstructure:"source_image_id"` + SourceImageName string `mapstructure:"source_image_name"` SubnetID string `mapstructure:"subnet_id"` UseIPv4Nat bool `mapstructure:"use_ipv4_nat"` UseIPv6 bool `mapstructure:"use_ipv6"` @@ -134,7 +138,10 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { } if c.PlatformID == "" { - c.PlatformID = "standard-v1" + c.PlatformID = defaultPlatformID + if c.InstanceGpus != 0 { + c.PlatformID = defaultGpuPlatformID + } } if es := c.Communicator.Prepare(&c.ctx); len(es) > 0 { @@ -142,9 +149,15 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { } // Process required parameters. - if c.SourceImageID == "" && c.SourceImageFamily == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("a source_image_id or source_image_family must be specified")) + if c.SourceImageID == "" { + if c.SourceImageFamily == "" && c.SourceImageName == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("a source_image_name or source_image_family must be specified")) + } + if c.SourceImageFamily != "" && c.SourceImageName != "" { + errs = packer.MultiErrorAppend( + errs, errors.New("one of source_image_name or source_image_family must be specified, not both")) + } } if c.Endpoint == "" { @@ -168,6 +181,11 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { c.FolderID = os.Getenv("YC_FOLDER_ID") } + if c.PlatformID != defaultGpuPlatformID && c.InstanceGpus != 0 { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("for instances with gpu platform_id must be specified as `%s`", defaultGpuPlatformID)) + } + if c.Token == "" && c.ServiceAccountKeyFile == "" { errs = packer.MultiErrorAppend( errs, errors.New("a token or service account key file must be specified")) diff --git a/builder/yandex/config_test.go b/builder/yandex/config_test.go index df8e7a0bb..7e777d0e7 100644 --- a/builder/yandex/config_test.go +++ b/builder/yandex/config_test.go @@ -214,6 +214,25 @@ func TestZone(t *testing.T) { } } +func TestGpuDefaultPlatformID(t *testing.T) { + raw := testConfig(t) + raw["instance_gpus"] = 1 + + c, _, _ := NewConfig(raw) + if c.PlatformID != "gpu-standard-v1" { + t.Fatalf("expected 'gpu-standard-v1', but got '%s'", c.PlatformID) + } +} + +func TestGpuWrongPlatformID(t *testing.T) { + raw := testConfig(t) + raw["instance_gpus"] = 1 + raw["platform_id"] = "standard-v1" + + _, warns, errs := NewConfig(raw) + testConfigErr(t, warns, errs, "incompatible GPU platform_id") +} + // Helper stuff below func testConfig(t *testing.T) (config map[string]interface{}) { diff --git a/builder/yandex/driver.go b/builder/yandex/driver.go index 83585e368..39c38c0a1 100644 --- a/builder/yandex/driver.go +++ b/builder/yandex/driver.go @@ -11,6 +11,7 @@ type Driver interface { SDK() *ycsdk.SDK GetImage(imageID string) (*Image, error) GetImageFromFolder(ctx context.Context, folderID string, family string) (*Image, error) + GetImageFromFolderByName(ctx context.Context, folderID string, name string) (*Image, error) DeleteDisk(ctx context.Context, diskID string) error DeleteInstance(ctx context.Context, instanceID string) error DeleteSubnet(ctx context.Context, subnetID string) error diff --git a/builder/yandex/driver_yc.go b/builder/yandex/driver_yc.go index 45a46f274..f68916848 100644 --- a/builder/yandex/driver_yc.go +++ b/builder/yandex/driver_yc.go @@ -2,6 +2,7 @@ package yandex import ( "context" + "fmt" "log" "github.com/hashicorp/packer/helper/useragent" @@ -15,6 +16,7 @@ import ( ycsdk "github.com/yandex-cloud/go-sdk" "github.com/yandex-cloud/go-sdk/iamkey" "github.com/yandex-cloud/go-sdk/pkg/requestid" + "github.com/yandex-cloud/go-sdk/sdkresolvers" ) type driverYC struct { @@ -110,6 +112,16 @@ func (d *driverYC) GetImageFromFolder(ctx context.Context, folderID string, fami }, nil } +func (d *driverYC) GetImageFromFolderByName(ctx context.Context, folderID string, imageName string) (*Image, error) { + imageResolver := sdkresolvers.ImageResolver(imageName, sdkresolvers.FolderID(folderID)) + + if err := d.sdk.Resolve(ctx, imageResolver); err != nil { + return nil, fmt.Errorf("failed to resolve image name: %s", err) + } + + return d.GetImage(imageResolver.ID()) +} + func (d *driverYC) DeleteImage(ID string) error { ctx := context.TODO() op, err := d.sdk.WrapOperation(d.sdk.Compute().Image().Delete(ctx, &compute.DeleteImageRequest{ diff --git a/builder/yandex/ssh.go b/builder/yandex/ssh.go new file mode 100644 index 000000000..c32f16b53 --- /dev/null +++ b/builder/yandex/ssh.go @@ -0,0 +1,10 @@ +package yandex + +import ( + "github.com/hashicorp/packer/helper/multistep" +) + +func commHost(state multistep.StateBag) (string, error) { + ipAddress := state.Get("instance_ip").(string) + return ipAddress, nil +} diff --git a/builder/yandex/step_create_instance.go b/builder/yandex/step_create_instance.go index 538d78ad7..5d7b28f10 100644 --- a/builder/yandex/step_create_instance.go +++ b/builder/yandex/step_create_instance.go @@ -91,11 +91,19 @@ func getImage(ctx context.Context, c *Config, d Driver) (*Image, error) { return d.GetImage(c.SourceImageID) } - familyName := c.SourceImageFamily - if c.SourceImageFolderID != "" { - return d.GetImageFromFolder(ctx, c.SourceImageFolderID, familyName) + folderID := c.SourceImageFolderID + if folderID == "" { + folderID = StandardImagesFolderID } - return d.GetImageFromFolder(ctx, StandardImagesFolderID, familyName) + + switch { + case c.SourceImageFamily != "": + return d.GetImageFromFolder(ctx, folderID, c.SourceImageFamily) + case c.SourceImageName != "": + return d.GetImageFromFolderByName(ctx, folderID, c.SourceImageName) + } + + return &Image{}, errors.New("neither source_image_name nor source_image_family defined in config") } func (s *stepCreateInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { @@ -173,6 +181,7 @@ runcmd: ResourcesSpec: &compute.ResourcesSpec{ Memory: toBytes(config.InstanceMemory), Cores: int64(config.InstanceCores), + Gpus: int64(config.InstanceGpus), }, Metadata: instanceMetadata, BootDiskSpec: &compute.AttachedDiskSpec{ diff --git a/builder/yandex/step_instance_info.go b/builder/yandex/step_instance_info.go index 1a3d77e78..fa85c052d 100644 --- a/builder/yandex/step_instance_info.go +++ b/builder/yandex/step_instance_info.go @@ -20,7 +20,7 @@ func (s *stepInstanceInfo) Run(ctx context.Context, state multistep.StateBag) mu c := state.Get("config").(*Config) instanceID := state.Get("instance_id").(string) - ui.Say("Waiting for instance to become active...") + ui.Say(fmt.Sprintf("Waiting for instance with id %s to become active...", instanceID)) ctx, cancel := context.WithTimeout(ctx, c.StateTimeout) defer cancel() diff --git a/website/source/docs/builders/yandex.html.md b/website/source/docs/builders/yandex.html.md index e28cde4e7..80f43a742 100644 --- a/website/source/docs/builders/yandex.html.md +++ b/website/source/docs/builders/yandex.html.md @@ -72,15 +72,21 @@ can be configured for this builder. ### Optional: +- `endpoint` (string) - Non standard api endpoint URL. + +- `instance_cores` (number) - The number of cores available to the instance. + +- `instance_gpus` (number) - The number of GPU available to the instance. + +- `instance_mem_gb` (number) - The amount of memory available to the instance, specified in gigabytes. + - `disk_name` (string) - The name of the disk, if unset the instance name will be used. - + - `disk_size_gb` (number) - The size of the disk in GB. This defaults to `10`, which is 10GB. - `disk_type` (string) - Specify disk type for the launched instance. Defaults to `network-hdd`. -- `endpoint` (string) - Non standard api endpoint URL. - - `image_description` (string) - The description of the resulting image. - `image_family` (string) - The family name of the resulting image. @@ -93,21 +99,15 @@ can be configured for this builder. - `image_product_ids` (list) - License IDs that indicate which licenses are attached to resulting image. -- `instance_cores` (number) - The number of cores available to the instance. - -- `instance_mem_gb` (number) - The amount of memory available to the instance, specified in gigabytes. - - `instance_name` (string) - The name assigned to the instance. - `labels` (object of key/value strings) - Key/value pair labels to apply to the launched instance. -- `metadata` (object of key/value strings) - Metadata applied to the launched - instance. - - `platform_id` (string) - Identifier of the hardware platform configuration for the instance. This defaults to `standard-v1`. -- `preemptible` (boolean) - Launch a preemptible instance. This defaults to `false`. +- `metadata` (object of key/value strings) - Metadata applied to the launched + instance. - `serial_log_file` (string) - File path to save serial port output of the launched instance. @@ -115,18 +115,10 @@ can be configured for this builder. is an alternative method to authenticate to Yandex.Cloud. Alternatively you may set environment variable `YC_SERVICE_ACCOUNT_KEY_FILE`. -- `source_image_folder_id` (string) - The ID of the folder containing the source image. - -- `source_image_id` (string) - The source image ID to use to create the new image - from. - - `source_image_family` (string) - The source image family to create the new image from. The image family always returns its latest image that is not deprecated. Example: `ubuntu-1804-lts`. -- `state_timeout` (string) - The time to wait for instance state changes. - Defaults to `5m`. - - `subnet_id` (string) - The Yandex VPC subnet id to use for the launched instance. Note, the zone of the subnet must match the `zone` in which the VM is launched. @@ -141,4 +133,14 @@ can be configured for this builder. created. This defaults to `false`, or not enabled. -> **Note:** ~> Usage of IPv6 will be available in the future. +- `source_image_folder_id` (string) - The ID of the folder containing the source image. + +- `source_image_id` (string) - The source image ID to use to create the new image from. + +- `source_image_name` (string) - The source image name to use to create the new image from. + Name will be looked up in `source_image_folder_id`. + +- `state_timeout` (string) - The time to wait for instance state changes. + Defaults to `5m`. + - `zone` (string) - The name of the zone to launch the instance. This defaults to `ru-central1-a`.