From 2c513fce64caaae60b22156de2b49183d2b817f5 Mon Sep 17 00:00:00 2001 From: "mingsheng.su" Date: Fri, 18 Oct 2019 14:52:20 +0800 Subject: [PATCH] update ucloud packer --- builder/ucloud/common/access_config.go | 2 + builder/ucloud/common/consts.go | 4 + builder/ucloud/common/image_config.go | 17 ++- builder/ucloud/uhost/builder.go | 7 +- builder/ucloud/uhost/step_copy_image.go | 15 +- builder/ucloud/uhost/step_create_image.go | 2 +- builder/ucloud/uhost/step_create_instance.go | 1 - command/plugin.go | 2 +- examples/ucloud/local/http/centos-6.10/ks.cfg | 25 ++++ examples/ucloud/local/http/centos-6.8/ks.cfg | 42 ------ examples/ucloud/local/local.json | 39 +++-- examples/ucloud/local/test.json | 62 -------- .../ucloud-import/post-processor.go | 134 ++++++++++-------- .../source/docs/builders/ucloud-uhost.html.md | 2 + .../post-processors/ucloud-import.html.md | 75 ++++++++++ 15 files changed, 229 insertions(+), 200 deletions(-) create mode 100644 examples/ucloud/local/http/centos-6.10/ks.cfg delete mode 100644 examples/ucloud/local/http/centos-6.8/ks.cfg delete mode 100644 examples/ucloud/local/test.json create mode 100644 website/source/docs/post-processors/ucloud-import.html.md diff --git a/builder/ucloud/common/access_config.go b/builder/ucloud/common/access_config.go index 9cfdaf695..b6be2b7b5 100644 --- a/builder/ucloud/common/access_config.go +++ b/builder/ucloud/common/access_config.go @@ -37,6 +37,8 @@ func (c *AccessConfig) Client() (*UCloudClient, error) { cfg.BaseUrl = c.BaseUrl } cfg.UserAgent = fmt.Sprintf("Packer-UCloud/%s", version.FormattedVersion()) + // set default max retry count + cfg.MaxRetries = 3 cred := auth.NewCredential() cred.PublicKey = c.PublicKey diff --git a/builder/ucloud/common/consts.go b/builder/ucloud/common/consts.go index 06d8becda..87d32b11c 100644 --- a/builder/ucloud/common/consts.go +++ b/builder/ucloud/common/consts.go @@ -18,6 +18,10 @@ const ( IpTypePrivate = "Private" ) +const ( + DefaultCreateImageTimeOut = 3600 +) + var BootDiskTypeMap = NewStringConverter(map[string]string{ "cloud_ssd": "CLOUD_SSD", "local_normal": "LOCAL_NORMAL", diff --git a/builder/ucloud/common/image_config.go b/builder/ucloud/common/image_config.go index 79f105917..e469d67f2 100644 --- a/builder/ucloud/common/image_config.go +++ b/builder/ucloud/common/image_config.go @@ -17,19 +17,20 @@ type ImageDestination struct { } type ImageConfig struct { - ImageName string `mapstructure:"image_name"` - ImageDescription string `mapstructure:"image_description"` - ImageDestinations []ImageDestination `mapstructure:"image_copy_to_mappings"` + ImageName string `mapstructure:"image_name"` + ImageDescription string `mapstructure:"image_description"` + ImageDestinations []ImageDestination `mapstructure:"image_copy_to_mappings"` + WaitImageReadyTimeout int `mapstructure:"wait_image_ready_timeout"` } -var imageNamePattern = regexp.MustCompile(`^[A-Za-z0-9\p{Han}-_\[\]:,.]{1,63}$`) +var ImageNamePattern = regexp.MustCompile(`^[A-Za-z0-9\p{Han}-_\[\]:,.]{1,63}$`) func (c *ImageConfig) Prepare(ctx *interpolate.Context) []error { var errs []error imageName := c.ImageName if imageName == "" { errs = append(errs, fmt.Errorf("%q must be set", "image_name")) - } else if !imageNamePattern.MatchString(imageName) { + } else if !ImageNamePattern.MatchString(imageName) { errs = append(errs, fmt.Errorf("expected %q to be 1-63 characters and only support chinese, english, numbers, '-_,.:[]', got %q", "image_name", imageName)) } @@ -43,6 +44,10 @@ func (c *ImageConfig) Prepare(ctx *interpolate.Context) []error { } } + if c.WaitImageReadyTimeout <= 0 { + c.WaitImageReadyTimeout = DefaultCreateImageTimeOut + } + if len(errs) > 0 { return errs } @@ -61,7 +66,7 @@ func (imageDestination *ImageDestination) validate() []error { errs = append(errs, fmt.Errorf("%q must be set", "image_copy_project")) } - if imageDestination.Name != "" && !imageNamePattern.MatchString(imageDestination.Name) { + if imageDestination.Name != "" && !ImageNamePattern.MatchString(imageDestination.Name) { errs = append(errs, fmt.Errorf("expected %q to be 1-63 characters and only support chinese, english, numbers, '-_,.:[]', got %q", "image_copy_name", imageDestination.Name)) } diff --git a/builder/ucloud/uhost/builder.go b/builder/ucloud/uhost/builder.go index 64a80500e..a9903bf4a 100644 --- a/builder/ucloud/uhost/builder.go +++ b/builder/ucloud/uhost/builder.go @@ -120,9 +120,10 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack &stepStopInstance{}, &stepCreateImage{}, &stepCopyUCloudImage{ - ImageDestinations: b.config.ImageDestinations, - RegionId: b.config.Region, - ProjectId: b.config.ProjectId, + ImageDestinations: b.config.ImageDestinations, + RegionId: b.config.Region, + ProjectId: b.config.ProjectId, + WaitImageReadyTimeout: b.config.WaitImageReadyTimeout, }, } diff --git a/builder/ucloud/uhost/step_copy_image.go b/builder/ucloud/uhost/step_copy_image.go index a24c520f5..1b8dba0bf 100644 --- a/builder/ucloud/uhost/step_copy_image.go +++ b/builder/ucloud/uhost/step_copy_image.go @@ -14,9 +14,10 @@ import ( ) type stepCopyUCloudImage struct { - ImageDestinations []ucloudcommon.ImageDestination - RegionId string - ProjectId string + ImageDestinations []ucloudcommon.ImageDestination + RegionId string + ProjectId string + WaitImageReadyTimeout int } func (s *stepCopyUCloudImage) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { @@ -63,9 +64,11 @@ func (s *stepCopyUCloudImage) Run(ctx context.Context, state multistep.StateBag) ui.Message("Waiting for the copied images to become available...") err := retry.Config{ - Tries: 200, - ShouldRetry: func(err error) bool { return ucloudcommon.IsNotCompleteError(err) }, - RetryDelay: (&retry.Backoff{InitialBackoff: 2 * time.Second, MaxBackoff: 12 * time.Second, Multiplier: 2}).Linear, + StartTimeout: time.Duration(s.WaitImageReadyTimeout) * time.Second, + ShouldRetry: func(err error) bool { + return ucloudcommon.IsNotCompleteError(err) + }, + RetryDelay: (&retry.Backoff{InitialBackoff: 2 * time.Second, MaxBackoff: 12 * time.Second, Multiplier: 2}).Linear, }.Run(ctx, func(ctx context.Context) error { for _, v := range expectedImages.GetAll() { imageSet, err := client.DescribeImageByInfo(v.ProjectId, v.Region, v.ImageId) diff --git a/builder/ucloud/uhost/step_create_image.go b/builder/ucloud/uhost/step_create_image.go index 80d13c978..235cdb7ff 100644 --- a/builder/ucloud/uhost/step_create_image.go +++ b/builder/ucloud/uhost/step_create_image.go @@ -38,7 +38,7 @@ func (s *stepCreateImage) Run(ctx context.Context, state multistep.StateBag) mul ui.Message(fmt.Sprintf("Waiting for the created image %q to become available...", resp.ImageId)) err = retry.Config{ - Tries: 200, + StartTimeout: time.Duration(config.WaitImageReadyTimeout) * time.Second, ShouldRetry: func(err error) bool { return ucloudcommon.IsExpectedStateError(err) }, diff --git a/builder/ucloud/uhost/step_create_instance.go b/builder/ucloud/uhost/step_create_instance.go index 331a6931e..fd35aaa64 100644 --- a/builder/ucloud/uhost/step_create_instance.go +++ b/builder/ucloud/uhost/step_create_instance.go @@ -243,7 +243,6 @@ func (s *stepCreateInstance) buildCreateInstanceRequest(state multistep.StateBag req.MinimalCpuPlatform = ucloud.String("Intel/Auto") if t.HostType == "o" { req.MachineType = ucloud.String("O") - req.MinimalCpuPlatform = ucloud.String("Intel/Cascadelake") } if v, ok := state.GetOk("security_group_id"); ok { diff --git a/command/plugin.go b/command/plugin.go index 7b24d76f1..619a162bf 100644 --- a/command/plugin.go +++ b/command/plugin.go @@ -175,7 +175,6 @@ var Provisioners = map[string]packer.Provisioner{ var PostProcessors = map[string]packer.PostProcessor{ "alicloud-import": new(alicloudimportpostprocessor.PostProcessor), - "ucloud-import": new(ucloudimportpostprocessor.PostProcessor), "amazon-import": new(amazonimportpostprocessor.PostProcessor), "artifice": new(artificepostprocessor.PostProcessor), "checksum": new(checksumpostprocessor.PostProcessor), @@ -190,6 +189,7 @@ var PostProcessors = map[string]packer.PostProcessor{ "googlecompute-import": new(googlecomputeimportpostprocessor.PostProcessor), "manifest": new(manifestpostprocessor.PostProcessor), "shell-local": new(shelllocalpostprocessor.PostProcessor), + "ucloud-import": new(ucloudimportpostprocessor.PostProcessor), "vagrant": new(vagrantpostprocessor.PostProcessor), "vagrant-cloud": new(vagrantcloudpostprocessor.PostProcessor), "vsphere": new(vspherepostprocessor.PostProcessor), diff --git a/examples/ucloud/local/http/centos-6.10/ks.cfg b/examples/ucloud/local/http/centos-6.10/ks.cfg new file mode 100644 index 000000000..31f57af6d --- /dev/null +++ b/examples/ucloud/local/http/centos-6.10/ks.cfg @@ -0,0 +1,25 @@ +install +cdrom +lang en_US.UTF-8 +keyboard us +network --bootproto=dhcp +rootpw ucloud_2019 +firewall --disabled +selinux --permissive +timezone UTC +unsupported_hardware +bootloader --location=mbr +text +skipx +zerombr +clearpart --all +autopart +auth --enableshadow --passalgo=sha512 +firstboot --disabled +reboot + +%packages --nobase --ignoremissing +sudo +gcc +make +%end \ No newline at end of file diff --git a/examples/ucloud/local/http/centos-6.8/ks.cfg b/examples/ucloud/local/http/centos-6.8/ks.cfg deleted file mode 100644 index 44f8403fb..000000000 --- a/examples/ucloud/local/http/centos-6.8/ks.cfg +++ /dev/null @@ -1,42 +0,0 @@ -install -cdrom -lang en_US.UTF-8 -keyboard us -network --bootproto=dhcp -rootpw vagrant -firewall --disabled -selinux --permissive -timezone UTC -unsupported_hardware -bootloader --location=mbr -text -skipx -zerombr -clearpart --all -autopart -auth --enableshadow --passalgo=sha512 -firstboot --disabled -reboot -user --name=vagrant --password=vagrant - -%packages --nobase --ignoremissing -# vagrant needs this to copy initial files via scp -openssh-clients -sudo -kernel-headers -kernel-devel -gcc -make -perl -wget -nfs-utils --fprintd-pam --intltool -%end - -%post -# Force to set SELinux to a permissive mode -sed -i -e 's/\(^SELINUX=\).*$/\1permissive/' /etc/selinux/config -# sudo -echo "%vagrant ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/vagrant -%end \ No newline at end of file diff --git a/examples/ucloud/local/local.json b/examples/ucloud/local/local.json index 77fee9390..df41acb50 100644 --- a/examples/ucloud/local/local.json +++ b/examples/ucloud/local/local.json @@ -1,22 +1,16 @@ {"variables": { - "build_timestamp": "{{isotime \"20060102150405\"}}", - "cpus": "1", + "ucloud_public_key": "{{env `UCLOUD_PUBLIC_KEY`}}", + "ucloud_private_key": "{{env `UCLOUD_PRIVATE_KEY`}}", + "ucloud_project_id": "{{env `UCLOUD_PROJECT_ID`}}", "disk_size": "4096", - "git_revision": "__unknown_git_revision__", "headless": "", - "http_proxy": "{{env `http_proxy`}}", - "https_proxy": "{{env `https_proxy`}}", - "iso_checksum": "7002b56184180591a8fa08c2fe0c7338", + "iso_checksum": "0da4a1206e7642906e33c0f155d2f835", "iso_checksum_type": "md5", - "iso_name": "CentOS-7-x86_64-Minimal-1908.iso", - "ks_path": "centos-6.8/ks.cfg", - "memory": "512", - "metadata": "floppy/dummy_metadata.json", - "mirror": "https://mirrors.sjtug.sjtu.edu.cn/centos", - "mirror_directory": "7.7.1908/isos/x86_64", - "no_proxy": "{{env `no_proxy`}}", - "template": "centos-7.7-x86_64", - "version": "2.1.TIMESTAMP" + "iso_name": "CentOS-6.10-x86_64-minimal.iso", + "ks_path": "centos-6.10/ks.cfg", + "mirror": "http://mirrors.ustc.edu.cn/centos", + "mirror_directory": "6.10/isos/x86_64", + "template": "centos-6.10-x86_64" }, "builders":[ { @@ -33,7 +27,7 @@ "iso_url": "{{user `mirror`}}/{{user `mirror_directory`}}/{{user `iso_name`}}", "output_directory": "packer-{{user `template`}}-qemu", "shutdown_command": "echo 'vagrant'|sudo -S /sbin/halt -h -p", - "ssh_password": "vagrant", + "ssh_password": "ucloud_2019", "ssh_port": 22, "ssh_username": "root", "ssh_timeout": "10000s", @@ -50,12 +44,15 @@ "post-processors":[ { "type":"ucloud-import", - "ufile_bucket_name": "packer", - "image_name": "packer_import", + "public_key": "{{user `ucloud_public_key`}}", + "private_key": "{{user `ucloud_private_key`}}", + "project_id": "{{user `ucloud_project_id`}}", + "region":"cn-bj2", + "ufile_bucket_name": "packer-test", + "image_name": "packer_import_test", "image_os_type": "CentOS", - "image_os_name": "CentOS 6.8 64位", - "format": "raw", - "region":"cn-bj2" + "image_os_name": "CentOS 6.10 64位", + "format": "raw" } ] } \ No newline at end of file diff --git a/examples/ucloud/local/test.json b/examples/ucloud/local/test.json deleted file mode 100644 index de5684d37..000000000 --- a/examples/ucloud/local/test.json +++ /dev/null @@ -1,62 +0,0 @@ -{"variables": { - "build_timestamp": "{{isotime \"20060102150405\"}}", - "cpus": "1", - "disk_size": "4096", - "git_revision": "__unknown_git_revision__", - "headless": "", - "http_proxy": "{{env `http_proxy`}}", - "https_proxy": "{{env `https_proxy`}}", - "iso_checksum": "7002b56184180591a8fa08c2fe0c7338", - "iso_checksum_type": "md5", - "iso_name": "CentOS-7-x86_64-Minimal-1908.iso", - "ks_path": "centos-6.8/ks.cfg", - "memory": "512", - "metadata": "floppy/dummy_metadata.json", - "mirror": "http://mirrors.aliyun.com/centos", - "mirror_directory": "7/isos/x86_64", - "no_proxy": "{{env `no_proxy`}}", - "template": "centos-7-x86_64", - "version": "2.1.TIMESTAMP" -}, - "builders":[ - { - "type": "qemu", - "boot_command": [ - " text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/{{user `ks_path`}}" - ], - "boot_wait": "10s", - "disk_size": "{{user `disk_size`}}", - "headless": "{{ user `headless` }}", - "http_directory": "http", - "iso_checksum": "{{user `iso_checksum`}}", - "iso_checksum_type": "{{user `iso_checksum_type`}}", - "iso_url": "{{user `mirror`}}/{{user `mirror_directory`}}/{{user `iso_name`}}", - "output_directory": "packer-{{user `template`}}-qemu", - "shutdown_command": "echo 'vagrant'|sudo -S /sbin/halt -h -p", - "ssh_password": "vagrant", - "ssh_port": 22, - "ssh_username": "root", - "ssh_timeout": "10000s", - "vm_name": "{{ user `template` }}.raw", - "net_device": "virtio-net", - "disk_interface": "virtio", - "format": "raw", - "use_default_display": "false", - "qemuargs": [ - ["-display", "cocoa"] - ] - } - ], - "post-processors":[ - { - "type":"ucloud-import", - "ufile_bucket_name": "packer", - "image_name": "packer_import", - "image_os_type": "CentOS", - "image_os_name": "CentOS 7.0 64位", - "format": "raw", - "region":"cn-bj2", - "project_id": "org-xp2ucn" - } - ] -} \ No newline at end of file diff --git a/post-processor/ucloud-import/post-processor.go b/post-processor/ucloud-import/post-processor.go index 20a0160ca..c65d0dfb0 100644 --- a/post-processor/ucloud-import/post-processor.go +++ b/post-processor/ucloud-import/post-processor.go @@ -24,6 +24,13 @@ const ( VHDFileFormat = "vhd" VMDKFileFormat = "vmdk" QCOW2FileFormat = "qcow2" + + CentOSOsType = "CentOS" + UbuntuOsType = "Ubuntu" + WindowsOsType = "Windows" + RedHatOsType = "RedHat" + DebianOsType = "Debian" + OtherOsType = "Other" ) var regionForFileMap = ucloudcommon.NewStringConverter(map[string]string{ @@ -43,14 +50,15 @@ type Config struct { ucloudcommon.AccessConfig `mapstructure:",squash"` // Variables specific to this post processor - UFileBucket string `mapstructure:"ufile_bucket_name"` - UFileKey string `mapstructure:"ufile_key_name"` - SkipClean bool `mapstructure:"skip_clean"` - ImageName string `mapstructure:"image_name"` - ImageDescription string `mapstructure:"image_description"` - OSType string `mapstructure:"image_os_type"` - OSName string `mapstructure:"image_os_name"` - Format string `mapstructure:"format"` + UFileBucket string `mapstructure:"ufile_bucket_name"` + UFileKey string `mapstructure:"ufile_key_name"` + SkipClean bool `mapstructure:"skip_clean"` + ImageName string `mapstructure:"image_name"` + ImageDescription string `mapstructure:"image_description"` + OSType string `mapstructure:"image_os_type"` + OSName string `mapstructure:"image_os_name"` + Format string `mapstructure:"format"` + WaitImageReadyTimeout int `mapstructure:"wait_image_ready_timeout"` ctx interpolate.Context } @@ -76,10 +84,13 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { // Set defaults if p.config.UFileKey == "" { - //fmt.Sprintf("packer_%s", uuid.TimeOrderedUUID()[:8]) p.config.UFileKey = "packer-import-{{timestamp}}." + p.config.Format } + if p.config.WaitImageReadyTimeout <= 0 { + p.config.WaitImageReadyTimeout = ucloudcommon.DefaultCreateImageTimeOut + } + errs := new(packer.MultiError) // Check and render ufile_key_name @@ -95,8 +106,9 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { templates := map[string]*string{ "ufile_bucket_name": &p.config.UFileBucket, "image_name": &p.config.ImageName, - "image_os_type": &p.config.OSName, - "image_os_name": &p.config.OSType, + "image_os_type": &p.config.OSType, + "image_os_name": &p.config.OSName, + "format": &p.config.Format, } // Check out required params are defined for key, ptr := range templates { @@ -106,11 +118,23 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } } + imageName := p.config.ImageName + if !ucloudcommon.ImageNamePattern.MatchString(imageName) { + errs = packer.MultiErrorAppend(errs, fmt.Errorf("expected %q to be 1-63 characters and only support chinese, english, numbers, '-_,.:[]', got %q", "image_name", imageName)) + } + + switch p.config.OSType { + case CentOSOsType, UbuntuOsType, WindowsOsType, RedHatOsType, DebianOsType, OtherOsType: + default: + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("expected %q only be one of 'CentOS', 'Ubuntu', 'Windows', 'RedHat', 'Debian' or 'Other', got %q", "image_os_type", p.config.OSType)) + } + switch p.config.Format { case VHDFileFormat, RAWFileFormat, VMDKFileFormat, QCOW2FileFormat: default: errs = packer.MultiErrorAppend( - errs, fmt.Errorf("invalid format '%s'. Only 'raw', 'vhd', 'vmdk', or 'qcow2' are allowed", p.config.Format)) + errs, fmt.Errorf("expected %q only be one of 'raw', 'vhd', 'vmdk', or 'qcow2', got %q", "format", p.config.Format)) } // Anything which flagged return back up the stack @@ -128,7 +152,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact client, err := p.config.Client() if err != nil { - return nil, false, false, err + return nil, false, false, fmt.Errorf("Failed to connect ucloud client %s", err) } uhostconn := client.UHostConn ufileconn := client.UFileConn @@ -139,9 +163,9 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact return nil, false, false, fmt.Errorf("Error rendering ufile_key_name template: %s", err) } - log.Printf("Rendered ufile_key_name as %s", p.config.UFileKey) + ui.Message(fmt.Sprintf("Rendered ufile_key_name as %s", p.config.UFileKey)) - log.Println("Looking for image in artifact") + ui.Message("Looking for image in artifact") // Locate the files output from the builder var source string for _, path := range artifact.Files() { @@ -156,8 +180,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact return nil, false, false, fmt.Errorf("No %s image file found in artifact from builder", p.config.Format) } - region := regionForFileMap.Convert(p.config.Region) - projectId := p.config.ProjectId + convertedRegion := regionForFileMap.Convert(p.config.Region) keyName := p.config.UFileKey bucketName := p.config.UFileBucket @@ -165,43 +188,42 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact PublicKey: p.config.PublicKey, PrivateKey: p.config.PrivateKey, BucketName: bucketName, - FileHost: fmt.Sprintf(region + ".ufileos.com"), + FileHost: fmt.Sprintf(convertedRegion + ".ufileos.com"), BucketHost: "api.ucloud.cn", } + + // query or create bucket + if err := queryOrCreateBucket(ufileconn, config); err != nil { + return nil, false, false, fmt.Errorf("Failed to query or create bucket, %s", err) + } + + bucketUrl := fmt.Sprintf("http://" + bucketName + "." + convertedRegion + ".ufileos.com") + + ui.Say(fmt.Sprintf("Waiting for uploading image file %s to UFile: %s/%s...", source, bucketUrl, p.config.UFileKey)) + + // upload file to bucket + ufileUrl, err := uploadFile(ufileconn, config, keyName, source) if err != nil { - return nil, false, false, fmt.Errorf("Load config error %s", err) + return nil, false, false, fmt.Errorf("Failed to Upload image file, %s", err) } - if err := queryOrCreateBucket(ufileconn, config, region, projectId); err != nil { - return nil, false, false, fmt.Errorf("Query or create bucket error %s", err) - } + ui.Say(fmt.Sprintf("Image file %s has been uploaded to UFile %s", source, ufileUrl)) - bucketUrl := fmt.Sprintf("http://" + bucketName + "." + region + ".ufileos.com") - ui.Say(fmt.Sprintf("Waiting for uploading file %s to %s/%s...", source, bucketUrl, p.config.UFileKey)) - - privateUrl, err := uploadFile(config, keyName, source, projectId) - if err != nil { - return nil, false, false, fmt.Errorf("Upload file error %s", err) - } - - ui.Say(fmt.Sprintf("Image file %s has been uploaded to UFile %s", source, privateUrl)) - - importImageRequest := p.buildImportImageRequest(uhostconn, privateUrl) + importImageRequest := p.buildImportImageRequest(uhostconn, ufileUrl) importImageResponse, err := uhostconn.ImportCustomImage(importImageRequest) if err != nil { - return nil, false, false, fmt.Errorf("Failed to import from %s/%s: %s", bucketUrl, p.config.UFileKey, err) + return nil, false, false, fmt.Errorf("Failed to import image from %s/%s, %s", bucketUrl, p.config.UFileKey, err) } - imageId := importImageResponse.ImageId - ui.Say(fmt.Sprintf("Waiting for importing %s/%s to ucloud...", bucketUrl, p.config.UFileKey)) + imageId := importImageResponse.ImageId err = retry.Config{ - Tries: 30, + StartTimeout: time.Duration(p.config.WaitImageReadyTimeout) * time.Second, ShouldRetry: func(err error) bool { return ucloudcommon.IsExpectedStateError(err) }, - RetryDelay: (&retry.Backoff{InitialBackoff: 2 * time.Second, MaxBackoff: 6 * time.Second, Multiplier: 2}).Linear, + RetryDelay: (&retry.Backoff{InitialBackoff: 2 * time.Second, MaxBackoff: 12 * time.Second, Multiplier: 2}).Linear, }.Run(ctx, func(ctx context.Context) error { image, err := client.DescribeImageById(imageId) if err != nil { @@ -209,7 +231,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact } if image.State == ucloudcommon.ImageStateUnavailable { - return fmt.Errorf("Unavailable importing image %s", imageId) + return fmt.Errorf("Unavailable importing image %q", imageId) } if image.State != ucloudcommon.ImageStateAvailable { @@ -225,7 +247,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact } // Add the reported UCloud image ID to the artifact list - ui.Say(fmt.Sprintf("Importing created ucloud image ID %s in region %s Finished.", imageId, p.config.Region)) + ui.Say(fmt.Sprintf("Importing created ucloud image %q in region %q Complete.", imageId, p.config.Region)) images := []ucloudcommon.ImageInfo{ { ImageId: imageId, @@ -241,9 +263,9 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact } if !p.config.SkipClean { - ui.Message(fmt.Sprintf("Deleting import source %s/%s/%s", bucketUrl, p.config.UFileBucket, p.config.UFileKey)) + ui.Message(fmt.Sprintf("Deleting import source UFile: %s/%s", p.config.UFileBucket, p.config.UFileKey)) if err = deleteFile(config, p.config.UFileKey); err != nil { - return nil, false, false, fmt.Errorf("Failed to delete %s/%s/%s: %s", bucketUrl, p.config.UFileBucket, p.config.UFileKey, err) + return nil, false, false, fmt.Errorf("Failed to delete UFile: %s/%s, %s", p.config.UFileBucket, p.config.UFileKey, err) } } @@ -262,7 +284,7 @@ func (p *PostProcessor) buildImportImageRequest(conn *uhost.UHostClient, private return req } -func queryOrCreateBucket(conn *ufile.UFileClient, config *ufsdk.Config, region, projectId string) error { +func queryOrCreateBucket(conn *ufile.UFileClient, config *ufsdk.Config) error { var limit = 100 var offset int var bucketList []ufile.UFileBucketSet @@ -272,7 +294,7 @@ func queryOrCreateBucket(conn *ufile.UFileClient, config *ufsdk.Config, region, req.Offset = ucloud.Int(offset) resp, err := conn.DescribeBucket(req) if err != nil { - return fmt.Errorf("error on reading bucket list, %s", err) + return fmt.Errorf("error on reading bucket list when create bucket, %s", err) } if resp == nil || len(resp.DataSet) < 1 { @@ -307,32 +329,30 @@ func queryOrCreateBucket(conn *ufile.UFileClient, config *ufsdk.Config, region, return nil } -func uploadFile(config *ufsdk.Config, keyName, filePath, projectId string) (string, error) { +func uploadFile(conn *ufile.UFileClient, config *ufsdk.Config, keyName, source string) (string, error) { reqFile, err := ufsdk.NewFileRequest(config, nil) if err != nil { - return "", fmt.Errorf("NewFileErr:%s", err) + return "", fmt.Errorf("error on building upload file request, %s", err) } - err = reqFile.AsyncMPut(filePath, keyName, "") + // upload file in segments + err = reqFile.AsyncMPut(source, keyName, "") if err != nil { - return "", fmt.Errorf("AsyncMPutErr:%s, Response:%s", err, reqFile.DumpResponse(true)) + return "", fmt.Errorf("error on upload file, %s, details: %s", err, reqFile.DumpResponse(true)) } - reqBucket, err := ufsdk.NewBucketRequest(config, nil) + reqBucket := conn.NewDescribeBucketRequest() + reqBucket.BucketName = ucloud.String(config.BucketName) + resp, err := conn.DescribeBucket(reqBucket) if err != nil { - return "", err + return "", fmt.Errorf("error on reading bucket list when upload file, %s", err) } - bucketList, err := reqBucket.DescribeBucket(config.BucketName, 0, 1, projectId) - if err != nil { - return "", nil - } - - if bucketList.DataSet[0].Type == "private" { + if resp.DataSet[0].Type == "private" { return reqFile.GetPrivateURL(keyName, 24*60*60), nil } - return reqBucket.GetPublicURL(keyName), nil + return reqFile.GetPublicURL(keyName), nil } func deleteFile(config *ufsdk.Config, keyName string) error { diff --git a/website/source/docs/builders/ucloud-uhost.html.md b/website/source/docs/builders/ucloud-uhost.html.md index ff165ef1e..c955befe4 100644 --- a/website/source/docs/builders/ucloud-uhost.html.md +++ b/website/source/docs/builders/ucloud-uhost.html.md @@ -83,6 +83,8 @@ builder. - `name` (string) - The copied image name. If not defined, builder will use `image_name` as default name. - `description` (number) - The copied image description. + +- `wait_image_ready_timeout` (int)Timeout of creating image or copying image. The default timeout is 3600 seconds if this option is not set or is set to 0. ## Examples diff --git a/website/source/docs/post-processors/ucloud-import.html.md b/website/source/docs/post-processors/ucloud-import.html.md new file mode 100644 index 000000000..5671cf561 --- /dev/null +++ b/website/source/docs/post-processors/ucloud-import.html.md @@ -0,0 +1,75 @@ +--- +description: | + The Packer UCloud Import post-processor takes the RAW, VHD, VMDK, or qcow2 artifact from various builders and imports it to UCloud customized image list for UHost Instance. +layout: docs +page_title: 'UCloud Import Post-Processors' +sidebar_current: 'docs-post-processors-ucloud-import' +--- + +# UCloud Import Post-Processor + +Type: `ucloud-import` + + The Packer UCloud Import post-processor takes the RAW, VHD, VMDK, or qcow2 artifact from various builders and imports it to UCloud customized image list for UHost Instance. + +## How Does it Work? + +The import process operates by making a temporary copy of the RAW, VHD, VMDK, or qcow2 to an UFile bucket, and calling an import task in UHost on the RAW, VHD, VMDK, or qcow2 file. Once completed, an UCloud UHost Image is returned. The temporary RAW, VHD, VMDK, or qcow2 copy in UFile can be discarded after the import is complete. + +## Configuration + +There are some configuration options available for the post-processor. There +are two categories: required and optional parameters. + +### Required: + +- `public_key` - (string) This is the UCloud public key. It must be provided, but it can also be sourced from the `UCLOUD_PUBLIC_KEY` environment variable. + +- `private_key` - (string) This is the UCloud private key. It must be provided, but it can also be sourced from the `UCLOUD_PRIVATE_KEY` environment variable. + +- `project_id` - (string) This is the UCloud project id. It must be provided, but it can also be sourced from the `UCLOUD_PROJECT_ID` environment variables. + +- `region` - (string) This is the UCloud region. It must be provided, but it can also be sourced from the `UCLOUD_REGION` environment variables. + +- `image_name` - (string) The name of the user-defined image, which contains 1-63 characters and only support Chinese, English, numbers, '-_,.:[]'. + +- `ufile_bucket_name` (string) - The name of the ufile bucket where the RAW, VHD, VMDK, or qcow2 file will be copied to for import. If the Bucket isn't exist, post-process will create it for you. + +- `image_os_type` (string) - Type of the OS. Possible values are: `CentOS`, `Ubuntu`, `Windows`, `RedHat`, `Debian`, `Other`. + +- `image_os_name` (string) - The name of OS. Such as: `CentOS 7.2 64位`, set `Other` When `image_os_type` is `Other`. + +- `format` (string) - The format of the import image , Possible values are: `RAW`, `VHD`, `VMDK`, or `qcow2`. + +### Optional: + + +- `ufile_key_name` (string) - The name of the object key in `ufile_bucket_name` where the RAW, VHD, VMDK, or qcow2 file will be copied to for import. + +- `skip_clean` (boolean) - Whether we should skip removing the RAW or VHD file uploaded to UFile after the import process has completed. Possible values are: `true` as leave it in the UFile bucket, `false` as clean it out.(Default: `false`). + +- `image_description` (string) - The description of the image. + +- `wait_image_ready_timeout`(number) - Timeout of importing image. The default timeout is 3600 seconds if this option is not set or is set + to 0. + +## Basic Example + +Here is a basic example. This assumes that the builder has produced a RAW artifact for us to work with. This will take the RAW image generated by a builder and upload it to UFile. Once uploaded, the import process will start, creating an UCloud UHost image to the region `cn-bj2`. + +``` json +"post-processors":[ + { + "type":"ucloud-import", + "public_key": "{{user `ucloud_public_key`}}", + "private_key": "{{user `ucloud_private_key`}}", + "project_id": "{{user `ucloud_project_id`}}", + "region":"cn-bj2", + "ufile_bucket_name": "packer-import", + "image_name": "packer_import", + "image_os_type": "CentOS", + "image_os_name": "CentOS 6.10 64位", + "format": "raw" + } + ] +```