384 lines
11 KiB
Go
384 lines
11 KiB
Go
//go:generate mapstructure-to-hcl2 -type Config
|
|
|
|
package digitaloceanimport
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/oauth2"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
|
"github.com/aws/aws-sdk-go/aws/session"
|
|
"github.com/aws/aws-sdk-go/service/s3"
|
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
|
"github.com/digitalocean/godo"
|
|
|
|
"github.com/hashicorp/hcl/v2/hcldec"
|
|
"github.com/hashicorp/packer/builder/digitalocean"
|
|
"github.com/hashicorp/packer/common"
|
|
"github.com/hashicorp/packer/helper/config"
|
|
"github.com/hashicorp/packer/packer"
|
|
"github.com/hashicorp/packer/template/interpolate"
|
|
)
|
|
|
|
const BuilderId = "packer.post-processor.digitalocean-import"
|
|
|
|
type Config struct {
|
|
common.PackerConfig `mapstructure:",squash"`
|
|
|
|
APIToken string `mapstructure:"api_token"`
|
|
SpacesKey string `mapstructure:"spaces_key"`
|
|
SpacesSecret string `mapstructure:"spaces_secret"`
|
|
|
|
SpacesRegion string `mapstructure:"spaces_region"`
|
|
SpaceName string `mapstructure:"space_name"`
|
|
ObjectName string `mapstructure:"space_object_name"`
|
|
SkipClean bool `mapstructure:"skip_clean"`
|
|
Tags []string `mapstructure:"image_tags"`
|
|
Name string `mapstructure:"image_name"`
|
|
Description string `mapstructure:"image_description"`
|
|
Distribution string `mapstructure:"image_distribution"`
|
|
ImageRegions []string `mapstructure:"image_regions"`
|
|
|
|
Timeout time.Duration `mapstructure:"timeout"`
|
|
|
|
ctx interpolate.Context
|
|
}
|
|
|
|
type PostProcessor struct {
|
|
config Config
|
|
}
|
|
|
|
type apiTokenSource struct {
|
|
AccessToken string
|
|
}
|
|
|
|
type logger struct {
|
|
logger *log.Logger
|
|
}
|
|
|
|
func (t *apiTokenSource) Token() (*oauth2.Token, error) {
|
|
return &oauth2.Token{
|
|
AccessToken: t.AccessToken,
|
|
}, nil
|
|
}
|
|
|
|
func (l logger) Log(args ...interface{}) {
|
|
l.logger.Println(args...)
|
|
}
|
|
|
|
func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() }
|
|
|
|
func (p *PostProcessor) Configure(raws ...interface{}) error {
|
|
err := config.Decode(&p.config, &config.DecodeOpts{
|
|
Interpolate: true,
|
|
InterpolateContext: &p.config.ctx,
|
|
InterpolateFilter: &interpolate.RenderFilter{
|
|
Exclude: []string{"space_object_name"},
|
|
},
|
|
}, raws...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if p.config.SpacesKey == "" {
|
|
p.config.SpacesKey = os.Getenv("DIGITALOCEAN_SPACES_ACCESS_KEY")
|
|
}
|
|
|
|
if p.config.SpacesSecret == "" {
|
|
p.config.SpacesSecret = os.Getenv("DIGITALOCEAN_SPACES_SECRET_KEY")
|
|
}
|
|
|
|
if p.config.APIToken == "" {
|
|
p.config.APIToken = os.Getenv("DIGITALOCEAN_API_TOKEN")
|
|
}
|
|
|
|
if p.config.ObjectName == "" {
|
|
p.config.ObjectName = "packer-import-{{timestamp}}"
|
|
}
|
|
|
|
if p.config.Distribution == "" {
|
|
p.config.Distribution = "Unkown"
|
|
}
|
|
|
|
if p.config.Timeout == 0 {
|
|
p.config.Timeout = 20 * time.Minute
|
|
}
|
|
|
|
errs := new(packer.MultiError)
|
|
|
|
if err = interpolate.Validate(p.config.ObjectName, &p.config.ctx); err != nil {
|
|
errs = packer.MultiErrorAppend(
|
|
errs, fmt.Errorf("Error parsing space_object_name template: %s", err))
|
|
}
|
|
|
|
requiredArgs := map[string]*string{
|
|
"api_token": &p.config.APIToken,
|
|
"spaces_key": &p.config.SpacesKey,
|
|
"spaces_secret": &p.config.SpacesSecret,
|
|
"spaces_region": &p.config.SpacesRegion,
|
|
"space_name": &p.config.SpaceName,
|
|
"image_name": &p.config.Name,
|
|
}
|
|
for key, ptr := range requiredArgs {
|
|
if *ptr == "" {
|
|
errs = packer.MultiErrorAppend(
|
|
errs, fmt.Errorf("%s must be set", key))
|
|
}
|
|
}
|
|
|
|
if len(p.config.ImageRegions) == 0 {
|
|
errs = packer.MultiErrorAppend(
|
|
errs, fmt.Errorf("image_regions must be set"))
|
|
}
|
|
|
|
if len(errs.Errors) > 0 {
|
|
return errs
|
|
}
|
|
|
|
packer.LogSecretFilter.Set(p.config.SpacesKey, p.config.SpacesSecret, p.config.APIToken)
|
|
log.Println(p.config)
|
|
return nil
|
|
}
|
|
|
|
func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) {
|
|
var err error
|
|
|
|
generatedData := artifact.State("generated_data")
|
|
if generatedData == nil {
|
|
// Make sure it's not a nil map so we can assign to it later.
|
|
generatedData = make(map[string]interface{})
|
|
}
|
|
p.config.ctx.Data = generatedData
|
|
|
|
p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx)
|
|
if err != nil {
|
|
return nil, false, false, fmt.Errorf("Error rendering space_object_name template: %s", err)
|
|
}
|
|
log.Printf("Rendered space_object_name as %s", p.config.ObjectName)
|
|
|
|
source := ""
|
|
artifacts := artifact.Files()
|
|
log.Println("Looking for image in artifact")
|
|
if len(artifacts) > 1 {
|
|
validSuffix := []string{"raw", "img", "qcow2", "vhdx", "vdi", "vmdk", "tar.bz2", "tar.xz", "tar.gz"}
|
|
for _, path := range artifact.Files() {
|
|
for _, suffix := range validSuffix {
|
|
if strings.HasSuffix(path, suffix) {
|
|
source = path
|
|
break
|
|
}
|
|
}
|
|
if source != "" {
|
|
break
|
|
}
|
|
}
|
|
} else {
|
|
source = artifact.Files()[0]
|
|
}
|
|
|
|
if source == "" {
|
|
return nil, false, false, fmt.Errorf("Image file not found")
|
|
}
|
|
|
|
spacesCreds := credentials.NewStaticCredentials(p.config.SpacesKey, p.config.SpacesSecret, "")
|
|
spacesEndpoint := fmt.Sprintf("https://%s.digitaloceanspaces.com", p.config.SpacesRegion)
|
|
spacesConfig := &aws.Config{
|
|
Credentials: spacesCreds,
|
|
Endpoint: aws.String(spacesEndpoint),
|
|
Region: aws.String(p.config.SpacesRegion),
|
|
LogLevel: aws.LogLevel(aws.LogDebugWithSigning),
|
|
Logger: &logger{
|
|
logger: log.New(os.Stderr, "", log.LstdFlags),
|
|
},
|
|
}
|
|
sess := session.New(spacesConfig)
|
|
|
|
ui.Message(fmt.Sprintf("Uploading %s to spaces://%s/%s", source, p.config.SpaceName, p.config.ObjectName))
|
|
err = uploadImageToSpaces(source, p, sess)
|
|
if err != nil {
|
|
return nil, false, false, err
|
|
}
|
|
ui.Message(fmt.Sprintf("Completed upload of %s to spaces://%s/%s", source, p.config.SpaceName, p.config.ObjectName))
|
|
|
|
client := godo.NewClient(oauth2.NewClient(oauth2.NoContext, &apiTokenSource{
|
|
AccessToken: p.config.APIToken,
|
|
}))
|
|
|
|
ui.Message(fmt.Sprintf("Started import of spaces://%s/%s", p.config.SpaceName, p.config.ObjectName))
|
|
image, err := importImageFromSpaces(p, client)
|
|
if err != nil {
|
|
return nil, false, false, err
|
|
}
|
|
|
|
ui.Message(fmt.Sprintf("Waiting for import of image %s to complete (may take a while)", p.config.Name))
|
|
err = waitUntilImageAvailable(client, image.ID, p.config.Timeout)
|
|
if err != nil {
|
|
return nil, false, false, fmt.Errorf("Import of image %s failed with error: %s", p.config.Name, err)
|
|
}
|
|
ui.Message(fmt.Sprintf("Import of image %s complete", p.config.Name))
|
|
|
|
if len(p.config.ImageRegions) > 1 {
|
|
// Remove the first region from the slice as the image is already there.
|
|
regions := p.config.ImageRegions
|
|
regions[0] = regions[len(regions)-1]
|
|
regions[len(regions)-1] = ""
|
|
regions = regions[:len(regions)-1]
|
|
|
|
ui.Message(fmt.Sprintf("Distributing image %s to additional regions: %v", p.config.Name, regions))
|
|
err = distributeImageToRegions(client, image.ID, regions, p.config.Timeout)
|
|
if err != nil {
|
|
return nil, false, false, err
|
|
}
|
|
}
|
|
|
|
log.Printf("Adding created image ID %v to output artifacts", image.ID)
|
|
artifact = &digitalocean.Artifact{
|
|
SnapshotName: image.Name,
|
|
SnapshotId: image.ID,
|
|
RegionNames: p.config.ImageRegions,
|
|
Client: client,
|
|
}
|
|
|
|
if !p.config.SkipClean {
|
|
ui.Message(fmt.Sprintf("Deleting import source spaces://%s/%s", p.config.SpaceName, p.config.ObjectName))
|
|
err = deleteImageFromSpaces(p, sess)
|
|
if err != nil {
|
|
return nil, false, false, err
|
|
}
|
|
}
|
|
|
|
return artifact, false, false, nil
|
|
}
|
|
|
|
func uploadImageToSpaces(source string, p *PostProcessor, s *session.Session) (err error) {
|
|
file, err := os.Open(source)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to open %s: %s", source, err)
|
|
}
|
|
|
|
uploader := s3manager.NewUploader(s)
|
|
_, err = uploader.Upload(&s3manager.UploadInput{
|
|
Body: file,
|
|
Bucket: &p.config.SpaceName,
|
|
Key: &p.config.ObjectName,
|
|
ACL: aws.String("public-read"),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to upload %s: %s", source, err)
|
|
}
|
|
|
|
file.Close()
|
|
|
|
return nil
|
|
}
|
|
|
|
func importImageFromSpaces(p *PostProcessor, client *godo.Client) (image *godo.Image, err error) {
|
|
log.Printf("Importing custom image from spaces://%s/%s", p.config.SpaceName, p.config.ObjectName)
|
|
|
|
url := fmt.Sprintf("https://%s.%s.digitaloceanspaces.com/%s", p.config.SpaceName, p.config.SpacesRegion, p.config.ObjectName)
|
|
createRequest := &godo.CustomImageCreateRequest{
|
|
Name: p.config.Name,
|
|
Url: url,
|
|
Region: p.config.ImageRegions[0],
|
|
Distribution: p.config.Distribution,
|
|
Description: p.config.Description,
|
|
Tags: p.config.Tags,
|
|
}
|
|
|
|
image, _, err = client.Images.Create(context.TODO(), createRequest)
|
|
if err != nil {
|
|
return image, fmt.Errorf("Failed to import from spaces://%s/%s: %s", p.config.SpaceName, p.config.ObjectName, err)
|
|
}
|
|
|
|
return image, nil
|
|
}
|
|
|
|
func waitUntilImageAvailable(client *godo.Client, imageId int, timeout time.Duration) (err error) {
|
|
done := make(chan struct{})
|
|
defer close(done)
|
|
|
|
result := make(chan error, 1)
|
|
go func() {
|
|
attempts := 0
|
|
for {
|
|
attempts += 1
|
|
|
|
log.Printf("Waiting for image to become available... (attempt: %d)", attempts)
|
|
image, _, err := client.Images.GetByID(context.TODO(), imageId)
|
|
if err != nil {
|
|
result <- err
|
|
return
|
|
}
|
|
|
|
if image.Status == "available" {
|
|
result <- nil
|
|
return
|
|
}
|
|
|
|
if image.ErrorMessage != "" {
|
|
result <- fmt.Errorf("%v", image.ErrorMessage)
|
|
return
|
|
}
|
|
|
|
time.Sleep(3 * time.Second)
|
|
|
|
select {
|
|
case <-done:
|
|
return
|
|
default:
|
|
}
|
|
}
|
|
}()
|
|
|
|
log.Printf("Waiting for up to %d seconds for image to become available", timeout/time.Second)
|
|
select {
|
|
case err := <-result:
|
|
return err
|
|
case <-time.After(timeout):
|
|
err := fmt.Errorf("Timeout while waiting to for action to become available")
|
|
return err
|
|
}
|
|
}
|
|
|
|
func distributeImageToRegions(client *godo.Client, imageId int, regions []string, timeout time.Duration) (err error) {
|
|
for _, region := range regions {
|
|
transferRequest := &godo.ActionRequest{
|
|
"type": "transfer",
|
|
"region": region,
|
|
}
|
|
log.Printf("Transferring image to %s", region)
|
|
action, _, err := client.ImageActions.Transfer(context.TODO(), imageId, transferRequest)
|
|
if err != nil {
|
|
return fmt.Errorf("Error transferring image: %s", err)
|
|
}
|
|
|
|
if err := digitalocean.WaitForImageState(godo.ActionCompleted, imageId, action.ID, client, timeout); err != nil {
|
|
if err != nil {
|
|
return fmt.Errorf("Error transferring image: %s", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func deleteImageFromSpaces(p *PostProcessor, s *session.Session) (err error) {
|
|
s3conn := s3.New(s)
|
|
_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
|
|
Bucket: &p.config.SpaceName,
|
|
Key: &p.config.ObjectName,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to delete spaces://%s/%s: %s", p.config.SpaceName, p.config.ObjectName, err)
|
|
}
|
|
|
|
return nil
|
|
}
|