iSharkFly-Docs
/
packer-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

extend artifact, code refactoring

This commit is contained in:
Gennady Lipenkov 2020-07-22 01:01:30 +03:00
parent e0cc7b9d8c
commit 53e7f4e71a
6 changed files with 282 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
post-processor/yandex-import/artifact.go
View File

@ -7,8 +7,13 @@ import (
const BuilderId = "packer.post-processor.yandex-import"
type Artifact struct {
imageID string
imageSource string
imageID string
sourceType string
sourceID string
// StateData should store data such as GeneratedData
// to be shared with post-processors
StateData map[string]interface{}
}
func (*Artifact) BuilderId() string {
@ -16,7 +21,7 @@ func (*Artifact) BuilderId() string {
}
func (a *Artifact) Id() string {
return a.imageSource
return a.imageID
}
func (a *Artifact) Files() []string {
@ -24,10 +29,14 @@ func (a *Artifact) Files() []string {
}
func (a *Artifact) String() string {
return fmt.Sprintf("Create image %v from source URL/image %v", a.imageID, a.imageSource)
return fmt.Sprintf("Create image %v from source type %v with ID/URL %v", a.imageID, a.sourceType, a.sourceID)
}
func (*Artifact) State(name string) interface{} {
func (a *Artifact) State(name string) interface{} {
if _, ok := a.StateData[name]; ok {
return a.StateData[name]
}
return nil
}

26
post-processor/yandex-import/artifact_test.go Normal file
View File

@ -0,0 +1,26 @@
package yandeximport
import (
"github.com/stretchr/testify/require"
"testing"
)
func TestArtifactState_StateData(t *testing.T) {
expectedData := "this is the data"
artifact := &Artifact{
StateData: map[string]interface{}{"state_data": expectedData},
}
// Valid state
result := artifact.State("state_data")
require.Equal(t, expectedData, result)
// Invalid state
result = artifact.State("invalid_key")
require.Equal(t, nil, result)
// Nil StateData should not fail and should return nil
artifact = &Artifact{}
result = artifact.State("key")
require.Equal(t, nil, result)
}

189
post-processor/yandex-import/post-processor.go
View File

@ -6,19 +6,11 @@ package yandeximport
import (
"context"
"fmt"
"net/url"
"os"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/packer/builder/yandex"
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
"github.com/yandex-cloud/go-sdk/iamkey"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/packer/builder/file"
"github.com/hashicorp/packer/builder/yandex"
"github.com/hashicorp/packer/common"
"github.com/hashicorp/packer/helper/config"
"github.com/hashicorp/packer/packer"
@ -26,6 +18,8 @@ import (
"github.com/hashicorp/packer/post-processor/compress"
yandexexport "github.com/hashicorp/packer/post-processor/yandex-export"
"github.com/hashicorp/packer/template/interpolate"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
"github.com/yandex-cloud/go-sdk/iamkey"
)
type Config struct {
@ -150,6 +144,10 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
}
func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) {
var imageSrc cloudImageSource
var fileSource bool
var err error
generatedData := artifact.State("generated_data")
if generatedData == nil {
// Make sure it's not a nil map so we can assign to it later.
@ -157,6 +155,11 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
}
p.config.ctx.Data = generatedData
p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx)
if err != nil {
return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
}
cfg := &yandex.Config{
Token: p.config.Token,
ServiceAccountKeyFile: p.config.ServiceAccountKeyFile,
@ -167,14 +170,6 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
return nil, false, false, err
}
p.config.ObjectName, err = interpolate.Render(p.config.ObjectName, &p.config.ctx)
if err != nil {
return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
}
var imageSource string
var fileSource bool
// Create temporary storage Access Key
respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
ServiceAccountId: p.config.ServiceAccountID,
@ -199,22 +194,29 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value")
}
imageSource, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
imageSrc, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
if err != nil {
return nil, false, false, err
}
case yandexexport.BuilderId, BuilderId:
case yandexexport.BuilderId:
// Artifact already in storage, just get URL
// OR artifact from prev yandex-import PP, reuse URL
imageSource, err = presignUrl(storageClient, ui, artifact.Id())
imageSrc, err = presignUrl(storageClient, ui, artifact.Id())
if err != nil {
return nil, false, false, err
}
case yandex.BuilderID:
// Artifact is plain Yandex Compute Image, just create new one based on provided.
imageSource = artifact.Id()
// Artifact is plain Yandex Compute Image, just create new one based on provided
imageSrc = &imageSource{
imageID: artifact.Id(),
}
case BuilderId:
// Artifact from prev yandex-import PP, reuse URL or Cloud Image ID
imageSrc, err = chooseSource(artifact)
if err != nil {
return nil, false, false, err
}
default:
err := fmt.Errorf(
@ -223,13 +225,13 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
return nil, false, false, err
}
ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, imageSource, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, imageSrc, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
if err != nil {
return nil, false, false, err
}
if fileSource && !p.config.SkipClean {
err = deleteFromBucket(storageClient, ui, imageSource)
err = deleteFromBucket(storageClient, ui, imageSrc)
if err != nil {
return nil, false, false, err
}
@ -244,128 +246,29 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
}
return &Artifact{
imageID: ycImage.GetId(),
imageSource: imageSource,
imageID: ycImage.GetId(),
StateData: map[string]interface{}{
"source_type": imageSrc.GetSourceType(),
"source_id": imageSrc.GetSourceID(),
},
}, false, false, nil
}
func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
ui.Say("Looking for qcow2 file in list of artifacts...")
source := ""
for _, path := range artifact.Files() {
ui.Say(fmt.Sprintf("Found artifact %v...", path))
if strings.HasSuffix(path, ".qcow2") {
source = path
break
}
func chooseSource(a packer.Artifact) (cloudImageSource, error) {
st := a.State("source_type").(string)
if st == "" {
return nil, fmt.Errorf("could not determine source type of yandex-import artifact: %v", a)
}
switch st {
case sourceType_IMAGE:
return &imageSource{
imageID: a.State("source_id").(string),
}, nil
if source == "" {
return "", fmt.Errorf("no qcow2 file found in list of artifacts")
case sourceType_OBJECT:
return &objectSource{
url: a.State("source_id").(string),
}, nil
}
artifactFile, err := os.Open(source)
if err != nil {
err := fmt.Errorf("error opening %v", source)
return "", err
}
ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName))
_, err = s3conn.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
Body: artifactFile,
})
if err != nil {
ui.Say(fmt.Sprintf("Failed to upload: %v", objectName))
return "", err
}
req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
})
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
req.Config.S3ForcePathStyle = aws.Bool(true)
return req.HTTPRequest.URL.String(), nil
}
func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, imageSource string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) {
req := &compute.CreateImageRequest{
FolderId: folderID,
Name: imageName,
Description: imageDescription,
Labels: imageLabels,
Family: imageFamily,
}
// switch on imageSource value: cloud image id or storage URL
_, err := url.Parse(imageSource)
if err == nil {
req.Source = &compute.CreateImageRequest_Uri{Uri: imageSource}
} else {
req.Source = &compute.CreateImageRequest_ImageId{ImageId: imageSource}
}
op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, req))
if err != nil {
ui.Say("Error creating Yandex Compute Image")
return nil, err
}
ui.Say(fmt.Sprintf("Source url for Image creation: %v", imageSource))
ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id()))
ui.Say("Waiting for Yandex Compute Image creation operation to complete...")
err = op.Wait(ctx)
// fail if image creation operation has an error
if err != nil {
return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err)
}
protoMetadata, err := op.Metadata()
if err != nil {
return nil, fmt.Errorf("error while get image create operation metadata: %s", err)
}
md, ok := protoMetadata.(*compute.CreateImageMetadata)
if !ok {
return nil, fmt.Errorf("could not get Image ID from create operation metadata")
}
image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
ImageId: md.ImageId,
})
if err != nil {
return nil, fmt.Errorf("error while image get request: %s", err)
}
return image, nil
}
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error {
bucket, objectName, err := s3URLToBucketKey(url)
if err != nil {
return err
}
ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
})
if err != nil {
ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName))
return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err)
}
return nil
return nil, fmt.Errorf("unknow source type of yandex-import artifact: %s", st)
}

44
post-processor/yandex-import/source.go Normal file
View File

@ -0,0 +1,44 @@
package yandeximport
import "fmt"
const sourceType_IMAGE = "image"
const sourceType_OBJECT = "object"
type cloudImageSource interface {
GetSourceID() string
GetSourceType() string
Description() string
}
type imageSource struct {
imageID string
}
func (i *imageSource) GetSourceID() string {
return i.imageID
}
func (i *imageSource) GetSourceType() string {
return sourceType_IMAGE
}
func (i *imageSource) Description() string {
return fmt.Sprintf("%s source, id: %s", i.GetSourceType(), i.imageID)
}
type objectSource struct {
url string
}
func (i *objectSource) GetSourceID() string {
return i.url
}
func (i *objectSource) GetSourceType() string {
return sourceType_OBJECT
}
func (i *objectSource) Description() string {
return fmt.Sprintf("%s source, url: %s", i.GetSourceType(), i.url)
}

10
post-processor/yandex-import/storage.go
View File

@ -46,10 +46,10 @@ func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, e
}
// Get path-style S3 URL and return presigned URL
func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) {
func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (cloudImageSource, error) {
bucket, key, err := s3URLToBucketKey(fullUrl)
if err != nil {
return "", err
return nil, err
}
req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
@ -63,10 +63,12 @@ func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) {
urlStr, _, err := req.PresignRequest(30 * time.Minute)
if err != nil {
ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
return "", err
return nil, err
}
return urlStr, nil
return &objectSource{
urlStr,
}, nil
}
func s3URLToBucketKey(storageURL string) (bucket string, key string, err error) {

146
post-processor/yandex-import/utils.go Normal file
View File

@ -0,0 +1,146 @@
package yandeximport
import (
"context"
"fmt"
"os"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/packer/builder/yandex"
"github.com/hashicorp/packer/packer"
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
)
func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (cloudImageSource, error) {
ui.Say("Looking for qcow2 file in list of artifacts...")
source := ""
for _, path := range artifact.Files() {
ui.Say(fmt.Sprintf("Found artifact %v...", path))
if strings.HasSuffix(path, ".qcow2") {
source = path
break
}
}
if source == "" {
return nil, fmt.Errorf("no qcow2 file found in list of artifacts")
}
artifactFile, err := os.Open(source)
if err != nil {
err := fmt.Errorf("error opening %v", source)
return nil, err
}
ui.Say(fmt.Sprintf("Uploading file %v to bucket %v/%v...", source, bucket, objectName))
_, err = s3conn.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
Body: artifactFile,
})
if err != nil {
ui.Say(fmt.Sprintf("Failed to upload: %v", objectName))
return nil, err
}
req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
})
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
req.Config.S3ForcePathStyle = aws.Bool(true)
return &objectSource{
url: req.HTTPRequest.URL.String(),
}, nil
}
func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, imageSrc cloudImageSource, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) {
req := &compute.CreateImageRequest{
FolderId: folderID,
Name: imageName,
Description: imageDescription,
Labels: imageLabels,
Family: imageFamily,
}
// switch on cloudImageSource type: cloud image id or storage URL
switch v := imageSrc.(type) {
case *imageSource:
req.Source = &compute.CreateImageRequest_ImageId{ImageId: v.imageID}
case *objectSource:
req.Source = &compute.CreateImageRequest_Uri{Uri: v.url}
}
op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, req))
if err != nil {
ui.Say("Error creating Yandex Compute Image")
return nil, err
}
ui.Say(fmt.Sprintf("Source of Image creation: %s", imageSrc.Description()))
ui.Say(fmt.Sprintf("Creating Yandex Compute Image %v within operation %#v", imageName, op.Id()))
ui.Say("Waiting for Yandex Compute Image creation operation to complete...")
err = op.Wait(ctx)
// fail if image creation operation has an error
if err != nil {
return nil, fmt.Errorf("failed to create Yandex Compute Image: %s", err)
}
protoMetadata, err := op.Metadata()
if err != nil {
return nil, fmt.Errorf("error while get image create operation metadata: %s", err)
}
md, ok := protoMetadata.(*compute.CreateImageMetadata)
if !ok {
return nil, fmt.Errorf("could not get Image ID from create operation metadata")
}
image, err := driver.SDK().Compute().Image().Get(ctx, &compute.GetImageRequest{
ImageId: md.ImageId,
})
if err != nil {
return nil, fmt.Errorf("error while image get request: %s", err)
}
return image, nil
}
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, imageSrc cloudImageSource) error {
var url string
// switch on cloudImageSource type: cloud image id or storage URL
switch v := imageSrc.(type) {
case *objectSource:
url = v.GetSourceID()
case *imageSource:
return fmt.Errorf("invalid argument for `deleteFromBucket` method: %v", v)
}
bucket, objectName, err := s3URLToBucketKey(url)
if err != nil {
return err
}
ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
})
if err != nil {
ui.Say(fmt.Sprintf("Failed to delete: %v/%v", bucket, objectName))
return fmt.Errorf("error deleting storage object %q in bucket %q: %s ", objectName, bucket, err)
}
return nil
}