From 9b641c9bfd96180e58c30d5c5a22f10d51baf11d Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 18 Mar 2021 17:22:27 -0500 Subject: [PATCH 1/5] Force NoDirectUpload for vagrantcloud if asset size > 5 GB --- .../vagrant-cloud/step_prepare_upload.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/post-processor/vagrant-cloud/step_prepare_upload.go b/post-processor/vagrant-cloud/step_prepare_upload.go index 4f5efcb5a..b68c8bbe0 100644 --- a/post-processor/vagrant-cloud/step_prepare_upload.go +++ b/post-processor/vagrant-cloud/step_prepare_upload.go @@ -3,11 +3,14 @@ package vagrantcloud import ( "context" "fmt" + "os" "github.com/hashicorp/packer-plugin-sdk/multistep" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" ) +const VAGRANT_CLOUD_DIRECT_UPLOAD_LIMIT = 5000000000 // Upload limit is 5GB + type Upload struct { UploadPath string `json:"upload_path"` CallbackPath string `json:"callback"` @@ -25,6 +28,18 @@ func (s *stepPrepareUpload) Run(ctx context.Context, state multistep.StateBag) m provider := state.Get("provider").(*Provider) artifactFilePath := state.Get("artifactFilePath").(string) + // If direct upload is enabled, the asset size must be <= 5 GB + if config.NoDirectUpload == false { + f, err := os.Stat(artifactFilePath) + if err != nil { + ui.Error(fmt.Sprintf("error determining size of upload artifact: %s", artifactFilePath)) + } + if f.Size() > VAGRANT_CLOUD_DIRECT_UPLOAD_LIMIT { + ui.Say(fmt.Sprintf("Asset %s is larger than the direct upload limit. Setting `NoDirectUpload` to true", artifactFilePath)) + config.NoDirectUpload = true + } + } + path := fmt.Sprintf("box/%s/version/%v/provider/%s/upload", box.Tag, version.Version, provider.Name) if !config.NoDirectUpload { path = path + "/direct" From 3a11820a41ca8df3995fe3e787d60558e37feced Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Wed, 24 Mar 2021 10:51:10 -0700 Subject: [PATCH 2/5] Adjust upload limit value and fix error value stored in state bag --- post-processor/vagrant-cloud/step_prepare_upload.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/post-processor/vagrant-cloud/step_prepare_upload.go b/post-processor/vagrant-cloud/step_prepare_upload.go index b68c8bbe0..07ab7d7de 100644 --- a/post-processor/vagrant-cloud/step_prepare_upload.go +++ b/post-processor/vagrant-cloud/step_prepare_upload.go @@ -9,7 +9,7 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" ) -const VAGRANT_CLOUD_DIRECT_UPLOAD_LIMIT = 5000000000 // Upload limit is 5GB +const VAGRANT_CLOUD_DIRECT_UPLOAD_LIMIT = 5368709120 // Upload limit is 5G type Upload struct { UploadPath string `json:"upload_path"` @@ -52,7 +52,7 @@ func (s *stepPrepareUpload) Run(ctx context.Context, state multistep.StateBag) m if err != nil || (resp.StatusCode != 200) { if resp == nil || resp.Body == nil { - state.Put("error", "No response from server.") + state.Put("error", fmt.Errorf("No response from server.")) } else { cloudErrors := &VagrantCloudErrors{} err = decodeBody(resp, cloudErrors) From 2de91e48627ccbe8b7c043f6bdfc438f0b8e4919 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Wed, 24 Mar 2021 10:52:56 -0700 Subject: [PATCH 3/5] Check configuration before running callback for upload confirmation --- post-processor/vagrant-cloud/step_confirm_upload.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/post-processor/vagrant-cloud/step_confirm_upload.go b/post-processor/vagrant-cloud/step_confirm_upload.go index fcb8e02ca..a177ecbcd 100644 --- a/post-processor/vagrant-cloud/step_confirm_upload.go +++ b/post-processor/vagrant-cloud/step_confirm_upload.go @@ -16,6 +16,11 @@ func (s *stepConfirmUpload) Run(ctx context.Context, state multistep.StateBag) m ui := state.Get("ui").(packersdk.Ui) upload := state.Get("upload").(*Upload) url := upload.CallbackPath + config := state.Get("config").(*Config) + + if config.NoDirectUpload { + return multistep.ActionContinue + } ui.Say("Confirming direct box upload completion") @@ -23,7 +28,7 @@ func (s *stepConfirmUpload) Run(ctx context.Context, state multistep.StateBag) m if err != nil || resp.StatusCode != 200 { if resp == nil || resp.Body == nil { - state.Put("error", "No response from server.") + state.Put("error", fmt.Errorf("No response from server.")) } else { cloudErrors := &VagrantCloudErrors{} err = decodeBody(resp, cloudErrors) From a665e6b822190ef9dbbbd9a4c3df7fd89bd3753b Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Wed, 24 Mar 2021 10:53:38 -0700 Subject: [PATCH 4/5] Always include all upload steps regardless of configuration --- post-processor/vagrant-cloud/post-processor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/post-processor/vagrant-cloud/post-processor.go b/post-processor/vagrant-cloud/post-processor.go index 53ff8c205..09e74e7b7 100644 --- a/post-processor/vagrant-cloud/post-processor.go +++ b/post-processor/vagrant-cloud/post-processor.go @@ -188,10 +188,10 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa new(stepCreateProvider), } if p.config.BoxDownloadUrl == "" { - steps = append(steps, new(stepPrepareUpload), new(stepUpload)) - if !p.config.NoDirectUpload { - steps = append(steps, new(stepConfirmUpload)) - } + steps = append(steps, + new(stepPrepareUpload), + new(stepUpload), + new(stepConfirmUpload)) } steps = append(steps, new(stepReleaseVersion)) From 4ea4c0570f27b1c7128dce091b54edff601527ef Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Wed, 24 Mar 2021 10:54:11 -0700 Subject: [PATCH 5/5] Add test coverage for direct upload file size limits --- .../vagrant-cloud/post-processor_test.go | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/post-processor/vagrant-cloud/post-processor_test.go b/post-processor/vagrant-cloud/post-processor_test.go index 298398154..c29f7ec2d 100644 --- a/post-processor/vagrant-cloud/post-processor_test.go +++ b/post-processor/vagrant-cloud/post-processor_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "os" + "runtime" "strings" "testing" @@ -361,6 +362,138 @@ func TestPostProcessor_PostProcess_uploadsAndNoRelease(t *testing.T) { } } +func TestPostProcessor_PostProcess_directUpload5GFile(t *testing.T) { + // Disable test on Windows due to unreliable sparse file creation + if runtime.GOOS == "windows" { + return + } + + // Boxes up to 5GB are supported for direct upload so + // set the box asset to be 5GB exactly + fSize := int64(5368709120) + files := tarFiles{ + {"foo.txt", "This is a foo file"}, + {"bar.txt", "This is a bar file"}, + {"metadata.json", `{"provider": "virtualbox"}`}, + } + f, err := createBox(files) + if err != nil { + t.Fatalf("%s", err) + } + defer os.Remove(f.Name()) + + if err := expandFile(f, fSize); err != nil { + t.Fatalf("failed to expand box file - %s", err) + } + + artifact := &packersdk.MockArtifact{ + BuilderIdValue: "mitchellh.post-processor.vagrant", + FilesValue: []string{f.Name()}, + } + f.Close() + + s := newStackServer( + []stubResponse{ + stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"}, + }, + ) + defer s.Close() + + stack := []stubResponse{ + stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"}, + stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`}, + stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`}, + stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`}, + stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload/direct"}, + stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-complete"}, + } + + server := newStackServer(stack) + defer server.Close() + config := testGoodConfig() + config["vagrant_cloud_url"] = server.URL + config["no_release"] = true + + // Set response here so we have API server URL available + stack[4].Response = `{"upload_path": "` + s.URL + `/box-upload-path", "callback": "` + server.URL + `/box-upload-complete"}` + + var p PostProcessor + + err = p.Configure(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + _, _, _, err = p.PostProcess(context.Background(), testUi(), artifact) + if err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestPostProcessor_PostProcess_directUploadOver5GFile(t *testing.T) { + // Disable test on Windows due to unreliable sparse file creation + if runtime.GOOS == "windows" { + return + } + + // Boxes over 5GB are not supported for direct upload so + // set the box asset to be one byte over 5GB + fSize := int64(5368709121) + files := tarFiles{ + {"foo.txt", "This is a foo file"}, + {"bar.txt", "This is a bar file"}, + {"metadata.json", `{"provider": "virtualbox"}`}, + } + f, err := createBox(files) + if err != nil { + t.Fatalf("%s", err) + } + defer os.Remove(f.Name()) + + if err := expandFile(f, fSize); err != nil { + t.Fatalf("failed to expand box file - %s", err) + } + f.Close() + + artifact := &packersdk.MockArtifact{ + BuilderIdValue: "mitchellh.post-processor.vagrant", + FilesValue: []string{f.Name()}, + } + + s := newStackServer( + []stubResponse{ + stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"}, + }, + ) + defer s.Close() + + stack := []stubResponse{ + stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"}, + stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`}, + stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`}, + stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`}, + stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload", Response: `{"upload_path": "` + s.URL + `/box-upload-path"}`}, + } + + server := newStackServer(stack) + defer server.Close() + config := testGoodConfig() + config["vagrant_cloud_url"] = server.URL + config["no_release"] = true + + var p PostProcessor + + err = p.Configure(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + _, _, _, err = p.PostProcess(context.Background(), testUi(), artifact) + if err != nil { + t.Fatalf("err: %s", err) + } +} + func TestPostProcessor_PostProcess_uploadsDirectAndReleases(t *testing.T) { files := tarFiles{ {"foo.txt", "This is a foo file"}, @@ -697,3 +830,21 @@ func createBox(files tarFiles) (boxfile *os.File, err error) { return boxfile, nil } + +func expandFile(f *os.File, finalSize int64) (err error) { + s, err := f.Stat() + if err != nil { + return + } + size := finalSize - s.Size() + if size < 1 { + return + } + if _, err = f.Seek(size-1, 2); err != nil { + return + } + if _, err = f.Write([]byte{0}); err != nil { + return + } + return nil +}