Add support for uploading directly to storage on Vagrant Cloud (#10193)

Vagrant Cloud provides support for uploading directly to the backend
storage instead of streaming through Vagrant Cloud. This adds support
for direct to storage uploads and sets it as the default upload method.
A new option has been added to disable this behavior and revert back
to streaming upload via Vagrant Cloud (`no_direct_upload`).

This default for uploading directly to the backend storage also matches
up with changes being added to Vagrant proper for box upload behavior:
hashicorp/vagrant#11916

post-processor/vagrant-cloud/client.go

@@ -156,6 +156,52 @@ func (v *VagrantCloudClient) Upload(path string, url string) (*http.Response, er
 	return resp, err
 }
 
+func (v *VagrantCloudClient) DirectUpload(path string, url string) (*http.Response, error) {
+	file, err := os.Open(path)
+
+	if err != nil {
+		return nil, fmt.Errorf("Error opening file for upload: %s", err)
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+
+	if err != nil {
+		return nil, fmt.Errorf("Error stating file for upload: %s", err)
+	}
+
+	request, err := http.NewRequest("PUT", url, file)
+
+	if err != nil {
+		return nil, fmt.Errorf("Error preparing upload request: %s", err)
+	}
+
+	log.Printf("Post-Processor Vagrant Cloud API Direct Upload: %s %s", path, url)
+
+	request.ContentLength = fi.Size()
+	resp, err := v.client.Do(request)
+
+	log.Printf("Post-Processor Vagrant Cloud Direct Upload Response: \n\n%+v", resp)
+
+	return resp, err
+}
+
+func (v *VagrantCloudClient) Callback(url string) (*http.Response, error) {
+	request, err := v.newRequest("PUT", url, nil)
+
+	if err != nil {
+		return nil, fmt.Errorf("Error preparing callback request: %s", err)
+	}
+
+	log.Printf("Post-Processor Vagrant Cloud API Direct Upload Callback: %s", url)
+
+	resp, err := v.client.Do(request)
+
+	log.Printf("Post-Processor Vagrant Cloud Direct Upload Callback Response: \n\n%+v", resp)
+
+	return resp, err
+}
+
 func (v *VagrantCloudClient) Post(path string, body interface{}) (*http.Response, error) {
 	reqUrl := fmt.Sprintf("%s/%s", v.BaseURL, path)
 

post-processor/vagrant-cloud/post-processor.go

@@ -45,8 +45,8 @@ type Config struct {
 	AccessToken           string `mapstructure:"access_token"`
 	VagrantCloudUrl       string `mapstructure:"vagrant_cloud_url"`
 	InsecureSkipTLSVerify bool   `mapstructure:"insecure_skip_tls_verify"`
-
+	BoxDownloadUrl        string `mapstructure:"box_download_url"`
-	BoxDownloadUrl string `mapstructure:"box_download_url"`
+	NoDirectUpload        bool   `mapstructure:"no_direct_upload"`
 
 	ctx interpolate.Context
 }
@@ -181,24 +181,18 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 	state.Put("boxDownloadUrl", boxDownloadUrl)
 
 	// Build the steps
-	steps := []multistep.Step{}
+	steps := []multistep.Step{
+		new(stepVerifyBox),
+		new(stepCreateVersion),
+		new(stepCreateProvider),
+	}
 	if p.config.BoxDownloadUrl == "" {
-		steps = []multistep.Step{
+		steps = append(steps, new(stepPrepareUpload), new(stepUpload))
-			new(stepVerifyBox),
+		if !p.config.NoDirectUpload {
-			new(stepCreateVersion),
+			steps = append(steps, new(stepConfirmUpload))
-			new(stepCreateProvider),
-			new(stepPrepareUpload),
-			new(stepUpload),
-			new(stepReleaseVersion),
-		}
-	} else {
-		steps = []multistep.Step{
-			new(stepVerifyBox),
-			new(stepCreateVersion),
-			new(stepCreateProvider),
-			new(stepReleaseVersion),
 		}
 	}
+	steps = append(steps, new(stepReleaseVersion))
 
 	// Run the steps
 	p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)

post-processor/vagrant-cloud/post-processor.hcl2spec.go

@@ -24,6 +24,7 @@ type FlatConfig struct {
 	VagrantCloudUrl       *string           `mapstructure:"vagrant_cloud_url" cty:"vagrant_cloud_url" hcl:"vagrant_cloud_url"`
 	InsecureSkipTLSVerify *bool             `mapstructure:"insecure_skip_tls_verify" cty:"insecure_skip_tls_verify" hcl:"insecure_skip_tls_verify"`
 	BoxDownloadUrl        *string           `mapstructure:"box_download_url" cty:"box_download_url" hcl:"box_download_url"`
+	NoDirectUpload        *bool             `mapstructure:"no_direct_upload" cty:"no_direct_upload" hcl:"no_direct_upload"`
 }
 
 // FlatMapstructure returns a new FlatConfig.
@@ -53,6 +54,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"vagrant_cloud_url":          &hcldec.AttrSpec{Name: "vagrant_cloud_url", Type: cty.String, Required: false},
 		"insecure_skip_tls_verify":   &hcldec.AttrSpec{Name: "insecure_skip_tls_verify", Type: cty.Bool, Required: false},
 		"box_download_url":           &hcldec.AttrSpec{Name: "box_download_url", Type: cty.String, Required: false},
+		"no_direct_upload":           &hcldec.AttrSpec{Name: "no_direct_upload", Type: cty.Bool, Required: false},
 	}
 	return s
 }

post-processor/vagrant-cloud/post-processor_test.go

@@ -17,6 +17,13 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+type stubResponse struct {
+	Path       string
+	Method     string
+	Response   string
+	StatusCode int
+}
+
 type tarFiles []struct {
 	Name, Body string
 }
@@ -46,6 +53,36 @@ func testNoAccessTokenProvidedConfig() map[string]interface{} {
 	}
 }
 
+func newStackServer(stack []stubResponse) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		if len(stack) < 1 {
+			rw.Header().Add("Error", fmt.Sprintf("Request stack is empty - Method: %s Path: %s", req.Method, req.URL.Path))
+			http.Error(rw, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+			return
+		}
+		match := stack[0]
+		stack = stack[1:]
+		if match.Method != "" && req.Method != match.Method {
+			rw.Header().Add("Error", fmt.Sprintf("Request %s != %s", match.Method, req.Method))
+			http.Error(rw, fmt.Sprintf("Request %s != %s", match.Method, req.Method), http.StatusInternalServerError)
+			return
+		}
+		if match.Path != "" && match.Path != req.URL.Path {
+			rw.Header().Add("Error", fmt.Sprintf("Request %s != %s", match.Path, req.URL.Path))
+			http.Error(rw, fmt.Sprintf("Request %s != %s", match.Path, req.URL.Path), http.StatusInternalServerError)
+			return
+		}
+		rw.Header().Add("Complete", fmt.Sprintf("Method: %s Path: %s", match.Method, match.Path))
+		rw.WriteHeader(match.StatusCode)
+		if match.Response != "" {
+			_, err := rw.Write([]byte(match.Response))
+			if err != nil {
+				panic("failed to write response: " + err.Error())
+			}
+		}
+	}))
+}
+
 func newSecureServer(token string, handler http.HandlerFunc) *httptest.Server {
 	token = fmt.Sprintf("Bearer %s", token)
 	return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
@@ -229,6 +266,156 @@ func TestPostProcessor_PostProcess_checkArtifactFileIsBox(t *testing.T) {
 	}
 }
 
+func TestPostProcessor_PostProcess_uploadsAndReleases(t *testing.T) {
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+		{"metadata.json", `{"provider": "virtualbox"}`},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	artifact := &packer.MockArtifact{
+		BuilderIdValue: "mitchellh.post-processor.vagrant",
+		FilesValue:     []string{boxfile.Name()},
+	}
+
+	s := newStackServer([]stubResponse{stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"}})
+	defer s.Close()
+
+	stack := []stubResponse{
+		stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"},
+		stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`},
+		stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`},
+		stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`},
+		stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload", Response: `{"upload_path": "` + s.URL + `/box-upload-path"}`},
+		stubResponse{StatusCode: 200, Method: "PUT", Path: "/box/hashicorp/precise64/version/0.5/release"},
+	}
+
+	server := newStackServer(stack)
+	defer server.Close()
+	config := testGoodConfig()
+	config["vagrant_cloud_url"] = server.URL
+	config["no_direct_upload"] = true
+
+	var p PostProcessor
+
+	err = p.Configure(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	_, _, _, err = p.PostProcess(context.Background(), testUi(), artifact)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestPostProcessor_PostProcess_uploadsAndNoRelease(t *testing.T) {
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+		{"metadata.json", `{"provider": "virtualbox"}`},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	artifact := &packer.MockArtifact{
+		BuilderIdValue: "mitchellh.post-processor.vagrant",
+		FilesValue:     []string{boxfile.Name()},
+	}
+
+	s := newStackServer([]stubResponse{stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"}})
+	defer s.Close()
+
+	stack := []stubResponse{
+		stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"},
+		stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`},
+		stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`},
+		stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`},
+		stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload", Response: `{"upload_path": "` + s.URL + `/box-upload-path"}`},
+	}
+
+	server := newStackServer(stack)
+	defer server.Close()
+	config := testGoodConfig()
+	config["vagrant_cloud_url"] = server.URL
+	config["no_direct_upload"] = true
+	config["no_release"] = true
+
+	var p PostProcessor
+
+	err = p.Configure(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	_, _, _, err = p.PostProcess(context.Background(), testUi(), artifact)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestPostProcessor_PostProcess_uploadsDirectAndReleases(t *testing.T) {
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+		{"metadata.json", `{"provider": "virtualbox"}`},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	artifact := &packer.MockArtifact{
+		BuilderIdValue: "mitchellh.post-processor.vagrant",
+		FilesValue:     []string{boxfile.Name()},
+	}
+
+	s := newStackServer(
+		[]stubResponse{
+			stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"},
+		},
+	)
+	defer s.Close()
+
+	stack := []stubResponse{
+		stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"},
+		stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`},
+		stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`},
+		stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`},
+		stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload/direct"},
+		stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-complete"},
+		stubResponse{StatusCode: 200, Method: "PUT", Path: "/box/hashicorp/precise64/version/0.5/release"},
+	}
+
+	server := newStackServer(stack)
+	defer server.Close()
+	config := testGoodConfig()
+	config["vagrant_cloud_url"] = server.URL
+
+	// Set response here so we have API server URL available
+	stack[4].Response = `{"upload_path": "` + s.URL + `/box-upload-path", "callback": "` + server.URL + `/box-upload-complete"}`
+
+	var p PostProcessor
+
+	err = p.Configure(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	_, _, _, err = p.PostProcess(context.Background(), testUi(), artifact)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
 func testUi() *packer.BasicUi {
 	return &packer.BasicUi{
 		Reader: new(bytes.Buffer),

post-processor/vagrant-cloud/step_confirm_upload.go

@@ -0,0 +1,43 @@
+package vagrantcloud
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+type stepConfirmUpload struct {
+}
+
+func (s *stepConfirmUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	client := state.Get("client").(*VagrantCloudClient)
+	ui := state.Get("ui").(packer.Ui)
+	upload := state.Get("upload").(*Upload)
+	url := upload.CallbackPath
+
+	ui.Say("Confirming direct box upload completion")
+
+	resp, err := client.Callback(url)
+
+	if err != nil || resp.StatusCode != 200 {
+		if resp == nil || resp.Body == nil {
+			state.Put("error", "No response from server.")
+		} else {
+			cloudErrors := &VagrantCloudErrors{}
+			err = decodeBody(resp, cloudErrors)
+			if err != nil {
+				ui.Error(fmt.Sprintf("error decoding error response: %s", err))
+			}
+			state.Put("error", fmt.Errorf("Error preparing upload: %s", cloudErrors.FormatErrors()))
+		}
+		return multistep.ActionHalt
+	}
+
+	return multistep.ActionContinue
+}
+
+func (s *stepConfirmUpload) Cleanup(state multistep.StateBag) {
+	// No cleanup
+}

post-processor/vagrant-cloud/step_prepare_upload.go

@@ -9,7 +9,8 @@ import (
 )
 
 type Upload struct {
-	UploadPath string `json:"upload_path"`
+	UploadPath   string `json:"upload_path"`
+	CallbackPath string `json:"callback"`
 }
 
 type stepPrepareUpload struct {
@@ -17,6 +18,7 @@ type stepPrepareUpload struct {
 
 func (s *stepPrepareUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
 	client := state.Get("client").(*VagrantCloudClient)
+	config := state.Get("config").(*Config)
 	ui := state.Get("ui").(packer.Ui)
 	box := state.Get("box").(*Box)
 	version := state.Get("version").(*Version)
@@ -24,6 +26,9 @@ func (s *stepPrepareUpload) Run(ctx context.Context, state multistep.StateBag) m
 	artifactFilePath := state.Get("artifactFilePath").(string)
 
 	path := fmt.Sprintf("box/%s/version/%v/provider/%s/upload", box.Tag, version.Version, provider.Name)
+	if !config.NoDirectUpload {
+		path = path + "/direct"
+	}
 	upload := &Upload{}
 
 	ui.Say(fmt.Sprintf("Preparing upload of box: %s", artifactFilePath))

post-processor/vagrant-cloud/step_upload.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"net/http"
 	"time"
 
 	"github.com/hashicorp/packer/common/retry"
@@ -16,6 +17,7 @@ type stepUpload struct {
 
 func (s *stepUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
 	client := state.Get("client").(*VagrantCloudClient)
+	config := state.Get("config").(*Config)
 	ui := state.Get("ui").(packer.Ui)
 	upload := state.Get("upload").(*Upload)
 	artifactFilePath := state.Get("artifactFilePath").(string)
@@ -32,7 +34,14 @@ func (s *stepUpload) Run(ctx context.Context, state multistep.StateBag) multiste
 	}.Run(ctx, func(ctx context.Context) error {
 		ui.Message(fmt.Sprintf("Uploading box"))
 
-		resp, err := client.Upload(artifactFilePath, url)
+		var err error
+		var resp *http.Response
+
+		if config.NoDirectUpload {
+			resp, err = client.Upload(artifactFilePath, url)
+		} else {
+			resp, err = client.DirectUpload(artifactFilePath, url)
+		}
 		if err != nil {
 			ui.Message(fmt.Sprintf(
 				"Error uploading box! Will retry in 10 seconds. Error: %s", err))

website/pages/docs/post-processors/vagrant-cloud.mdx

@@ -114,6 +114,9 @@ on Vagrant Cloud, as well as authentication and version information.
   - `Provider`: The Vagrant provider the box is for
   - `ArtifactId`: The ID of the input artifact.
 
+- `no_direct_upload` (boolean) - When true, upload the box artifact through
+  Vagrant Cloud instead of directly to the backend storage.
+
 ## Use with the Vagrant Post-Processor
 
 An example configuration is shown below. Note the use of the nested array that