Add support for uploading directly to storage on Vagrant Cloud (#10193)

Vagrant Cloud provides support for uploading directly to the backend
storage instead of streaming through Vagrant Cloud. This adds support
for direct to storage uploads and sets it as the default upload method.
A new option has been added to disable this behavior and revert back
to streaming upload via Vagrant Cloud (`no_direct_upload`).

This default for uploading directly to the backend storage also matches
up with changes being added to Vagrant proper for box upload behavior:
hashicorp/vagrant#11916
This commit is contained in:
Chris Roberts 2020-11-05 17:01:55 -08:00 committed by GitHub
parent 7a1680df97
commit d8222b1656
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 308 additions and 19 deletions

View File

@ -156,6 +156,52 @@ func (v *VagrantCloudClient) Upload(path string, url string) (*http.Response, er
return resp, err return resp, err
} }
func (v *VagrantCloudClient) DirectUpload(path string, url string) (*http.Response, error) {
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("Error opening file for upload: %s", err)
}
defer file.Close()
fi, err := file.Stat()
if err != nil {
return nil, fmt.Errorf("Error stating file for upload: %s", err)
}
request, err := http.NewRequest("PUT", url, file)
if err != nil {
return nil, fmt.Errorf("Error preparing upload request: %s", err)
}
log.Printf("Post-Processor Vagrant Cloud API Direct Upload: %s %s", path, url)
request.ContentLength = fi.Size()
resp, err := v.client.Do(request)
log.Printf("Post-Processor Vagrant Cloud Direct Upload Response: \n\n%+v", resp)
return resp, err
}
func (v *VagrantCloudClient) Callback(url string) (*http.Response, error) {
request, err := v.newRequest("PUT", url, nil)
if err != nil {
return nil, fmt.Errorf("Error preparing callback request: %s", err)
}
log.Printf("Post-Processor Vagrant Cloud API Direct Upload Callback: %s", url)
resp, err := v.client.Do(request)
log.Printf("Post-Processor Vagrant Cloud Direct Upload Callback Response: \n\n%+v", resp)
return resp, err
}
func (v *VagrantCloudClient) Post(path string, body interface{}) (*http.Response, error) { func (v *VagrantCloudClient) Post(path string, body interface{}) (*http.Response, error) {
reqUrl := fmt.Sprintf("%s/%s", v.BaseURL, path) reqUrl := fmt.Sprintf("%s/%s", v.BaseURL, path)

View File

@ -45,8 +45,8 @@ type Config struct {
AccessToken string `mapstructure:"access_token"` AccessToken string `mapstructure:"access_token"`
VagrantCloudUrl string `mapstructure:"vagrant_cloud_url"` VagrantCloudUrl string `mapstructure:"vagrant_cloud_url"`
InsecureSkipTLSVerify bool `mapstructure:"insecure_skip_tls_verify"` InsecureSkipTLSVerify bool `mapstructure:"insecure_skip_tls_verify"`
BoxDownloadUrl string `mapstructure:"box_download_url"` BoxDownloadUrl string `mapstructure:"box_download_url"`
NoDirectUpload bool `mapstructure:"no_direct_upload"`
ctx interpolate.Context ctx interpolate.Context
} }
@ -181,24 +181,18 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
state.Put("boxDownloadUrl", boxDownloadUrl) state.Put("boxDownloadUrl", boxDownloadUrl)
// Build the steps // Build the steps
steps := []multistep.Step{} steps := []multistep.Step{
new(stepVerifyBox),
new(stepCreateVersion),
new(stepCreateProvider),
}
if p.config.BoxDownloadUrl == "" { if p.config.BoxDownloadUrl == "" {
steps = []multistep.Step{ steps = append(steps, new(stepPrepareUpload), new(stepUpload))
new(stepVerifyBox), if !p.config.NoDirectUpload {
new(stepCreateVersion), steps = append(steps, new(stepConfirmUpload))
new(stepCreateProvider),
new(stepPrepareUpload),
new(stepUpload),
new(stepReleaseVersion),
}
} else {
steps = []multistep.Step{
new(stepVerifyBox),
new(stepCreateVersion),
new(stepCreateProvider),
new(stepReleaseVersion),
} }
} }
steps = append(steps, new(stepReleaseVersion))
// Run the steps // Run the steps
p.runner = common.NewRunner(steps, p.config.PackerConfig, ui) p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)

View File

@ -24,6 +24,7 @@ type FlatConfig struct {
VagrantCloudUrl *string `mapstructure:"vagrant_cloud_url" cty:"vagrant_cloud_url" hcl:"vagrant_cloud_url"` VagrantCloudUrl *string `mapstructure:"vagrant_cloud_url" cty:"vagrant_cloud_url" hcl:"vagrant_cloud_url"`
InsecureSkipTLSVerify *bool `mapstructure:"insecure_skip_tls_verify" cty:"insecure_skip_tls_verify" hcl:"insecure_skip_tls_verify"` InsecureSkipTLSVerify *bool `mapstructure:"insecure_skip_tls_verify" cty:"insecure_skip_tls_verify" hcl:"insecure_skip_tls_verify"`
BoxDownloadUrl *string `mapstructure:"box_download_url" cty:"box_download_url" hcl:"box_download_url"` BoxDownloadUrl *string `mapstructure:"box_download_url" cty:"box_download_url" hcl:"box_download_url"`
NoDirectUpload *bool `mapstructure:"no_direct_upload" cty:"no_direct_upload" hcl:"no_direct_upload"`
} }
// FlatMapstructure returns a new FlatConfig. // FlatMapstructure returns a new FlatConfig.
@ -53,6 +54,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
"vagrant_cloud_url": &hcldec.AttrSpec{Name: "vagrant_cloud_url", Type: cty.String, Required: false}, "vagrant_cloud_url": &hcldec.AttrSpec{Name: "vagrant_cloud_url", Type: cty.String, Required: false},
"insecure_skip_tls_verify": &hcldec.AttrSpec{Name: "insecure_skip_tls_verify", Type: cty.Bool, Required: false}, "insecure_skip_tls_verify": &hcldec.AttrSpec{Name: "insecure_skip_tls_verify", Type: cty.Bool, Required: false},
"box_download_url": &hcldec.AttrSpec{Name: "box_download_url", Type: cty.String, Required: false}, "box_download_url": &hcldec.AttrSpec{Name: "box_download_url", Type: cty.String, Required: false},
"no_direct_upload": &hcldec.AttrSpec{Name: "no_direct_upload", Type: cty.Bool, Required: false},
} }
return s return s
} }

View File

@ -17,6 +17,13 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
type stubResponse struct {
Path string
Method string
Response string
StatusCode int
}
type tarFiles []struct { type tarFiles []struct {
Name, Body string Name, Body string
} }
@ -46,6 +53,36 @@ func testNoAccessTokenProvidedConfig() map[string]interface{} {
} }
} }
func newStackServer(stack []stubResponse) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
if len(stack) < 1 {
rw.Header().Add("Error", fmt.Sprintf("Request stack is empty - Method: %s Path: %s", req.Method, req.URL.Path))
http.Error(rw, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
match := stack[0]
stack = stack[1:]
if match.Method != "" && req.Method != match.Method {
rw.Header().Add("Error", fmt.Sprintf("Request %s != %s", match.Method, req.Method))
http.Error(rw, fmt.Sprintf("Request %s != %s", match.Method, req.Method), http.StatusInternalServerError)
return
}
if match.Path != "" && match.Path != req.URL.Path {
rw.Header().Add("Error", fmt.Sprintf("Request %s != %s", match.Path, req.URL.Path))
http.Error(rw, fmt.Sprintf("Request %s != %s", match.Path, req.URL.Path), http.StatusInternalServerError)
return
}
rw.Header().Add("Complete", fmt.Sprintf("Method: %s Path: %s", match.Method, match.Path))
rw.WriteHeader(match.StatusCode)
if match.Response != "" {
_, err := rw.Write([]byte(match.Response))
if err != nil {
panic("failed to write response: " + err.Error())
}
}
}))
}
func newSecureServer(token string, handler http.HandlerFunc) *httptest.Server { func newSecureServer(token string, handler http.HandlerFunc) *httptest.Server {
token = fmt.Sprintf("Bearer %s", token) token = fmt.Sprintf("Bearer %s", token)
return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
@ -229,6 +266,156 @@ func TestPostProcessor_PostProcess_checkArtifactFileIsBox(t *testing.T) {
} }
} }
func TestPostProcessor_PostProcess_uploadsAndReleases(t *testing.T) {
files := tarFiles{
{"foo.txt", "This is a foo file"},
{"bar.txt", "This is a bar file"},
{"metadata.json", `{"provider": "virtualbox"}`},
}
boxfile, err := createBox(files)
if err != nil {
t.Fatalf("%s", err)
}
defer os.Remove(boxfile.Name())
artifact := &packer.MockArtifact{
BuilderIdValue: "mitchellh.post-processor.vagrant",
FilesValue: []string{boxfile.Name()},
}
s := newStackServer([]stubResponse{stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"}})
defer s.Close()
stack := []stubResponse{
stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload", Response: `{"upload_path": "` + s.URL + `/box-upload-path"}`},
stubResponse{StatusCode: 200, Method: "PUT", Path: "/box/hashicorp/precise64/version/0.5/release"},
}
server := newStackServer(stack)
defer server.Close()
config := testGoodConfig()
config["vagrant_cloud_url"] = server.URL
config["no_direct_upload"] = true
var p PostProcessor
err = p.Configure(config)
if err != nil {
t.Fatalf("err: %s", err)
}
_, _, _, err = p.PostProcess(context.Background(), testUi(), artifact)
if err != nil {
t.Fatalf("err: %s", err)
}
}
func TestPostProcessor_PostProcess_uploadsAndNoRelease(t *testing.T) {
files := tarFiles{
{"foo.txt", "This is a foo file"},
{"bar.txt", "This is a bar file"},
{"metadata.json", `{"provider": "virtualbox"}`},
}
boxfile, err := createBox(files)
if err != nil {
t.Fatalf("%s", err)
}
defer os.Remove(boxfile.Name())
artifact := &packer.MockArtifact{
BuilderIdValue: "mitchellh.post-processor.vagrant",
FilesValue: []string{boxfile.Name()},
}
s := newStackServer([]stubResponse{stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"}})
defer s.Close()
stack := []stubResponse{
stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload", Response: `{"upload_path": "` + s.URL + `/box-upload-path"}`},
}
server := newStackServer(stack)
defer server.Close()
config := testGoodConfig()
config["vagrant_cloud_url"] = server.URL
config["no_direct_upload"] = true
config["no_release"] = true
var p PostProcessor
err = p.Configure(config)
if err != nil {
t.Fatalf("err: %s", err)
}
_, _, _, err = p.PostProcess(context.Background(), testUi(), artifact)
if err != nil {
t.Fatalf("err: %s", err)
}
}
func TestPostProcessor_PostProcess_uploadsDirectAndReleases(t *testing.T) {
files := tarFiles{
{"foo.txt", "This is a foo file"},
{"bar.txt", "This is a bar file"},
{"metadata.json", `{"provider": "virtualbox"}`},
}
boxfile, err := createBox(files)
if err != nil {
t.Fatalf("%s", err)
}
defer os.Remove(boxfile.Name())
artifact := &packer.MockArtifact{
BuilderIdValue: "mitchellh.post-processor.vagrant",
FilesValue: []string{boxfile.Name()},
}
s := newStackServer(
[]stubResponse{
stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"},
},
)
defer s.Close()
stack := []stubResponse{
stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload/direct"},
stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-complete"},
stubResponse{StatusCode: 200, Method: "PUT", Path: "/box/hashicorp/precise64/version/0.5/release"},
}
server := newStackServer(stack)
defer server.Close()
config := testGoodConfig()
config["vagrant_cloud_url"] = server.URL
// Set response here so we have API server URL available
stack[4].Response = `{"upload_path": "` + s.URL + `/box-upload-path", "callback": "` + server.URL + `/box-upload-complete"}`
var p PostProcessor
err = p.Configure(config)
if err != nil {
t.Fatalf("err: %s", err)
}
_, _, _, err = p.PostProcess(context.Background(), testUi(), artifact)
if err != nil {
t.Fatalf("err: %s", err)
}
}
func testUi() *packer.BasicUi { func testUi() *packer.BasicUi {
return &packer.BasicUi{ return &packer.BasicUi{
Reader: new(bytes.Buffer), Reader: new(bytes.Buffer),

View File

@ -0,0 +1,43 @@
package vagrantcloud
import (
"context"
"fmt"
"github.com/hashicorp/packer/helper/multistep"
"github.com/hashicorp/packer/packer"
)
type stepConfirmUpload struct {
}
func (s *stepConfirmUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
client := state.Get("client").(*VagrantCloudClient)
ui := state.Get("ui").(packer.Ui)
upload := state.Get("upload").(*Upload)
url := upload.CallbackPath
ui.Say("Confirming direct box upload completion")
resp, err := client.Callback(url)
if err != nil || resp.StatusCode != 200 {
if resp == nil || resp.Body == nil {
state.Put("error", "No response from server.")
} else {
cloudErrors := &VagrantCloudErrors{}
err = decodeBody(resp, cloudErrors)
if err != nil {
ui.Error(fmt.Sprintf("error decoding error response: %s", err))
}
state.Put("error", fmt.Errorf("Error preparing upload: %s", cloudErrors.FormatErrors()))
}
return multistep.ActionHalt
}
return multistep.ActionContinue
}
func (s *stepConfirmUpload) Cleanup(state multistep.StateBag) {
// No cleanup
}

View File

@ -10,6 +10,7 @@ import (
type Upload struct { type Upload struct {
UploadPath string `json:"upload_path"` UploadPath string `json:"upload_path"`
CallbackPath string `json:"callback"`
} }
type stepPrepareUpload struct { type stepPrepareUpload struct {
@ -17,6 +18,7 @@ type stepPrepareUpload struct {
func (s *stepPrepareUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { func (s *stepPrepareUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
client := state.Get("client").(*VagrantCloudClient) client := state.Get("client").(*VagrantCloudClient)
config := state.Get("config").(*Config)
ui := state.Get("ui").(packer.Ui) ui := state.Get("ui").(packer.Ui)
box := state.Get("box").(*Box) box := state.Get("box").(*Box)
version := state.Get("version").(*Version) version := state.Get("version").(*Version)
@ -24,6 +26,9 @@ func (s *stepPrepareUpload) Run(ctx context.Context, state multistep.StateBag) m
artifactFilePath := state.Get("artifactFilePath").(string) artifactFilePath := state.Get("artifactFilePath").(string)
path := fmt.Sprintf("box/%s/version/%v/provider/%s/upload", box.Tag, version.Version, provider.Name) path := fmt.Sprintf("box/%s/version/%v/provider/%s/upload", box.Tag, version.Version, provider.Name)
if !config.NoDirectUpload {
path = path + "/direct"
}
upload := &Upload{} upload := &Upload{}
ui.Say(fmt.Sprintf("Preparing upload of box: %s", artifactFilePath)) ui.Say(fmt.Sprintf("Preparing upload of box: %s", artifactFilePath))

View File

@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"log" "log"
"net/http"
"time" "time"
"github.com/hashicorp/packer/common/retry" "github.com/hashicorp/packer/common/retry"
@ -16,6 +17,7 @@ type stepUpload struct {
func (s *stepUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { func (s *stepUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
client := state.Get("client").(*VagrantCloudClient) client := state.Get("client").(*VagrantCloudClient)
config := state.Get("config").(*Config)
ui := state.Get("ui").(packer.Ui) ui := state.Get("ui").(packer.Ui)
upload := state.Get("upload").(*Upload) upload := state.Get("upload").(*Upload)
artifactFilePath := state.Get("artifactFilePath").(string) artifactFilePath := state.Get("artifactFilePath").(string)
@ -32,7 +34,14 @@ func (s *stepUpload) Run(ctx context.Context, state multistep.StateBag) multiste
}.Run(ctx, func(ctx context.Context) error { }.Run(ctx, func(ctx context.Context) error {
ui.Message(fmt.Sprintf("Uploading box")) ui.Message(fmt.Sprintf("Uploading box"))
resp, err := client.Upload(artifactFilePath, url) var err error
var resp *http.Response
if config.NoDirectUpload {
resp, err = client.Upload(artifactFilePath, url)
} else {
resp, err = client.DirectUpload(artifactFilePath, url)
}
if err != nil { if err != nil {
ui.Message(fmt.Sprintf( ui.Message(fmt.Sprintf(
"Error uploading box! Will retry in 10 seconds. Error: %s", err)) "Error uploading box! Will retry in 10 seconds. Error: %s", err))

View File

@ -114,6 +114,9 @@ on Vagrant Cloud, as well as authentication and version information.
- `Provider`: The Vagrant provider the box is for - `Provider`: The Vagrant provider the box is for
- `ArtifactId`: The ID of the input artifact. - `ArtifactId`: The ID of the input artifact.
- `no_direct_upload` (boolean) - When true, upload the box artifact through
Vagrant Cloud instead of directly to the backend storage.
## Use with the Vagrant Post-Processor ## Use with the Vagrant Post-Processor
An example configuration is shown below. Note the use of the nested array that An example configuration is shown below. Note the use of the nested array that