iSharkFly-Docs
/
packer-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for uploading directly to storage on Vagrant Cloud (#10193) 

Vagrant Cloud provides support for uploading directly to the backend
storage instead of streaming through Vagrant Cloud. This adds support
for direct to storage uploads and sets it as the default upload method.
A new option has been added to disable this behavior and revert back
to streaming upload via Vagrant Cloud (`no_direct_upload`).

This default for uploading directly to the backend storage also matches
up with changes being added to Vagrant proper for box upload behavior:
hashicorp/vagrant#11916
This commit is contained in:
Chris Roberts 2020-11-05 17:01:55 -08:00 committed by GitHub
parent 7a1680df97
commit d8222b1656
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 308 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
post-processor/vagrant-cloud/client.go
View File

@ -156,6 +156,52 @@ func (v *VagrantCloudClient) Upload(path string, url string) (*http.Response, er
return resp, err
}
func (v *VagrantCloudClient) DirectUpload(path string, url string) (*http.Response, error) {
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("Error opening file for upload: %s", err)
}
defer file.Close()
fi, err := file.Stat()
if err != nil {
return nil, fmt.Errorf("Error stating file for upload: %s", err)
}
request, err := http.NewRequest("PUT", url, file)
if err != nil {
return nil, fmt.Errorf("Error preparing upload request: %s", err)
}
log.Printf("Post-Processor Vagrant Cloud API Direct Upload: %s %s", path, url)
request.ContentLength = fi.Size()
resp, err := v.client.Do(request)
log.Printf("Post-Processor Vagrant Cloud Direct Upload Response: \n\n%+v", resp)
return resp, err
}
func (v *VagrantCloudClient) Callback(url string) (*http.Response, error) {
request, err := v.newRequest("PUT", url, nil)
if err != nil {
return nil, fmt.Errorf("Error preparing callback request: %s", err)
}
log.Printf("Post-Processor Vagrant Cloud API Direct Upload Callback: %s", url)
resp, err := v.client.Do(request)
log.Printf("Post-Processor Vagrant Cloud Direct Upload Callback Response: \n\n%+v", resp)
return resp, err
}
func (v *VagrantCloudClient) Post(path string, body interface{}) (*http.Response, error) {
reqUrl := fmt.Sprintf("%s/%s", v.BaseURL, path)

28
post-processor/vagrant-cloud/post-processor.go
View File

@ -45,8 +45,8 @@ type Config struct {
AccessToken string `mapstructure:"access_token"`
VagrantCloudUrl string `mapstructure:"vagrant_cloud_url"`
InsecureSkipTLSVerify bool `mapstructure:"insecure_skip_tls_verify"`
BoxDownloadUrl string `mapstructure:"box_download_url"`
BoxDownloadUrl string `mapstructure:"box_download_url"`
NoDirectUpload bool `mapstructure:"no_direct_upload"`
ctx interpolate.Context
}
@ -181,24 +181,18 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
state.Put("boxDownloadUrl", boxDownloadUrl)
// Build the steps
steps := []multistep.Step{}
steps := []multistep.Step{
new(stepVerifyBox),
new(stepCreateVersion),
new(stepCreateProvider),
}
if p.config.BoxDownloadUrl == "" {
steps = []multistep.Step{
new(stepVerifyBox),
new(stepCreateVersion),
new(stepCreateProvider),
new(stepPrepareUpload),
new(stepUpload),
new(stepReleaseVersion),
}
} else {
steps = []multistep.Step{
new(stepVerifyBox),
new(stepCreateVersion),
new(stepCreateProvider),
new(stepReleaseVersion),
steps = append(steps, new(stepPrepareUpload), new(stepUpload))
if !p.config.NoDirectUpload {
steps = append(steps, new(stepConfirmUpload))
}
}
steps = append(steps, new(stepReleaseVersion))
// Run the steps
p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)

2
post-processor/vagrant-cloud/post-processor.hcl2spec.go
View File

@ -24,6 +24,7 @@ type FlatConfig struct {
VagrantCloudUrl *string `mapstructure:"vagrant_cloud_url" cty:"vagrant_cloud_url" hcl:"vagrant_cloud_url"`
InsecureSkipTLSVerify *bool `mapstructure:"insecure_skip_tls_verify" cty:"insecure_skip_tls_verify" hcl:"insecure_skip_tls_verify"`
BoxDownloadUrl *string `mapstructure:"box_download_url" cty:"box_download_url" hcl:"box_download_url"`
NoDirectUpload *bool `mapstructure:"no_direct_upload" cty:"no_direct_upload" hcl:"no_direct_upload"`
}
// FlatMapstructure returns a new FlatConfig.
@ -53,6 +54,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
"vagrant_cloud_url": &hcldec.AttrSpec{Name: "vagrant_cloud_url", Type: cty.String, Required: false},
"insecure_skip_tls_verify": &hcldec.AttrSpec{Name: "insecure_skip_tls_verify", Type: cty.Bool, Required: false},
"box_download_url": &hcldec.AttrSpec{Name: "box_download_url", Type: cty.String, Required: false},
"no_direct_upload": &hcldec.AttrSpec{Name: "no_direct_upload", Type: cty.Bool, Required: false},
}
return s
}

187
post-processor/vagrant-cloud/post-processor_test.go
View File

@ -17,6 +17,13 @@ import (
"github.com/stretchr/testify/assert"
)
type stubResponse struct {
Path string
Method string
Response string
StatusCode int
}
type tarFiles []struct {
Name, Body string
}
@ -46,6 +53,36 @@ func testNoAccessTokenProvidedConfig() map[string]interface{} {
}
}
func newStackServer(stack []stubResponse) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
if len(stack) < 1 {
rw.Header().Add("Error", fmt.Sprintf("Request stack is empty - Method: %s Path: %s", req.Method, req.URL.Path))
http.Error(rw, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
match := stack[0]
stack = stack[1:]
if match.Method != "" && req.Method != match.Method {
rw.Header().Add("Error", fmt.Sprintf("Request %s != %s", match.Method, req.Method))
http.Error(rw, fmt.Sprintf("Request %s != %s", match.Method, req.Method), http.StatusInternalServerError)
return
}
if match.Path != "" && match.Path != req.URL.Path {
rw.Header().Add("Error", fmt.Sprintf("Request %s != %s", match.Path, req.URL.Path))
http.Error(rw, fmt.Sprintf("Request %s != %s", match.Path, req.URL.Path), http.StatusInternalServerError)
return
}
rw.Header().Add("Complete", fmt.Sprintf("Method: %s Path: %s", match.Method, match.Path))
rw.WriteHeader(match.StatusCode)
if match.Response != "" {
_, err := rw.Write([]byte(match.Response))
if err != nil {
panic("failed to write response: " + err.Error())
}
}
}))
}
func newSecureServer(token string, handler http.HandlerFunc) *httptest.Server {
token = fmt.Sprintf("Bearer %s", token)
return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
@ -229,6 +266,156 @@ func TestPostProcessor_PostProcess_checkArtifactFileIsBox(t *testing.T) {
}
}
func TestPostProcessor_PostProcess_uploadsAndReleases(t *testing.T) {
files := tarFiles{
{"foo.txt", "This is a foo file"},
{"bar.txt", "This is a bar file"},
{"metadata.json", `{"provider": "virtualbox"}`},
}
boxfile, err := createBox(files)
if err != nil {
t.Fatalf("%s", err)
}
defer os.Remove(boxfile.Name())
artifact := &packer.MockArtifact{
BuilderIdValue: "mitchellh.post-processor.vagrant",
FilesValue: []string{boxfile.Name()},
}
s := newStackServer([]stubResponse{stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"}})
defer s.Close()
stack := []stubResponse{
stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload", Response: `{"upload_path": "` + s.URL + `/box-upload-path"}`},
stubResponse{StatusCode: 200, Method: "PUT", Path: "/box/hashicorp/precise64/version/0.5/release"},
}
server := newStackServer(stack)
defer server.Close()
config := testGoodConfig()
config["vagrant_cloud_url"] = server.URL
config["no_direct_upload"] = true
var p PostProcessor
err = p.Configure(config)
if err != nil {
t.Fatalf("err: %s", err)
}
_, _, _, err = p.PostProcess(context.Background(), testUi(), artifact)
if err != nil {
t.Fatalf("err: %s", err)
}
}
func TestPostProcessor_PostProcess_uploadsAndNoRelease(t *testing.T) {
files := tarFiles{
{"foo.txt", "This is a foo file"},
{"bar.txt", "This is a bar file"},
{"metadata.json", `{"provider": "virtualbox"}`},
}
boxfile, err := createBox(files)
if err != nil {
t.Fatalf("%s", err)
}
defer os.Remove(boxfile.Name())
artifact := &packer.MockArtifact{
BuilderIdValue: "mitchellh.post-processor.vagrant",
FilesValue: []string{boxfile.Name()},
}
s := newStackServer([]stubResponse{stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"}})
defer s.Close()
stack := []stubResponse{
stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload", Response: `{"upload_path": "` + s.URL + `/box-upload-path"}`},
}
server := newStackServer(stack)
defer server.Close()
config := testGoodConfig()
config["vagrant_cloud_url"] = server.URL
config["no_direct_upload"] = true
config["no_release"] = true
var p PostProcessor
err = p.Configure(config)
if err != nil {
t.Fatalf("err: %s", err)
}
_, _, _, err = p.PostProcess(context.Background(), testUi(), artifact)
if err != nil {
t.Fatalf("err: %s", err)
}
}
func TestPostProcessor_PostProcess_uploadsDirectAndReleases(t *testing.T) {
files := tarFiles{
{"foo.txt", "This is a foo file"},
{"bar.txt", "This is a bar file"},
{"metadata.json", `{"provider": "virtualbox"}`},
}
boxfile, err := createBox(files)
if err != nil {
t.Fatalf("%s", err)
}
defer os.Remove(boxfile.Name())
artifact := &packer.MockArtifact{
BuilderIdValue: "mitchellh.post-processor.vagrant",
FilesValue: []string{boxfile.Name()},
}
s := newStackServer(
[]stubResponse{
stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-path"},
},
)
defer s.Close()
stack := []stubResponse{
stubResponse{StatusCode: 200, Method: "GET", Path: "/authenticate"},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64", Response: `{"tag": "hashicorp/precise64"}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/versions", Response: `{}`},
stubResponse{StatusCode: 200, Method: "POST", Path: "/box/hashicorp/precise64/version/0.5/providers", Response: `{}`},
stubResponse{StatusCode: 200, Method: "GET", Path: "/box/hashicorp/precise64/version/0.5/provider/id/upload/direct"},
stubResponse{StatusCode: 200, Method: "PUT", Path: "/box-upload-complete"},
stubResponse{StatusCode: 200, Method: "PUT", Path: "/box/hashicorp/precise64/version/0.5/release"},
}
server := newStackServer(stack)
defer server.Close()
config := testGoodConfig()
config["vagrant_cloud_url"] = server.URL
// Set response here so we have API server URL available
stack[4].Response = `{"upload_path": "` + s.URL + `/box-upload-path", "callback": "` + server.URL + `/box-upload-complete"}`
var p PostProcessor
err = p.Configure(config)
if err != nil {
t.Fatalf("err: %s", err)
}
_, _, _, err = p.PostProcess(context.Background(), testUi(), artifact)
if err != nil {
t.Fatalf("err: %s", err)
}
}
func testUi() *packer.BasicUi {
return &packer.BasicUi{
Reader: new(bytes.Buffer),

43
post-processor/vagrant-cloud/step_confirm_upload.go Normal file
View File

@ -0,0 +1,43 @@
package vagrantcloud
import (
"context"
"fmt"
"github.com/hashicorp/packer/helper/multistep"
"github.com/hashicorp/packer/packer"
)
type stepConfirmUpload struct {
}
func (s *stepConfirmUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
client := state.Get("client").(*VagrantCloudClient)
ui := state.Get("ui").(packer.Ui)
upload := state.Get("upload").(*Upload)
url := upload.CallbackPath
ui.Say("Confirming direct box upload completion")
resp, err := client.Callback(url)
if err != nil || resp.StatusCode != 200 {
if resp == nil || resp.Body == nil {
state.Put("error", "No response from server.")
} else {
cloudErrors := &VagrantCloudErrors{}
err = decodeBody(resp, cloudErrors)
if err != nil {
ui.Error(fmt.Sprintf("error decoding error response: %s", err))
}
state.Put("error", fmt.Errorf("Error preparing upload: %s", cloudErrors.FormatErrors()))
}
return multistep.ActionHalt
}
return multistep.ActionContinue
}
func (s *stepConfirmUpload) Cleanup(state multistep.StateBag) {
// No cleanup
}

7
post-processor/vagrant-cloud/step_prepare_upload.go
View File

@ -9,7 +9,8 @@ import (
)
type Upload struct {
UploadPath string `json:"upload_path"`
UploadPath string `json:"upload_path"`
CallbackPath string `json:"callback"`
}
type stepPrepareUpload struct {
@ -17,6 +18,7 @@ type stepPrepareUpload struct {
func (s *stepPrepareUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
client := state.Get("client").(*VagrantCloudClient)
config := state.Get("config").(*Config)
ui := state.Get("ui").(packer.Ui)
box := state.Get("box").(*Box)
version := state.Get("version").(*Version)
@ -24,6 +26,9 @@ func (s *stepPrepareUpload) Run(ctx context.Context, state multistep.StateBag) m
artifactFilePath := state.Get("artifactFilePath").(string)
path := fmt.Sprintf("box/%s/version/%v/provider/%s/upload", box.Tag, version.Version, provider.Name)
if !config.NoDirectUpload {
path = path + "/direct"
}
upload := &Upload{}
ui.Say(fmt.Sprintf("Preparing upload of box: %s", artifactFilePath))

11
post-processor/vagrant-cloud/step_upload.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log"
"net/http"
"time"
"github.com/hashicorp/packer/common/retry"
@ -16,6 +17,7 @@ type stepUpload struct {
func (s *stepUpload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
client := state.Get("client").(*VagrantCloudClient)
config := state.Get("config").(*Config)
ui := state.Get("ui").(packer.Ui)
upload := state.Get("upload").(*Upload)
artifactFilePath := state.Get("artifactFilePath").(string)
@ -32,7 +34,14 @@ func (s *stepUpload) Run(ctx context.Context, state multistep.StateBag) multiste
}.Run(ctx, func(ctx context.Context) error {
ui.Message(fmt.Sprintf("Uploading box"))
resp, err := client.Upload(artifactFilePath, url)
var err error
var resp *http.Response
if config.NoDirectUpload {
resp, err = client.Upload(artifactFilePath, url)
} else {
resp, err = client.DirectUpload(artifactFilePath, url)
}
if err != nil {
ui.Message(fmt.Sprintf(
"Error uploading box! Will retry in 10 seconds. Error: %s", err))

3
website/pages/docs/post-processors/vagrant-cloud.mdx
View File

@ -114,6 +114,9 @@ on Vagrant Cloud, as well as authentication and version information.
- `Provider`: The Vagrant provider the box is for
- `ArtifactId`: The ID of the input artifact.
- `no_direct_upload` (boolean) - When true, upload the box artifact through
Vagrant Cloud instead of directly to the backend storage.
## Use with the Vagrant Post-Processor
An example configuration is shown below. Note the use of the nested array that