iSharkFly-Docs
/
packer-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/Hashicorp/packer

This commit is contained in:
Megan Marsh 2018-09-10 12:00:21 -07:00
parent 928ab4d07a 551c19e1b7
commit eb847d5bad
24 changed files with 465 additions and 558 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
command/fix_test.go
View File

@ -2,18 +2,39 @@ package command
import (
"path/filepath"
"strings"
"testing"
"github.com/hashicorp/packer/packer"
"github.com/stretchr/testify/assert"
)
func TestFix(t *testing.T) {
s := &strings.Builder{}
ui := &packer.BasicUi{
Writer: s,
}
c := &FixCommand{
Meta: testMeta(t),
}
c.Ui = ui
args := []string{filepath.Join(testFixture("fix"), "template.json")}
if code := c.Run(args); code != 0 {
fatalCommand(t, c.Meta)
}
expected := `{
"builders": [
{
"type": "dummy"
}
],
"push": {
"name": "foo/bar"
}
}`
assert.Equal(t, expected, strings.TrimSpace(s.String()))
}
func TestFix_invalidTemplate(t *testing.T) {

6
common/config.go
View File

@ -9,6 +9,8 @@ import (
"runtime"
"strings"
"time"
"github.com/hashicorp/packer/packer"
)
// PackerKeyEnv is used to specify the key interval (delay) between keystrokes
@ -41,7 +43,7 @@ func SupportedProtocol(u *url.URL) bool {
// build a dummy NewDownloadClient since this is the only place that valid
// protocols are actually exposed.
cli := NewDownloadClient(&DownloadConfig{}, nil)
cli := NewDownloadClient(&DownloadConfig{}, new(packer.NoopUi))
// Iterate through each downloader to see if a protocol was found.
ok := false
@ -173,7 +175,7 @@ func FileExistsLocally(original string) bool {
// First create a dummy downloader so we can figure out which
// protocol to use.
cli := NewDownloadClient(&DownloadConfig{}, nil)
cli := NewDownloadClient(&DownloadConfig{}, new(packer.NoopUi))
d, ok := cli.config.DownloaderMap[u.Scheme]
if !ok {
return false

164
common/download.go
View File

@ -10,19 +10,17 @@ import (
"errors"
"fmt"
"hash"
"io"
"log"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"runtime"
"strings"
)
// imports related to each Downloader implementation
import (
"io"
"net/http"
"path/filepath"
"github.com/hashicorp/packer/packer"
)
// DownloadConfig is the configuration given to instantiate a new
@ -58,8 +56,7 @@ type DownloadConfig struct {
// A DownloadClient helps download, verify checksums, etc.
type DownloadClient struct {
config *DownloadConfig
downloader Downloader
config *DownloadConfig
}
// HashForType returns the Hash implementation for the given string
@ -81,33 +78,24 @@ func HashForType(t string) hash.Hash {
// NewDownloadClient returns a new DownloadClient for the given
// configuration.
func NewDownloadClient(c *DownloadConfig, bar ProgressBar) *DownloadClient {
const mtu = 1500 /* ethernet */ - 20 /* ipv4 */ - 20 /* tcp */
// If bar is nil, then use a dummy progress bar that doesn't do anything
if bar == nil {
bar = GetDummyProgressBar()
}
func NewDownloadClient(c *DownloadConfig, ui packer.Ui) *DownloadClient {
// Create downloader map if it hasn't been specified already.
if c.DownloaderMap == nil {
c.DownloaderMap = map[string]Downloader{
"file": &FileDownloader{progress: bar, bufferSize: nil},
"http": &HTTPDownloader{progress: bar, userAgent: c.UserAgent},
"https": &HTTPDownloader{progress: bar, userAgent: c.UserAgent},
"smb": &SMBDownloader{progress: bar, bufferSize: nil},
"file": &FileDownloader{Ui: ui, bufferSize: nil},
"http": &HTTPDownloader{Ui: ui, userAgent: c.UserAgent},
"https": &HTTPDownloader{Ui: ui, userAgent: c.UserAgent},
"smb": &SMBDownloader{Ui: ui, bufferSize: nil},
}
}
return &DownloadClient{config: c}
}
// A downloader implements the ability to transfer a file, and cancel or resume
// it.
// Downloader defines what capabilities a downloader should have.
type Downloader interface {
Resume()
Cancel()
Progress() uint64
Total() uint64
ProgressBar() packer.ProgressBar
}
// A LocalDownloader is responsible for converting a uri to a local path
@ -150,17 +138,17 @@ func (d *DownloadClient) Get() (string, error) {
var finalPath string
var ok bool
d.downloader, ok = d.config.DownloaderMap[u.Scheme]
downloader, ok := d.config.DownloaderMap[u.Scheme]
if !ok {
return "", fmt.Errorf("No downloader for scheme: %s", u.Scheme)
}
remote, ok := d.downloader.(RemoteDownloader)
remote, ok := downloader.(RemoteDownloader)
if !ok {
return "", fmt.Errorf("Unable to treat uri scheme %s as a Downloader. : %T", u.Scheme, d.downloader)
return "", fmt.Errorf("Unable to treat uri scheme %s as a Downloader. : %T", u.Scheme, downloader)
}
local, ok := d.downloader.(LocalDownloader)
local, ok := downloader.(LocalDownloader)
if !ok && !d.config.CopyFile {
d.config.CopyFile = true
}
@ -233,11 +221,9 @@ func (d *DownloadClient) VerifyChecksum(path string) (bool, error) {
// HTTPDownloader is an implementation of Downloader that downloads
// files over HTTP.
type HTTPDownloader struct {
current uint64
total uint64
userAgent string
progress ProgressBar
Ui packer.Ui
}
func (d *HTTPDownloader) Cancel() {
@ -256,8 +242,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error {
return err
}
// Reset our progress
d.current = 0
var current int64
// Make the request. We first make a HEAD request so we can check
// if the server supports range queries. If the server/URL doesn't
@ -301,7 +286,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error {
if _, err = dst.Seek(0, os.SEEK_END); err == nil {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size()))
d.current = uint64(fi.Size())
current = fi.Size()
}
}
}
@ -328,23 +313,22 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error {
return fmt.Errorf("HTTP error: %s", err.Error())
}
d.total = d.current + uint64(resp.ContentLength)
total := current + resp.ContentLength
bar := d.progress
bar.SetTotal64(int64(d.total))
progressBar := bar.Start()
progressBar.Set64(int64(d.current))
bar := d.ProgressBar()
bar.Start(total)
defer bar.Finish()
bar.Add(current)
body := bar.NewProxyReader(resp.Body)
var buffer [4096]byte
for {
n, err := resp.Body.Read(buffer[:])
n, err := body.Read(buffer[:])
if err != nil && err != io.EOF {
return err
}
d.current += uint64(n)
progressBar.Set64(int64(d.current))
if _, werr := dst.Write(buffer[:n]); werr != nil {
return werr
}
@ -353,36 +337,16 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error {
break
}
}
progressBar.Finish()
return nil
}
func (d *HTTPDownloader) Progress() uint64 {
return d.current
}
func (d *HTTPDownloader) Total() uint64 {
return d.total
}
// FileDownloader is an implementation of Downloader that downloads
// files using the regular filesystem.
type FileDownloader struct {
bufferSize *uint
active bool
current uint64
total uint64
progress ProgressBar
}
func (d *FileDownloader) Progress() uint64 {
return d.current
}
func (d *FileDownloader) Total() uint64 {
return d.total
active bool
Ui packer.Ui
}
func (d *FileDownloader) Cancel() {
@ -449,7 +413,6 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error {
}
/* download the file using the operating system's facilities */
d.current = 0
d.active = true
f, err := os.Open(realpath)
@ -463,44 +426,37 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error {
if err != nil {
return err
}
d.total = uint64(fi.Size())
bar := d.progress
bar.SetTotal64(int64(d.total))
progressBar := bar.Start()
progressBar.Set64(int64(d.current))
bar := d.ProgressBar()
bar.Start(fi.Size())
defer bar.Finish()
fProxy := bar.NewProxyReader(f)
// no bufferSize specified, so copy synchronously.
if d.bufferSize == nil {
var n int64
n, err = io.Copy(dst, f)
_, err = io.Copy(dst, fProxy)
d.active = false
d.current += uint64(n)
progressBar.Set64(int64(d.current))
// use a goro in case someone else wants to enable cancel/resume
} else {
errch := make(chan error)
go func(d *FileDownloader, r io.Reader, w io.Writer, e chan error) {
for d.active {
n, err := io.CopyN(w, r, int64(*d.bufferSize))
_, err := io.CopyN(w, r, int64(*d.bufferSize))
if err != nil {
break
}
d.current += uint64(n)
progressBar.Set64(int64(d.current))
}
d.active = false
e <- err
}(d, f, dst, errch)
}(d, fProxy, dst, errch)
// ...and we spin until it's done
err = <-errch
}
progressBar.Finish()
f.Close()
return err
}
@ -509,19 +465,8 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error {
type SMBDownloader struct {
bufferSize *uint
active bool
current uint64
total uint64
progress ProgressBar
}
func (d *SMBDownloader) Progress() uint64 {
return d.current
}
func (d *SMBDownloader) Total() uint64 {
return d.total
active bool
Ui packer.Ui
}
func (d *SMBDownloader) Cancel() {
@ -570,7 +515,6 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error {
}
/* Open up the "\\"-prefixed path using the Windows filesystem */
d.current = 0
d.active = true
f, err := os.Open(realpath)
@ -584,43 +528,39 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error {
if err != nil {
return err
}
d.total = uint64(fi.Size())
bar := d.progress
bar.SetTotal64(int64(d.total))
progressBar := bar.Start()
progressBar.Set64(int64(d.current))
bar := d.ProgressBar()
bar.Start(fi.Size())
defer bar.Finish()
fProxy := bar.NewProxyReader(f)
// no bufferSize specified, so copy synchronously.
if d.bufferSize == nil {
var n int64
n, err = io.Copy(dst, f)
_, err = io.Copy(dst, fProxy)
d.active = false
d.current += uint64(n)
progressBar.Set64(int64(d.current))
// use a goro in case someone else wants to enable cancel/resume
} else {
errch := make(chan error)
go func(d *SMBDownloader, r io.Reader, w io.Writer, e chan error) {
for d.active {
n, err := io.CopyN(w, r, int64(*d.bufferSize))
_, err := io.CopyN(w, r, int64(*d.bufferSize))
if err != nil {
break
}
d.current += uint64(n)
progressBar.Set64(int64(d.current))
}
d.active = false
e <- err
}(d, f, dst, errch)
}(d, fProxy, dst, errch)
// ...and as usual we spin until it's done
err = <-errch
}
progressBar.Finish()
f.Close()
return err
}
func (d *HTTPDownloader) ProgressBar() packer.ProgressBar { return d.Ui.ProgressBar() }
func (d *FileDownloader) ProgressBar() packer.ProgressBar { return d.Ui.ProgressBar() }
func (d *SMBDownloader) ProgressBar() packer.ProgressBar { return d.Ui.ProgressBar() }

24
common/download_test.go
View File

@ -12,6 +12,8 @@ import (
"runtime"
"strings"
"testing"
"github.com/hashicorp/packer/packer"
)
func TestDownloadClientVerifyChecksum(t *testing.T) {
@ -36,7 +38,7 @@ func TestDownloadClientVerifyChecksum(t *testing.T) {
Checksum: checksum,
}
d := NewDownloadClient(config, nil)
d := NewDownloadClient(config, new(packer.NoopUi))
result, err := d.VerifyChecksum(tf.Name())
if err != nil {
t.Fatalf("Verify err: %s", err)
@ -59,7 +61,7 @@ func TestDownloadClient_basic(t *testing.T) {
Url: ts.URL + "/basic.txt",
TargetPath: tf.Name(),
CopyFile: true,
}, nil)
}, new(packer.NoopUi))
path, err := client.Get()
if err != nil {
@ -95,7 +97,7 @@ func TestDownloadClient_checksumBad(t *testing.T) {
Hash: HashForType("md5"),
Checksum: checksum,
CopyFile: true,
}, nil)
}, new(packer.NoopUi))
if _, err := client.Get(); err == nil {
t.Fatal("should error")
@ -121,7 +123,7 @@ func TestDownloadClient_checksumGood(t *testing.T) {
Hash: HashForType("md5"),
Checksum: checksum,
CopyFile: true,
}, nil)
}, new(packer.NoopUi))
path, err := client.Get()
if err != nil {
@ -153,7 +155,7 @@ func TestDownloadClient_checksumNoDownload(t *testing.T) {
Hash: HashForType("md5"),
Checksum: checksum,
CopyFile: true,
}, nil)
}, new(packer.NoopUi))
path, err := client.Get()
if err != nil {
t.Fatalf("err: %s", err)
@ -183,7 +185,7 @@ func TestDownloadClient_notFound(t *testing.T) {
client := NewDownloadClient(&DownloadConfig{
Url: ts.URL + "/not-found.txt",
TargetPath: tf.Name(),
}, nil)
}, new(packer.NoopUi))
if _, err := client.Get(); err == nil {
t.Fatal("should error")
@ -211,7 +213,7 @@ func TestDownloadClient_resume(t *testing.T) {
Url: ts.URL,
TargetPath: tf.Name(),
CopyFile: true,
}, nil)
}, new(packer.NoopUi))
path, err := client.Get()
if err != nil {
@ -273,7 +275,7 @@ func TestDownloadClient_usesDefaultUserAgent(t *testing.T) {
CopyFile: true,
}
client := NewDownloadClient(config, nil)
client := NewDownloadClient(config, new(packer.NoopUi))
_, err = client.Get()
if err != nil {
t.Fatal(err)
@ -306,7 +308,7 @@ func TestDownloadClient_setsUserAgent(t *testing.T) {
CopyFile: true,
}
client := NewDownloadClient(config, nil)
client := NewDownloadClient(config, new(packer.NoopUi))
_, err = client.Get()
if err != nil {
t.Fatal(err)
@ -405,7 +407,7 @@ func TestDownloadFileUrl(t *testing.T) {
CopyFile: false,
}
client := NewDownloadClient(config, nil)
client := NewDownloadClient(config, new(packer.NoopUi))
// Verify that we fail to match the checksum
_, err = client.Get()
@ -436,7 +438,7 @@ func SimulateFileUriDownload(t *testing.T, uri string) (string, error) {
}
// go go go
client := NewDownloadClient(config, nil)
client := NewDownloadClient(config, new(packer.NoopUi))
path, err := client.Get()
// ignore any non-important checksum errors if it's not a unc path

145
common/progress.go
View File

@ -1,145 +0,0 @@
package common
import (
"fmt"
"github.com/cheggaaa/pb"
"github.com/hashicorp/packer/helper/multistep"
"github.com/hashicorp/packer/packer"
"github.com/hashicorp/packer/packer/rpc"
"log"
"reflect"
"time"
)
// This is the arrow from packer/ui.go -> TargetedUI.prefixLines
const targetedUIArrowText = "==>"
// The ProgressBar interface is used for abstracting cheggaaa's progress-
// bar, or any other progress bar. If a UI does not support a progress-
// bar, then it must return a null progress bar.
const (
DefaultProgressBarWidth = 80
)
type ProgressBar = *pb.ProgressBar
// Figure out the terminal dimensions and use it to calculate the available rendering space
func calculateProgressBarWidth(length int) int {
// If the UI's width is signed, then this is an interface that doesn't really benefit from a progress bar
if length < 0 {
log.Println("Refusing to render progress-bar for unsupported UI.")
return length
}
// Figure out the terminal width if possible
width, _, err := GetTerminalDimensions()
if err != nil {
newerr := fmt.Errorf("Unable to determine terminal dimensions: %v", err)
log.Printf("Using default width (%d) for progress-bar due to error: %s", DefaultProgressBarWidth, newerr)
return DefaultProgressBarWidth
}
// If the terminal width is smaller than the requested length, then complain
if width < length {
newerr := fmt.Errorf("Terminal width (%d) is smaller than UI message width (%d).", width, length)
log.Printf("Using default width (%d) for progress-bar due to error: %s", DefaultProgressBarWidth, newerr)
return DefaultProgressBarWidth
}
// Otherwise subtract the minimum length and return it
return width - length
}
// Get a progress bar with the default appearance
func GetDefaultProgressBar() ProgressBar {
bar := pb.New64(0)
bar.ShowPercent = true
bar.ShowCounters = true
bar.ShowSpeed = false
bar.ShowBar = true
bar.ShowTimeLeft = false
bar.ShowFinalTime = false
bar.SetUnits(pb.U_BYTES)
bar.Format("[=>-]")
bar.SetRefreshRate(5 * time.Second)
return bar
}
// Return a dummy progress bar that doesn't do anything
func GetDummyProgressBar() ProgressBar {
bar := pb.New64(0)
bar.ManualUpdate = true
return bar
}
// Given a packer.Ui, calculate the number of characters that a packer.Ui will
// prefix a message with. Then we can use this to calculate the progress bar's width.
func calculateUiPrefixLength(ui packer.Ui) int {
var recursiveCalculateUiPrefixLength func(packer.Ui, int) int
// Define a recursive closure that traverses through all the known packer.Ui types
// and aggregates the length of the message prefix from each particular type
recursiveCalculateUiPrefixLength = func(ui packer.Ui, agg int) int {
switch ui.(type) {
case *packer.ColoredUi:
// packer.ColoredUi is simply a wrapper around .Ui
u := ui.(*packer.ColoredUi)
return recursiveCalculateUiPrefixLength(u.Ui, agg)
case *packer.TargetedUI:
// A TargetedUI adds the .Target and an arrow by default
u := ui.(*packer.TargetedUI)
res := fmt.Sprintf("%s %s: ", targetedUIArrowText, u.Target)
return recursiveCalculateUiPrefixLength(u.Ui, agg+len(res))
case *packer.BasicUi:
// The standard BasicUi appends only a newline
return agg + len("\n")
// packer.rpc.Ui returns 0 here to trigger the hack described later
case *rpc.Ui:
return 0
case *packer.MachineReadableUi:
// MachineReadableUi doesn't emit anything...like at all
return 0
}
log.Printf("Calculating the message prefix length for packer.Ui type (%T) is not implemented. Using the current aggregated length of %d.", ui, agg)
return agg
}
return recursiveCalculateUiPrefixLength(ui, 0)
}
func GetPackerConfigFromStateBag(state multistep.StateBag) *PackerConfig {
config := state.Get("config")
rConfig := reflect.Indirect(reflect.ValueOf(config))
iPackerConfig := rConfig.FieldByName("PackerConfig").Interface()
packerConfig := iPackerConfig.(PackerConfig)
return &packerConfig
}
func GetProgressBar(ui packer.Ui, config *PackerConfig) ProgressBar {
// Figure out the prefix length by quering the UI
uiPrefixLength := calculateUiPrefixLength(ui)
// hack to deal with packer.rpc.Ui courtesy of @Swampdragons
if _, ok := ui.(*rpc.Ui); uiPrefixLength == 0 && config != nil && ok {
res := fmt.Sprintf("%s %s: \n", targetedUIArrowText, config.PackerBuildName)
uiPrefixLength = len(res)
}
// Now we can use the prefix length to calculate the progress bar width
width := calculateProgressBarWidth(uiPrefixLength)
log.Printf("ProgressBar: Using progress bar width: %d\n", width)
// Get a default progress bar and set some output defaults
bar := GetDefaultProgressBar()
bar.SetWidth(width)
bar.Callback = func(message string) {
ui.Message(message)
}
return bar
}

142
common/progress_test.go
View File

@ -1,142 +0,0 @@
package common
import (
"github.com/hashicorp/packer/packer"
"testing"
)
// test packer.Ui implementation to verify that progress bar is being written
type testProgressBarUi struct {
messageCalled bool
messageMessage string
}
func (u *testProgressBarUi) Say(string) {}
func (u *testProgressBarUi) Error(string) {}
func (u *testProgressBarUi) Machine(string, ...string) {}
func (u *testProgressBarUi) Ask(string) (string, error) {
return "", nil
}
func (u *testProgressBarUi) Message(message string) {
u.messageCalled = true
u.messageMessage = message
}
// ..and now let's begin our actual tests
func TestCalculateUiPrefixLength_Unknown(t *testing.T) {
ui := &testProgressBarUi{}
expected := 0
if res := calculateUiPrefixLength(ui); res != expected {
t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected)
}
}
func TestCalculateUiPrefixLength_BasicUi(t *testing.T) {
ui := &packer.BasicUi{}
expected := 1
if res := calculateUiPrefixLength(ui); res != expected {
t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected)
}
}
func TestCalculateUiPrefixLength_TargetedUI(t *testing.T) {
ui := &packer.TargetedUI{}
ui.Target = "TestTarget"
arrowText := "==>"
expected := len(arrowText + " " + ui.Target + ": ")
if res := calculateUiPrefixLength(ui); res != expected {
t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected)
}
}
func TestCalculateUiPrefixLength_TargetedUIWrappingBasicUi(t *testing.T) {
ui := &packer.TargetedUI{}
ui.Target = "TestTarget"
ui.Ui = &packer.BasicUi{}
arrowText := "==>"
expected := len(arrowText + " " + ui.Target + ": " + "\n")
if res := calculateUiPrefixLength(ui); res != expected {
t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected)
}
}
func TestCalculateUiPrefixLength_TargetedUIWrappingMachineUi(t *testing.T) {
ui := &packer.TargetedUI{}
ui.Target = "TestTarget"
ui.Ui = &packer.MachineReadableUi{}
expected := 0
if res := calculateUiPrefixLength(ui); res != expected {
t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected)
}
}
func TestDefaultProgressBar(t *testing.T) {
var callbackCalled bool
// Initialize the default progress bar
bar := GetDefaultProgressBar()
bar.Callback = func(state string) {
callbackCalled = true
t.Logf("TestDefaultProgressBar emitted %#v", state)
}
bar.SetTotal64(1)
// Set it off
progressBar := bar.Start()
progressBar.Set64(1)
// Check to see that the callback was hit
if !callbackCalled {
t.Fatalf("TestDefaultProgressBar.Callback should be called")
}
}
func TestDummyProgressBar(t *testing.T) {
var callbackCalled bool
// Initialize the dummy progress bar
bar := GetDummyProgressBar()
bar.Callback = func(state string) {
callbackCalled = true
t.Logf("TestDummyProgressBar emitted %#v", state)
}
bar.SetTotal64(1)
// Now we can go
progressBar := bar.Start()
progressBar.Set64(1)
// Check to see that the callback was hit
if callbackCalled {
t.Fatalf("TestDummyProgressBar.Callback should not be called")
}
}
func TestUiProgressBar(t *testing.T) {
ui := &testProgressBarUi{}
// Initialize the Ui progress bar
bar := GetProgressBar(ui, nil)
bar.SetTotal64(1)
// Ensure that callback has been set to something
if bar.Callback == nil {
t.Fatalf("TestUiProgressBar.Callback should be initialized")
}
// Now we can go
progressBar := bar.Start()
progressBar.Set64(1)
// Check to see that the callback was hit
if !ui.messageCalled {
t.Fatalf("TestUiProgressBar.messageCalled should be called")
}
t.Logf("TestUiProgressBar emitted %#v", ui.messageMessage)
}

14
common/step_download.go
View File

@ -63,9 +63,6 @@ func (s *StepDownload) Run(_ context.Context, state multistep.StateBag) multiste
ui.Say(fmt.Sprintf("Retrieving %s", s.Description))
// Get a progress bar from the ui so we can hand it off to the download client
bar := GetProgressBar(ui, GetPackerConfigFromStateBag(state))
// First try to use any already downloaded file
// If it fails, proceed to regular download logic
@ -99,7 +96,7 @@ func (s *StepDownload) Run(_ context.Context, state multistep.StateBag) multiste
}
downloadConfigs[i] = config
if match, _ := NewDownloadClient(config, bar).VerifyChecksum(config.TargetPath); match {
if match, _ := NewDownloadClient(config, ui).VerifyChecksum(config.TargetPath); match {
ui.Message(fmt.Sprintf("Found already downloaded, initial checksum matched, no download needed: %s", url))
finalPath = config.TargetPath
break
@ -143,11 +140,8 @@ func (s *StepDownload) download(config *DownloadConfig, state multistep.StateBag
var path string
ui := state.Get("ui").(packer.Ui)
// Get a progress bar and hand it off to the download client
bar := GetProgressBar(ui, GetPackerConfigFromStateBag(state))
// Create download client with config and progress bar
download := NewDownloadClient(config, bar)
// Create download client with config
download := NewDownloadClient(config, ui)
downloadCompleteCh := make(chan error, 1)
go func() {
@ -159,7 +153,6 @@ func (s *StepDownload) download(config *DownloadConfig, state multistep.StateBag
for {
select {
case err := <-downloadCompleteCh:
bar.Finish()
if err != nil {
return "", err, true
@ -174,7 +167,6 @@ func (s *StepDownload) download(config *DownloadConfig, state multistep.StateBag
case <-time.After(1 * time.Second):
if _, ok := state.GetOk(multistep.StateCancelled); ok {
bar.Finish()
ui.Say("Interrupt received. Cancelling download...")
return "", nil, false
}

12
fix/fixer_docker_email.go
View File

@ -5,10 +5,14 @@ import "github.com/mitchellh/mapstructure"
type FixerDockerEmail struct{}
func (FixerDockerEmail) Fix(input map[string]interface{}) (map[string]interface{}, error) {
if input["post-processors"] == nil {
return input, nil
}
// Our template type we'll use for this fixer only
type template struct {
Builders []map[string]interface{}
PostProcessors []map[string]interface{} `mapstructure:"post-processors"`
Builders []map[string]interface{}
PP `mapstructure:",squash"`
}
// Decode the input into our structure, if we can
@ -27,7 +31,9 @@ func (FixerDockerEmail) Fix(input map[string]interface{}) (map[string]interface{
}
// Go through each post-processor and delete `docker_login` if present
for _, pp := range tpl.PostProcessors {
pps := tpl.ppList()
for _, pp := range pps {
_, ok := pp["login_email"]
if !ok {
continue

20
fix/fixer_pp_manifest_filename.go
View File

@ -8,10 +8,13 @@ import (
type FixerManifestFilename struct{}
func (FixerManifestFilename) Fix(input map[string]interface{}) (map[string]interface{}, error) {
if input["post-processors"] == nil {
return input, nil
}
// Our template type we'll use for this fixer only
type template struct {
PostProcessors []interface{} `mapstructure:"post-processors"`
PP `mapstructure:",squash"`
}
// Decode the input into our structure, if we can
@ -21,20 +24,7 @@ func (FixerManifestFilename) Fix(input map[string]interface{}) (map[string]inter
}
// Go through each post-processor and get out all the complex configs
pps := make([]map[string]interface{}, 0, len(tpl.PostProcessors))
for _, rawPP := range tpl.PostProcessors {
switch pp := rawPP.(type) {
case string:
case map[string]interface{}:
pps = append(pps, pp)
case []interface{}:
for _, innerRawPP := range pp {
if innerPP, ok := innerRawPP.(map[string]interface{}); ok {
pps = append(pps, innerPP)
}
}
}
}
pps := tpl.ppList()
for _, pp := range pps {
ppTypeRaw, ok := pp["type"]

11
fix/fixer_pp_manifest_filename_test.go
View File

@ -1,8 +1,9 @@
package fix
import (
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
func TestFixerManifestPPFilename_Impl(t *testing.T) {
@ -43,11 +44,7 @@ func TestFixerManifestPPFilename_Fix(t *testing.T) {
}
output, err := f.Fix(input)
if err != nil {
t.Fatalf("err: %s", err)
}
assert.NoError(t, err)
if !reflect.DeepEqual(output, expected) {
t.Fatalf("unexpected: %#v\nexpected: %#v\n", output, expected)
}
assert.Equal(t, expected, output)
}

26
fix/fixer_pp_vagrant_override.go
View File

@ -1,8 +1,6 @@
package fix
import (
"github.com/mitchellh/mapstructure"
)
import "github.com/mitchellh/mapstructure"
// FixerVagrantPPOverride is a Fixer that replaces the provider-specific
// overrides for the Vagrant post-processor with the new style introduced
@ -10,9 +8,13 @@ import (
type FixerVagrantPPOverride struct{}
func (FixerVagrantPPOverride) Fix(input map[string]interface{}) (map[string]interface{}, error) {
if input["post-processors"] == nil {
return input, nil
}
// Our template type we'll use for this fixer only
type template struct {
PostProcessors []interface{} `mapstructure:"post-processors"`
PP `mapstructure:",squash"`
}
// Decode the input into our structure, if we can
@ -21,21 +23,7 @@ func (FixerVagrantPPOverride) Fix(input map[string]interface{}) (map[string]inte
return nil, err
}
// Go through each post-processor and get out all the complex configs
pps := make([]map[string]interface{}, 0, len(tpl.PostProcessors))
for _, rawPP := range tpl.PostProcessors {
switch pp := rawPP.(type) {
case string:
case map[string]interface{}:
pps = append(pps, pp)
case []interface{}:
for _, innerRawPP := range pp {
if innerPP, ok := innerRawPP.(map[string]interface{}); ok {
pps = append(pps, innerPP)
}
}
}
}
pps := tpl.ppList()
// Go through each post-processor and make the fix if necessary
possible := []string{"aws", "digitalocean", "virtualbox", "vmware"}

11
fix/fixer_pp_vagrant_override_test.go
View File

@ -1,8 +1,9 @@
package fix
import (
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
func TestFixerVagrantPPOverride_Impl(t *testing.T) {
@ -69,11 +70,7 @@ func TestFixerVagrantPPOverride_Fix(t *testing.T) {
}
output, err := f.Fix(input)
if err != nil {
t.Fatalf("err: %s", err)
}
assert.NoError(t, err)
if !reflect.DeepEqual(output, expected) {
t.Fatalf("unexpected: %#v\nexpected: %#v\n", output, expected)
}
assert.Equal(t, expected, output)
}

25
fix/helpers.go Normal file
View File

@ -0,0 +1,25 @@
package fix
// PP is a convenient way to interact with the post-processors within a fixer
type PP struct {
PostProcessors []interface{} `mapstructure:"post-processors"`
}
// postProcessors converts the variable structure of the template to a list
func (pp *PP) ppList() []map[string]interface{} {
pps := make([]map[string]interface{}, 0, len(pp.PostProcessors))
for _, rawPP := range pp.PostProcessors {
switch pp := rawPP.(type) {
case string:
case map[string]interface{}:
pps = append(pps, pp)
case []interface{}:
for _, innerRawPP := range pp {
if innerPP, ok := innerRawPP.(map[string]interface{}); ok {
pps = append(pps, innerPP)
}
}
}
}
return pps
}

147
packer/progressbar.go Normal file
View File

@ -0,0 +1,147 @@
package packer
import (
"fmt"
"io"
"sync"
"sync/atomic"
"github.com/cheggaaa/pb"
)
// ProgressBar allows to graphically display
// a self refreshing progress bar.
type ProgressBar interface {
Start(total int64)
Add(current int64)
NewProxyReader(r io.Reader) (proxy io.Reader)
Finish()
}
// StackableProgressBar is a progress bar that
// allows to track multiple downloads at once.
// Every call to Start increments a counter that
// will display the number of current loadings.
// Every call to Start will add total to an internal
// total that is the total displayed.
// First call to Start will start a goroutine
// that is waiting for every download to be finished.
// Last call to Finish triggers a cleanup.
// When all active downloads are finished
// StackableProgressBar will clean itself to a default
// state.
type StackableProgressBar struct {
mtx sync.Mutex // locks in Start & Finish
BasicProgressBar
items int32
total int64
started bool
}
var _ ProgressBar = new(StackableProgressBar)
func (spb *StackableProgressBar) start() {
spb.BasicProgressBar.ProgressBar = pb.New(0)
spb.BasicProgressBar.ProgressBar.SetUnits(pb.U_BYTES)
spb.BasicProgressBar.ProgressBar.Start()
spb.started = true
}
func (spb *StackableProgressBar) Start(total int64) {
spb.mtx.Lock()
spb.total += total
spb.items++
if !spb.started {
spb.start()
}
spb.SetTotal64(spb.total)
spb.prefix()
spb.mtx.Unlock()
}
func (spb *StackableProgressBar) prefix() {
spb.BasicProgressBar.ProgressBar.Prefix(fmt.Sprintf("%d items: ", atomic.LoadInt32(&spb.items)))
}
func (spb *StackableProgressBar) Finish() {
spb.mtx.Lock()
spb.items--
if spb.items == 0 {
// slef cleanup
spb.BasicProgressBar.ProgressBar.Finish()
spb.BasicProgressBar.ProgressBar = nil
spb.started = false
spb.total = 0
return
}
spb.prefix()
spb.mtx.Unlock()
}
// BasicProgressBar is packer's basic progress bar.
// Current implementation will always try to keep
// itself at the bottom of a terminal.
type BasicProgressBar struct {
*pb.ProgressBar
}
var _ ProgressBar = new(BasicProgressBar)
func (bpb *BasicProgressBar) Start(total int64) {
bpb.SetTotal64(total)
bpb.ProgressBar.Start()
}
func (bpb *BasicProgressBar) Add(current int64) {
bpb.ProgressBar.Add64(current)
}
func (bpb *BasicProgressBar) NewProxyReader(r io.Reader) io.Reader {
return &ProxyReader{
Reader: r,
ProgressBar: bpb,
}
}
func (bpb *BasicProgressBar) NewProxyReadCloser(r io.ReadCloser) io.ReadCloser {
return &ProxyReader{
Reader: r,
ProgressBar: bpb,
}
}
// NoopProgressBar is a silent progress bar.
type NoopProgressBar struct {
}
var _ ProgressBar = new(NoopProgressBar)
func (npb *NoopProgressBar) Start(int64) {}
func (npb *NoopProgressBar) Add(int64) {}
func (npb *NoopProgressBar) Finish() {}
func (npb *NoopProgressBar) NewProxyReader(r io.Reader) io.Reader { return r }
func (npb *NoopProgressBar) NewProxyReadCloser(r io.ReadCloser) io.ReadCloser { return r }
// ProxyReader implements io.ReadCloser but sends
// count of read bytes to a progress bar
type ProxyReader struct {
io.Reader
ProgressBar
}
func (r *ProxyReader) Read(p []byte) (n int, err error) {
n, err = r.Reader.Read(p)
r.ProgressBar.Add(int64(n))
return
}
// Close the reader if it implements io.Closer
func (r *ProxyReader) Close() (err error) {
if closer, ok := r.Reader.(io.Closer); ok {
return closer.Close()
}
return
}

3
packer/rpc/server.go
View File

@ -114,7 +114,8 @@ func (s *Server) RegisterProvisioner(p packer.Provisioner) {
func (s *Server) RegisterUi(ui packer.Ui) {
s.server.RegisterName(DefaultUiEndpoint, &UiServer{
ui: ui,
ui: ui,
register: s.server.RegisterName,
})
}

54
packer/rpc/ui.go
View File

@ -1,6 +1,7 @@
package rpc
import (
"io"
"log"
"net/rpc"
@ -14,10 +15,13 @@ type Ui struct {
endpoint string
}
var _ packer.Ui = new(Ui)
// UiServer wraps a packer.Ui implementation and makes it exportable
// as part of a Golang RPC server.
type UiServer struct {
ui packer.Ui
ui packer.Ui
register func(name string, rcvr interface{}) error
}
// The arguments sent to Ui.Machine
@ -60,6 +64,31 @@ func (u *Ui) Say(message string) {
}
}
func (u *Ui) ProgressBar() packer.ProgressBar {
if err := u.client.Call("Ui.ProgressBar", new(interface{}), new(interface{})); err != nil {
log.Printf("Error in Ui RPC call: %s", err)
}
return u // Ui is also a progress bar !!
}
var _ packer.ProgressBar = new(Ui)
func (pb *Ui) Start(total int64) {
pb.client.Call("Ui.Start", total, new(interface{}))
}
func (pb *Ui) Add(current int64) {
pb.client.Call("Ui.Add", current, new(interface{}))
}
func (pb *Ui) Finish() {
pb.client.Call("Ui.Finish", nil, new(interface{}))
}
func (pb *Ui) NewProxyReader(r io.Reader) io.Reader {
return &packer.ProxyReader{Reader: r, ProgressBar: pb}
}
func (u *UiServer) Ask(query string, reply *string) (err error) {
*reply, err = u.ui.Ask(query)
return
@ -91,3 +120,26 @@ func (u *UiServer) Say(message *string, reply *interface{}) error {
*reply = nil
return nil
}
func (u *UiServer) ProgressBar(_ *string, reply *interface{}) error {
// No-op for now, this function might be
// used in the future if we want to use
// different progress bars with identifiers.
u.ui.ProgressBar()
return nil
}
func (pb *UiServer) Finish(_ string, _ *interface{}) error {
pb.ui.ProgressBar().Finish()
return nil
}
func (pb *UiServer) Start(total int64, _ *interface{}) error {
pb.ui.ProgressBar().Start(total)
return nil
}
func (pb *UiServer) Add(current int64, _ *interface{}) error {
pb.ui.ProgressBar().Add(current)
return nil
}

51
packer/rpc/ui_test.go
View File

@ -1,8 +1,11 @@
package rpc
import (
"io"
"reflect"
"testing"
"github.com/hashicorp/packer/packer"
)
type testUi struct {
@ -17,6 +20,12 @@ type testUi struct {
messageMessage string
sayCalled bool
sayMessage string
progressBarCalled bool
progressBarStartCalled bool
progressBarAddCalled bool
progressBarFinishCalled bool
progressBarNewProxyReaderCalled bool
}
func (u *testUi) Ask(query string) (string, error) {
@ -46,6 +55,28 @@ func (u *testUi) Say(message string) {
u.sayMessage = message
}
func (u *testUi) ProgressBar() packer.ProgressBar {
u.progressBarCalled = true
return u
}
func (u *testUi) Start(int64) {
u.progressBarStartCalled = true
}
func (u *testUi) Add(int64) {
u.progressBarAddCalled = true
}
func (u *testUi) Finish() {
u.progressBarFinishCalled = true
}
func (u *testUi) NewProxyReader(r io.Reader) io.Reader {
u.progressBarNewProxyReaderCalled = true
return r
}
func TestUiRPC(t *testing.T) {
// Create the UI to test
ui := new(testUi)
@ -88,6 +119,26 @@ func TestUiRPC(t *testing.T) {
t.Fatalf("bad: %#v", ui.errorMessage)
}
bar := uiClient.ProgressBar()
if ui.progressBarCalled != true {
t.Errorf("ProgressBar not called.")
}
bar.Start(100)
if ui.progressBarStartCalled != true {
t.Errorf("progressBar.Start not called.")
}
bar.Add(1)
if ui.progressBarAddCalled != true {
t.Errorf("progressBar.Add not called.")
}
bar.Finish()
if ui.progressBarFinishCalled != true {
t.Errorf("progressBar.Finish not called.")
}
uiClient.Machine("foo", "bar", "baz")
if !ui.machineCalled {
t.Fatal("machine should be called")

37
packer/ui.go
View File

@ -37,8 +37,20 @@ type Ui interface {
Message(string)
Error(string)
Machine(string, ...string)
ProgressBar() ProgressBar
}
type NoopUi struct{}
var _ Ui = new(NoopUi)
func (*NoopUi) Ask(string) (string, error) { return "", errors.New("this is a noop ui") }
func (*NoopUi) Say(string) { return }
func (*NoopUi) Message(string) { return }
func (*NoopUi) Error(string) { return }
func (*NoopUi) Machine(string, ...string) { return }
func (*NoopUi) ProgressBar() ProgressBar { return new(NoopProgressBar) }
// ColoredUi is a UI that is colored using terminal colors.
type ColoredUi struct {
Color UiColor
@ -46,6 +58,8 @@ type ColoredUi struct {
Ui Ui
}
var _ Ui = new(ColoredUi)
// TargetedUI is a UI that wraps another UI implementation and modifies
// the output to indicate a specific target. Specifically, all Say output
// is prefixed with the target name. Message output is not prefixed but
@ -56,6 +70,8 @@ type TargetedUI struct {
Ui Ui
}
var _ Ui = new(TargetedUI)
// The BasicUI is a UI that reads and writes from a standard Go reader
// and writer. It is safe to be called from multiple goroutines. Machine
// readable output is simply logged for this UI.
@ -66,6 +82,13 @@ type BasicUi struct {
l sync.Mutex
interrupted bool
scanner *bufio.Scanner
StackableProgressBar
}
var _ Ui = new(BasicUi)
func (bu *BasicUi) ProgressBar() ProgressBar {
return &bu.StackableProgressBar
}
// MachineReadableUi is a UI that only outputs machine-readable output
@ -74,6 +97,8 @@ type MachineReadableUi struct {
Writer io.Writer
}
var _ Ui = new(MachineReadableUi)
func (u *ColoredUi) Ask(query string) (string, error) {
return u.Ui.Ask(u.colorize(query, u.Color, true))
}
@ -100,6 +125,10 @@ func (u *ColoredUi) Machine(t string, args ...string) {
u.Ui.Machine(t, args...)
}
func (u *ColoredUi) ProgressBar() ProgressBar {
return u.Ui.ProgressBar() //TODO(adrien): color me
}
func (u *ColoredUi) colorize(message string, color UiColor, bold bool) string {
if !u.supportsColors() {
return message
@ -153,6 +182,10 @@ func (u *TargetedUI) Machine(t string, args ...string) {
u.Ui.Machine(fmt.Sprintf("%s,%s", u.Target, t), args...)
}
func (u *TargetedUI) ProgressBar() ProgressBar {
return u.Ui.ProgressBar()
}
func (u *TargetedUI) prefixLines(arrow bool, message string) string {
arrowText := "==>"
if !arrow {
@ -305,3 +338,7 @@ func (u *MachineReadableUi) Machine(category string, args ...string) {
}
}
}
func (u *MachineReadableUi) ProgressBar() ProgressBar {
return new(NoopProgressBar)
}

7
provisioner/ansible-local/provisioner_test.go
View File

@ -8,11 +8,12 @@ import (
"testing"
"fmt"
"os/exec"
"github.com/hashicorp/packer/builder/docker"
"github.com/hashicorp/packer/packer"
"github.com/hashicorp/packer/provisioner/file"
"github.com/hashicorp/packer/template"
"os/exec"
)
func TestProvisioner_Impl(t *testing.T) {
@ -132,7 +133,7 @@ func TestProvisionerProvision_PlaybookFiles(t *testing.T) {
}
comm := &communicatorMock{}
if err := p.Provision(&uiStub{}, comm); err != nil {
if err := p.Provision(new(packer.NoopUi), comm); err != nil {
t.Fatalf("err: %s", err)
}
@ -166,7 +167,7 @@ func TestProvisionerProvision_PlaybookFilesWithPlaybookDir(t *testing.T) {
}
comm := &communicatorMock{}
if err := p.Provision(&uiStub{}, comm); err != nil {
if err := p.Provision(new(packer.NoopUi), comm); err != nil {
t.Fatalf("err: %s", err)
}

15
provisioner/ansible-local/ui_stub.go
View File

@ -1,15 +0,0 @@
package ansiblelocal
type uiStub struct{}
func (su *uiStub) Ask(string) (string, error) {
return "", nil
}
func (su *uiStub) Error(string) {}
func (su *uiStub) Machine(string, ...string) {}
func (su *uiStub) Message(string) {}
func (su *uiStub) Say(msg string) {}

32
provisioner/ansible/adapter_test.go
View File

@ -24,7 +24,7 @@ func TestAdapter_Serve(t *testing.T) {
config := &ssh.ServerConfig{}
ui := new(ui)
ui := new(packer.NoopUi)
sut := newAdapter(done, &l, config, "", newUi(ui), communicator{})
go func() {
@ -93,36 +93,6 @@ func (a addr) String() string {
return "test"
}
type ui int
func (u *ui) Ask(s string) (string, error) {
*u++
return s, nil
}
func (u *ui) Say(s string) {
*u++
log.Println(s)
}
func (u *ui) Message(s string) {
*u++
log.Println(s)
}
func (u *ui) Error(s string) {
*u++
log.Println(s)
}
func (u *ui) Machine(s1 string, s2 ...string) {
*u++
log.Println(s1)
for _, s := range s2 {
log.Println(s)
}
}
type communicator struct{}
func (c communicator) Start(*packer.RemoteCmd) error {

4
provisioner/ansible/provisioner.go
View File

@ -612,3 +612,7 @@ func (ui *Ui) Machine(t string, args ...string) {
ui.ui.Machine(t, args...)
<-ui.sem
}
func (ui *Ui) ProgressBar() packer.ProgressBar {
return new(packer.NoopProgressBar)
}

12
provisioner/file/provisioner.go
View File

@ -127,12 +127,12 @@ func (p *Provisioner) ProvisionDownload(ui packer.Ui, comm packer.Communicator)
defer f.Close()
// Get a default progress bar
pb := common.GetProgressBar(ui, &p.config.PackerConfig)
bar := pb.Start()
defer bar.Finish()
pb := packer.NoopProgressBar{}
pb.Start(0) // TODO: find size ? Remove ?
defer pb.Finish()
// Create MultiWriter for the current progress
pf := io.MultiWriter(f, bar)
pf := io.MultiWriter(f)
// Download the file
if err = comm.Download(src, pf); err != nil {
@ -176,8 +176,8 @@ func (p *Provisioner) ProvisionUpload(ui packer.Ui, comm packer.Communicator) er
}
// Get a default progress bar
pb := common.GetProgressBar(ui, &p.config.PackerConfig)
bar := pb.Start()
bar := ui.ProgressBar()
bar.Start(info.Size())
defer bar.Finish()
// Create ProxyReader for the current progress

40
provisioner/file/provisioner_test.go
View File

@ -1,6 +1,7 @@
package file
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
@ -99,27 +100,6 @@ func TestProvisionerPrepare_EmptyDestination(t *testing.T) {
}
}
type stubUi struct {
sayMessages string
}
func (su *stubUi) Ask(string) (string, error) {
return "", nil
}
func (su *stubUi) Error(string) {
}
func (su *stubUi) Machine(string, ...string) {
}
func (su *stubUi) Message(string) {
}
func (su *stubUi) Say(msg string) {
su.sayMessages += msg
}
func TestProvisionerProvision_SendsFile(t *testing.T) {
var p Provisioner
tf, err := ioutil.TempFile("", "packer")
@ -141,18 +121,21 @@ func TestProvisionerProvision_SendsFile(t *testing.T) {
t.Fatalf("err: %s", err)
}
ui := &stubUi{}
b := bytes.NewBuffer(nil)
ui := &packer.BasicUi{
Writer: b,
}
comm := &packer.MockCommunicator{}
err = p.Provision(ui, comm)
if err != nil {
t.Fatalf("should successfully provision: %s", err)
}
if !strings.Contains(ui.sayMessages, tf.Name()) {
if !strings.Contains(b.String(), tf.Name()) {
t.Fatalf("should print source filename")
}
if !strings.Contains(ui.sayMessages, "something") {
if !strings.Contains(b.String(), "something") {
t.Fatalf("should print destination filename")
}
@ -197,18 +180,21 @@ func TestProvisionDownloadMkdirAll(t *testing.T) {
if err := p.Prepare(config); err != nil {
t.Fatalf("err: %s", err)
}
ui := &stubUi{}
b := bytes.NewBuffer(nil)
ui := &packer.BasicUi{
Writer: b,
}
comm := &packer.MockCommunicator{}
err = p.ProvisionDownload(ui, comm)
if err != nil {
t.Fatalf("should successfully provision: %s", err)
}
if !strings.Contains(ui.sayMessages, tf.Name()) {
if !strings.Contains(b.String(), tf.Name()) {
t.Fatalf("should print source filename")
}
if !strings.Contains(ui.sayMessages, "something") {
if !strings.Contains(b.String(), "something") {
t.Fatalf("should print destination filename")
}