builder/common: Create a downloader

builder/common/download.go

@@ -0,0 +1,186 @@
+package common
+
+import (
+	"bytes"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+)
+
+// DownloadConfig is the configuration given to instantiate a new
+// download instance. Once a configuration is used to instantiate
+// a download client, it must not be modified.
+type DownloadConfig struct {
+	// The source URL in the form of a string.
+	Url string
+
+	// This is the path to download the file to.
+	TargetPath string
+
+	// DownloaderMap maps a schema to a Download.
+	DownloaderMap map[string]Downloader
+
+	// If true, this will copy even a local file to the target
+	// location. If false, then it will "download" the file by just
+	// returning the local path to the file.
+	CopyFile bool
+
+	// The hashing implementation to use to checksum the downloaded file.
+	Hash hash.Hash
+
+	// The checksum for the downloaded file. The hash implementation configuration
+	// for the downloader will be used to verify with this checksum after
+	// it is downloaded.
+	Checksum []byte
+}
+
+// A DownloadClient helps download, verify checksums, etc.
+type DownloadClient struct {
+	config     *DownloadConfig
+	downloader Downloader
+}
+
+// NewDownloadClient returns a new DownloadClient for the given
+// configuration.
+func NewDownloadClient(c *DownloadConfig) *DownloadClient {
+	if c.DownloaderMap == nil {
+		c.DownloaderMap = map[string]Downloader{
+			"http": new(HTTPDownloader),
+		}
+	}
+
+	return &DownloadClient{config: c}
+}
+
+// A downloader is responsible for actually taking a remote URL and
+// downloading it.
+type Downloader interface {
+	Cancel()
+	Download(io.Writer, *url.URL) error
+	Progress() uint
+	Total() uint
+}
+
+func (d *DownloadClient) Cancel() {
+	// TODO(mitchellh): Implement
+}
+
+func (d *DownloadClient) Get() (string, error) {
+	// If we already have the file and it matches, then just return the target path.
+	if verify, _ := d.VerifyChecksum(d.config.TargetPath); verify {
+		return d.config.TargetPath, nil
+	}
+
+	url, err := url.Parse(d.config.Url)
+	if err != nil {
+		return "", err
+	}
+
+	// Files when we don't copy the file are special cased.
+	var finalPath string
+	if url.Scheme == "file" && !d.config.CopyFile {
+		finalPath = url.Path
+	} else {
+		var ok bool
+		d.downloader, ok = d.config.DownloaderMap[url.Scheme]
+		if !ok {
+			return "", fmt.Errorf("No downloader for scheme: %s", url.Scheme)
+		}
+
+		// Otherwise, download using the downloader.
+		f, err := os.Create(d.config.TargetPath)
+		if err != nil {
+			return "", err
+		}
+		defer f.Close()
+
+		err = d.downloader.Download(f, url)
+	}
+
+	if d.config.Hash != nil {
+		var verify bool
+		verify, err = d.VerifyChecksum(finalPath)
+		if err == nil && !verify {
+			err = fmt.Errorf("checksums didn't match expected: %s", hex.EncodeToString(d.config.Checksum))
+		}
+	}
+
+	return finalPath, err
+}
+
+// PercentProgress returns the download progress as a percentage.
+func (d *DownloadClient) PercentProgress() uint {
+	return uint((float64(d.downloader.Progress()) / float64(d.downloader.Total())) * 100)
+}
+
+// VerifyChecksum tests that the path matches the checksum for the
+// download.
+func (d *DownloadClient) VerifyChecksum(path string) (bool, error) {
+	if d.config.Checksum == nil || d.config.Hash == nil {
+		return false, errors.New("Checksum or Hash isn't set on download.")
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	d.config.Hash.Reset()
+	io.Copy(d.config.Hash, f)
+	return bytes.Compare(d.config.Hash.Sum(nil), d.config.Checksum) == 0, nil
+}
+
+// HTTPDownloader is an implementation of Downloader that downloads
+// files over HTTP.
+type HTTPDownloader struct {
+	progress uint
+	total    uint
+}
+
+func (*HTTPDownloader) Cancel() {
+	// TODO(mitchellh): Implement
+}
+
+func (d *HTTPDownloader) Download(dst io.Writer, src *url.URL) error {
+	resp, err := http.Get(src.String())
+	if err != nil {
+		return err
+	}
+
+	d.progress = 0
+	d.total = uint(resp.ContentLength)
+
+	var buffer [4096]byte
+	for {
+		n, err := resp.Body.Read(buffer[:])
+		if err != nil && err != io.EOF {
+			return err
+		}
+
+		d.progress += uint(n)
+
+		if _, werr := dst.Write(buffer[:n]); werr != nil {
+			return werr
+		}
+
+		if err == io.EOF {
+			break
+		}
+	}
+
+	return nil
+}
+
+func (d *HTTPDownloader) Progress() uint {
+	return d.progress
+}
+
+func (d *HTTPDownloader) Total() uint {
+	return d.total
+}

builder/common/download_test.go

@@ -0,0 +1,42 @@
+package common
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestDownloadClient_VerifyChecksum(t *testing.T) {
+	tf, err := ioutil.TempFile("", "packer")
+	if err != nil {
+		t.Fatalf("tempfile error: %s", err)
+	}
+	defer os.Remove(tf.Name())
+
+	// "foo"
+	checksum, err := hex.DecodeString("acbd18db4cc2f85cedef654fccc4a4d8")
+	if err != nil {
+		t.Fatalf("decode err: %s", err)
+	}
+
+	// Write the file
+	tf.Write([]byte("foo"))
+	tf.Close()
+
+	config := &DownloadConfig{
+		Hash:     md5.New(),
+		Checksum: checksum,
+	}
+
+	d := NewDownloadClient(config)
+	result, err := d.VerifyChecksum(tf.Name())
+	if err != nil {
+		t.Fatalf("Verify err: %s", err)
+	}
+
+	if !result {
+		t.Fatal("didn't verify")
+	}
+}

builder/vmware/step_download_iso.go

@@ -5,13 +5,9 @@ import (
 	"encoding/hex"
 	"fmt"
 	"github.com/mitchellh/multistep"
+	"github.com/mitchellh/packer/builder/common"
 	"github.com/mitchellh/packer/packer"
-	"io"
 	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"strings"
 	"time"
 )
 
@@ -31,62 +27,52 @@ func (s stepDownloadISO) Run(state map[string]interface{}) multistep.StepAction
 	config := state["config"].(*config)
 	ui := state["ui"].(packer.Ui)
 
+	checksum, err := hex.DecodeString(config.ISOMD5)
+	if err != nil {
+		ui.Error(fmt.Sprintf("Error parsing checksum: %s", err))
+		return multistep.ActionHalt
+	}
+
 	log.Printf("Acquiring lock to download the ISO.")
 	cachePath := cache.Lock(config.ISOUrl)
 	defer cache.Unlock(config.ISOUrl)
 
-	err := s.checkMD5(cachePath, config.ISOMD5)
+	downloadConfig := &common.DownloadConfig{
-	haveFile := err == nil
+		Url:        config.ISOUrl,
-	if err != nil {
+		TargetPath: cachePath,
-		if !os.IsNotExist(err) {
+		CopyFile:   false,
-			ui.Say(fmt.Sprintf("Error validating MD5 of ISO: %s", err))
+		Hash:       md5.New(),
-			return multistep.ActionHalt
+		Checksum:   checksum,
-		}
 	}
 
-	if !haveFile {
+	download := common.NewDownloadClient(downloadConfig)
-		url, err := url.Parse(config.ISOUrl)
-		if err != nil {
-			ui.Error(fmt.Sprintf("Error parsing iso_url: %s", err))
-			return multistep.ActionHalt
-		}
 
-		// Start the download in a goroutine so that we cancel it and such.
+	downloadCompleteCh := make(chan error, 1)
-		var progress uint
+	go func() {
-		downloadComplete := make(chan bool, 1)
+		ui.Say("Copying or downloading ISO. Progress will be reported periodically.")
-		go func() {
+		cachePath, err = download.Get()
-			ui.Say("Copying or downloading ISO. Progress will be shown periodically.")
+		downloadCompleteCh <- err
-			cachePath, err = s.downloadUrl(cachePath, url, &progress)
+	}()
-			downloadComplete <- true
-		}()
 
-		progressTimer := time.NewTicker(15 * time.Second)
+	progressTicker := time.NewTicker(5 * time.Second)
-		defer progressTimer.Stop()
+	defer progressTicker.Stop()
 
-	DownloadWaitLoop:
+DownloadWaitLoop:
-		for {
+	for {
-			select {
+		select {
-			case <-downloadComplete:
+		case err := <-downloadCompleteCh:
-				log.Println("Download of ISO completed.")
+			if err != nil {
-				break DownloadWaitLoop
+				ui.Error(fmt.Sprintf("Error downloading ISO: %s", err))
-			case <-progressTimer.C:
-				ui.Say(fmt.Sprintf("Download progress: %d%%", progress))
-			case <-time.After(1 * time.Second):
-				if _, ok := state[multistep.StateCancelled]; ok {
-					ui.Say("Interrupt received. Cancelling download...")
-					return multistep.ActionHalt
-				}
 			}
-		}
 
-		if err != nil {
+			break DownloadWaitLoop
-			ui.Error(fmt.Sprintf("Error downloading ISO: %s", err))
+		case <-progressTicker.C:
-			return multistep.ActionHalt
+			ui.Say(fmt.Sprintf("Download progress: %d%%", download.PercentProgress()))
-		}
+		case <-time.After(1 * time.Second):
-
+			if _, ok := state[multistep.StateCancelled]; ok {
-		if err = s.checkMD5(cachePath, config.ISOMD5); err != nil {
+				ui.Say("Interrupt received. Cancelling download...")
-			ui.Say(fmt.Sprintf("Error validating MD5 of ISO: %s", err))
+				return multistep.ActionHalt
-			return multistep.ActionHalt
+			}
 		}
 	}
 
@@ -97,61 +83,3 @@ func (s stepDownloadISO) Run(state map[string]interface{}) multistep.StepAction
 }
 
 func (stepDownloadISO) Cleanup(map[string]interface{}) {}
-
-func (stepDownloadISO) checkMD5(path string, expected string) error {
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-
-	hash := md5.New()
-	io.Copy(hash, f)
-	result := strings.ToLower(hex.EncodeToString(hash.Sum(nil)))
-	if result != expected {
-		return fmt.Errorf("result != expected: %s != %s", result, expected)
-	}
-
-	return nil
-}
-
-func (stepDownloadISO) downloadUrl(path string, url *url.URL, progress *uint) (string, error) {
-	if url.Scheme == "file" {
-		// If it is just a file URL, then we already have the ISO
-		return url.Path, nil
-	}
-
-	// Otherwise, it is an HTTP URL, and we must download it.
-	f, err := os.Create(path)
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-
-	log.Printf("Beginning download of ISO: %s", url.String())
-	resp, err := http.Get(url.String())
-	if err != nil {
-		return "", err
-	}
-
-	var buffer [4096]byte
-	var totalRead int64
-	for {
-		n, err := resp.Body.Read(buffer[:])
-		if err != nil && err != io.EOF {
-			return "", err
-		}
-
-		totalRead += int64(n)
-		*progress = uint((float64(totalRead) / float64(resp.ContentLength)) * 100)
-
-		if _, werr := f.Write(buffer[:n]); werr != nil {
-			return "", werr
-		}
-
-		if err == io.EOF {
-			break
-		}
-	}
-
-	return path, nil
-}

command/build/command.go

@@ -102,7 +102,7 @@ func (c Command) Run(env packer.Environment, args []string) int {
 
 		ui = &packer.ColoredUi{
 			Color: colors[i%len(colors)],
-			Ui: env.Ui(),
+			Ui:    env.Ui(),
 		}
 
 		ui = &packer.PrefixedUi{