file provisioner improvements

* allow specify source/destination as dir
* allow specify many files as source

Signed-off-by: Vasiliy Tolstov <v.tolstov@selfip.ru>

builder/amazon/chroot/communicator.go

@@ -3,7 +3,6 @@ package chroot
 import (
 	"bytes"
 	"fmt"
-	"github.com/mitchellh/packer/packer"
 	"io"
 	"io/ioutil"
 	"log"
@@ -12,6 +11,8 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
+
+	"github.com/mitchellh/packer/packer"
 )
 
 // Communicator is a special communicator that works by executing
@@ -114,6 +115,10 @@ func (c *Communicator) UploadDir(dst string, src string, exclude []string) error
 	return err
 }
 
+func (c *Communicator) DownloadDir(src string, dst string, exclude []string) error {
+	return fmt.Errorf("DownloadDir is not implemented for amazon-chroot")
+}
+
 func (c *Communicator) Download(src string, w io.Writer) error {
 	src = filepath.Join(c.Chroot, src)
 	log.Printf("Downloading from chroot dir: %s", src)

builder/docker/communicator.go

@@ -233,6 +233,10 @@ func (c *Communicator) Download(src string, dst io.Writer) error {
 	return nil
 }
 
+func (c *Communicator) DownloadDir(src string, dst string, exclude []string) error {
+	return fmt.Errorf("DownloadDir is not implemented for docker")
+}
+
 // canExec tells us whether `docker exec` is supported
 func (c *Communicator) canExec() bool {
 	execConstraint, err := version.NewConstraint(">= 1.4.0")

communicator/ssh/communicator.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -156,12 +157,73 @@ func (c *comm) UploadDir(dst string, src string, excl []string) error {
 	}
 }
 
+func (c *comm) DownloadDir(src string, dst string, excl []string) error {
+	log.Printf("Download dir '%s' to '%s'", src, dst)
+	scpFunc := func(w io.Writer, stdoutR *bufio.Reader) error {
+		for {
+			fmt.Fprint(w, "\x00")
+
+			// read file info
+			fi, err := stdoutR.ReadString('\n')
+			if err != nil {
+				return err
+			}
+
+			if len(fi) < 0 {
+				return fmt.Errorf("empty response from server")
+			}
+
+			switch fi[0] {
+			case '\x01', '\x02':
+				return fmt.Errorf("%s", fi[1:len(fi)])
+			case 'C', 'D':
+				break
+			default:
+				return fmt.Errorf("unexpected server response (%x)", fi[0])
+			}
+
+			var mode string
+			var size int64
+			var name string
+			log.Printf("Download dir str:%s", fi)
+			n, err := fmt.Sscanf(fi, "%6s %d %s", &mode, &size, &name)
+			if err != nil || n != 3 {
+				return fmt.Errorf("can't parse server response (%s)", fi)
+			}
+			if size < 0 {
+				return fmt.Errorf("negative file size")
+			}
+
+			log.Printf("Download dir mode:%s size:%d name:%s", mode, size, name)
+			switch fi[0] {
+			case 'D':
+				err = os.MkdirAll(filepath.Join(dst, name), os.FileMode(0755))
+				if err != nil {
+					return err
+				}
+				fmt.Fprint(w, "\x00")
+				return nil
+			case 'C':
+				fmt.Fprint(w, "\x00")
+				err = scpDownloadFile(filepath.Join(dst, name), stdoutR, size, os.FileMode(0644))
+				if err != nil {
+					return err
+				}
+			}
+
+			if err := checkSCPStatus(stdoutR); err != nil {
+				return err
+			}
+		}
+	}
+	return c.scpSession("scp -vrf "+src, scpFunc)
+}
+
 func (c *comm) Download(path string, output io.Writer) error {
 	if c.config.UseSftp {
 		return c.sftpDownloadSession(path, output)
-	} else {
-		return c.scpDownloadSession(path, output)
 	}
+	return c.scpDownloadSession(path, output)
 }
 
 func (c *comm) newSession() (session *ssh.Session, err error) {
@@ -563,6 +625,9 @@ func (c *comm) scpDownloadSession(path string, output io.Writer) error {
 		return nil
 	}
 
+	if strings.Index(path, " ") == -1 {
+		return c.scpSession("scp -vf "+path, scpFunc)
+	}
 	return c.scpSession("scp -vf "+strconv.Quote(path), scpFunc)
 }
 
@@ -668,6 +733,18 @@ func checkSCPStatus(r *bufio.Reader) error {
 	return nil
 }
 
+func scpDownloadFile(dst string, src io.Reader, size int64, mode os.FileMode) error {
+	f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := io.CopyN(f, src, size); err != nil {
+		return err
+	}
+	return nil
+}
+
 func scpUploadFile(dst string, src io.Reader, w io.Writer, r *bufio.Reader, fi *os.FileInfo) error {
 	var mode os.FileMode
 	var size int64

communicator/winrm/communicator.go

@@ -140,6 +140,10 @@ func (c *Communicator) Download(src string, dst io.Writer) error {
 	return fmt.Errorf("WinRM doesn't support download.")
 }
 
+func (c *Communicator) DownloadDir(src string, dst string, exclude []string) error {
+	return fmt.Errorf("WinRM doesn't support download dir.")
+}
+
 func (c *Communicator) newCopyClient() (*winrmcp.Winrmcp, error) {
 	addr := fmt.Sprintf("%s:%d", c.endpoint.Host, c.endpoint.Port)
 	return winrmcp.New(addr, &winrmcp.Config{

packer/communicator.go

@@ -75,6 +75,8 @@ type Communicator interface {
 	// with the contents writing to the given writer. This method will
 	// block until it completes.
 	Download(string, io.Writer) error
+
+	DownloadDir(src string, dst string, exclude []string) error
 }
 
 // StartWithUi runs the remote command and streams the output to any

packer/communicator_mock.go

@@ -25,6 +25,10 @@ type MockCommunicator struct {
 	UploadDirSrc     string
 	UploadDirExclude []string
 
+	DownloadDirDst     string
+	DownloadDirSrc     string
+	DownloadDirExclude []string
+
 	DownloadCalled bool
 	DownloadPath   string
 	DownloadData   string
@@ -98,3 +102,11 @@ func (c *MockCommunicator) Download(path string, w io.Writer) error {
 
 	return nil
 }
+
+func (c *MockCommunicator) DownloadDir(src string, dst string, excl []string) error {
+	c.DownloadDirDst = dst
+	c.DownloadDirSrc = src
+	c.DownloadDirExclude = excl
+
+	return nil
+}

packer/rpc/communicator.go

@@ -52,6 +52,12 @@ type CommunicatorUploadDirArgs struct {
 	Exclude []string
 }
 
+type CommunicatorDownloadDirArgs struct {
+	Dst     string
+	Src     string
+	Exclude []string
+}
+
 func Communicator(client *rpc.Client) *communicator {
 	return &communicator{client: client}
 }
@@ -135,6 +141,22 @@ func (c *communicator) UploadDir(dst string, src string, exclude []string) error
 	return err
 }
 
+func (c *communicator) DownloadDir(src string, dst string, exclude []string) error {
+	args := &CommunicatorDownloadDirArgs{
+		Dst:     dst,
+		Src:     src,
+		Exclude: exclude,
+	}
+
+	var reply error
+	err := c.client.Call("Communicator.DownloadDir", args, &reply)
+	if err == nil {
+		err = reply
+	}
+
+	return err
+}
+
 func (c *communicator) Download(path string, w io.Writer) (err error) {
 	// Serve a single connection and a single copy
 	streamId := c.mux.NextId()
@@ -253,6 +275,10 @@ func (c *CommunicatorServer) UploadDir(args *CommunicatorUploadDirArgs, reply *e
 	return c.c.UploadDir(args.Dst, args.Src, args.Exclude)
 }
 
+func (c *CommunicatorServer) DownloadDir(args *CommunicatorUploadDirArgs, reply *error) error {
+	return c.c.DownloadDir(args.Src, args.Dst, args.Exclude)
+}
+
 func (c *CommunicatorServer) Download(args *CommunicatorDownloadArgs, reply *interface{}) (err error) {
 	writerC, err := c.mux.Dial(args.WriterStreamId)
 	if err != nil {

post-processor/shell-local/communicator.go

@@ -57,3 +57,7 @@ func (c *Communicator) UploadDir(string, string, []string) error {
 func (c *Communicator) Download(string, io.Writer) error {
 	return fmt.Errorf("download not supported")
 }
+
+func (c *Communicator) DownloadDir(src string, dst string, exclude []string) error {
+	return fmt.Errorf("downloadDir not supported")
+}

provisioner/file/provisioner.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/mitchellh/packer/common"
 	"github.com/mitchellh/packer/helper/config"
@@ -16,6 +17,7 @@ type Config struct {
 
 	// The local path of the file to upload.
 	Source  string
+	Sources []string
 
 	// The remote path where the local file will be uploaded to.
 	Destination string
@@ -52,13 +54,23 @@ func (p *Provisioner) Prepare(raws ...interface{}) error {
 		errs = packer.MultiErrorAppend(errs,
 			errors.New("Direction must be one of: download, upload."))
 	}
+	if p.config.Source != "" {
+		p.config.Sources = append(p.config.Sources, p.config.Source)
+	}
 
 	if p.config.Direction == "upload" {
-		if _, err := os.Stat(p.config.Source); err != nil {
+		for _, src := range p.config.Sources {
+			if _, err := os.Stat(src); err != nil {
 				errs = packer.MultiErrorAppend(errs,
-				fmt.Errorf("Bad source '%s': %s", p.config.Source, err))
+					fmt.Errorf("Bad source '%s': %s", src, err))
 			}
 		}
+	}
+
+	if len(p.config.Sources) < 1 {
+		errs = packer.MultiErrorAppend(errs,
+			errors.New("Source must be specified."))
+	}
 
 	if p.config.Destination == "" {
 		errs = packer.MultiErrorAppend(errs,
@@ -81,7 +93,16 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
 }
 
 func (p *Provisioner) ProvisionDownload(ui packer.Ui, comm packer.Communicator) error {
-	ui.Say(fmt.Sprintf("Downloading %s => %s", p.config.Source, p.config.Destination))
+	for _, src := range p.config.Sources {
+		ui.Say(fmt.Sprintf("Downloading %s => %s", src, p.config.Destination))
+
+		if strings.HasSuffix(p.config.Destination, "/") {
+			err := os.MkdirAll(p.config.Destination, os.FileMode(0755))
+			if err != nil {
+				return err
+			}
+			return comm.DownloadDir(src, p.config.Destination, nil)
+		}
 
 		f, err := os.OpenFile(p.config.Destination, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
 		if err != nil {
@@ -89,27 +110,31 @@ func (p *Provisioner) ProvisionDownload(ui packer.Ui, comm packer.Communicator)
 		}
 		defer f.Close()
 
-	err = comm.Download(p.config.Source, f)
+		err = comm.Download(src, f)
 		if err != nil {
 			ui.Error(fmt.Sprintf("Download failed: %s", err))
-	}
 			return err
+		}
+	}
+	return nil
 }
 
 func (p *Provisioner) ProvisionUpload(ui packer.Ui, comm packer.Communicator) error {
-	ui.Say(fmt.Sprintf("Uploading %s => %s", p.config.Source, p.config.Destination))
+	for _, src := range p.config.Sources {
-	info, err := os.Stat(p.config.Source)
+		ui.Say(fmt.Sprintf("Uploading %s => %s", src, p.config.Destination))
+
+		info, err := os.Stat(src)
 		if err != nil {
 			return err
 		}
 
 		// If we're uploading a directory, short circuit and do that
 		if info.IsDir() {
-		return comm.UploadDir(p.config.Destination, p.config.Source, nil)
+			return comm.UploadDir(p.config.Destination, src, nil)
 		}
 
 		// We're uploading a file...
-	f, err := os.Open(p.config.Source)
+		f, err := os.Open(src)
 		if err != nil {
 			return err
 		}
@@ -123,8 +148,10 @@ func (p *Provisioner) ProvisionUpload(ui packer.Ui, comm packer.Communicator) er
 		err = comm.Upload(p.config.Destination, f, &fi)
 		if err != nil {
 			ui.Error(fmt.Sprintf("Upload failed: %s", err))
-	}
 			return err
+		}
+	}
+	return nil
 }
 
 func (p *Provisioner) Cancel() {

provisioner/shell-local/communicator.go

@@ -76,6 +76,10 @@ func (c *Communicator) Download(string, io.Writer) error {
 	return fmt.Errorf("download not supported")
 }
 
+func (c *Communicator) DownloadDir(string, string, []string) error {
+	return fmt.Errorf("downloadDir not supported")
+}
+
 type ExecuteCommandTemplate struct {
 	Command string
 }