packer-cn/communicator/ssh/communicator.go

937 lines
23 KiB
Go
Raw Normal View History

2013-05-20 18:47:41 -04:00
package ssh
import (
"bufio"
2013-05-20 18:47:41 -04:00
"bytes"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
2013-05-20 18:47:41 -04:00
"log"
"net"
"os"
"path/filepath"
2015-02-08 20:24:31 -05:00
"strconv"
"strings"
"time"
2017-04-04 16:39:01 -04:00
"github.com/hashicorp/packer/packer"
Add tmp package that offers Dir & File funcs this regroups the calls of tmp func calls into one package. the tmp pkg allows to store tmp files under a common directory for easier managment Squashed commit of the following: commit 74b674c01560c6b547e9cab5e8afb486f0fe7d6c Merge: fc94b5714 66001525d Author: Matthew Patton <mpatton@enquizit.com> Date: Fri Nov 2 02:02:38 2018 -0400 Merge branch 'remove_TMPDIR_docs' of github.com:tb3088/packer into remove_TMPDIR_docs commit fc94b571428447aaf2e5171967c95cdf9e5c73ff Author: Matthew Patton <mpatton@enquizit.com> Date: Fri Nov 2 01:42:38 2018 -0400 update tests to honor configured temporary directory commit 5a562359f4937af506607dd96ed1c46426093f52 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 18:52:38 2018 -0400 sync straggler to refactor. implement tests at packer/configfile instead of buried in docker. commit 9d5a4a0935bfd049e2f8f50ad73e4455b42c9410 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 15:39:46 2018 -0400 insert URL to GoLang API for os.TempDir() commit fbae4c2e051898aecd8705e9d8a20353430ede23 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 07:41:29 2018 -0400 revise documentation to match code commit 3590fae8bd0afa92a161a49ed794ee32e357d583 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 07:21:59 2018 -0400 refacter config_file. replace all hard-coded os.TempDir with wrapper commit d5c5306a97b648ded06a8eb76ce728f07c9924f0 Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 20:11:36 2018 -0400 close massive file overwrite hole with TempDir commit 0a72297da86089fbd07e4f7d9472a92fa4d1c733 Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 01:06:00 2018 -0400 adjust var declaration scope commit 20f68228b6c372d984ea5b055cfc8bda9b110ac5 Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 00:54:35 2018 -0400 use mktemp() equivalent to create temporary directory commit c73ebe3d8321664a2685a0baa8e441dd895b0db4 Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 20:40:58 2018 -0400 remove extraneous variable declaration, fix FOR loop commit 63549b8bd745a0090b15ed0b0ebf644162db27db Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 19:30:44 2018 -0400 match styistic convension with rest of docs commit 976101074942db36f10d3a686d6d29ddb7c01926 Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 19:08:39 2018 -0400 revert dangling config file change commit 95159afbc05eac96ed11c3183af5fb0abe2f2d5c Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Fri Dec 29 23:53:43 2017 -0500 replace invalid TMPDIR variable wth PACKER_TMP_DIR. update ConfigTmpDir() to try common temporary paths first and only write to configDir() as a last resort. commit 66001525d72de56a4cf9339b900c46f59bc6e91a Author: Matthew Patton <mpatton@enquizit.com> Date: Fri Nov 2 01:42:38 2018 -0400 update tests to honor configured temporary directory commit e9b6adefeae8c65eb8aa47fef38cbf0aa424338c Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 18:52:38 2018 -0400 sync straggler to refactor. implement tests at packer/configfile instead of buried in docker. commit 852113ed076e2d14e5dca6815ea680da1e2896bb Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 15:39:46 2018 -0400 insert URL to GoLang API for os.TempDir() commit 54add38d1d1c6e283cd444b367ed8bd49a5f3699 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 07:41:29 2018 -0400 revise documentation to match code commit 6b5b8f6d4edad0e187ca13d64ac4118f34eee643 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 07:21:59 2018 -0400 refacter config_file. replace all hard-coded os.TempDir with wrapper commit c22092c601c33484327674f322c7379fa41506d7 Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 20:11:36 2018 -0400 close massive file overwrite hole with TempDir commit 7a730450916bf8e5dbc2a741ec233a49466ab7cc Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 01:06:00 2018 -0400 adjust var declaration scope commit 0f2933adb6f6922dfeab78a95371a444ec8918ab Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 00:54:35 2018 -0400 use mktemp() equivalent to create temporary directory commit d74839ede05dacf712b55a7bb48aec19fe6b007f Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 20:40:58 2018 -0400 remove extraneous variable declaration, fix FOR loop commit eb65416619437e4a3dec90277770803dd5b2281c Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 19:30:44 2018 -0400 match styistic convension with rest of docs commit acaa2b31ed463219c4ef099f351eec72406e2989 Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 19:08:39 2018 -0400 revert dangling config file change commit e573fde668dcb418561e61535c1d68b2888f5b0f Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Fri Dec 29 23:53:43 2017 -0500 replace invalid TMPDIR variable wth PACKER_TMP_DIR. update ConfigTmpDir() to try common temporary paths first and only write to configDir() as a last resort. commit 39a9874afc9dea71b36753375fb7feb61ffecf69 Merge: 8a413cfe8 3315812c2 Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 18:19:26 2018 -0400 Merge branch 'master' of https://github.com/hashicorp/packer into prestine commit 8a413cfe83b0a70dbd99c32d936334ca5788ca9b Merge: e07491de5 4e14710a6 Author: Matthew Patton <mpatton@enquizit.com> Date: Mon Oct 1 20:18:10 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit e07491de59cead0b337a7f57f4a6c625e1f560ab Merge: 42610a35d a1fa35dff Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Aug 21 13:26:19 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit 42610a35d5ef65af8844a58ad70e2ec75262da6a Merge: 5298142da 0d63cf7bc Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Tue Jun 19 22:45:05 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit 5298142da6da95354844f201deeeef3c599e48b2 Merge: 7bb110bc7 9d9736552 Author: Matthew Patton <mpatton@enquizit.com> Date: Mon Jun 11 15:10:09 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit 7bb110bc744c9797fcaec25a265d9b85a22d6d6b Merge: a61c869ca 6189d66e7 Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Wed May 9 23:41:22 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit a61c869ca7268acf0e3b1e1fa7a8ee7feea65984 Merge: 098101dd9 4be5f072c Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Fri May 4 10:13:04 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit 098101dd99f08b0ca110d33eff6904537c6d21de Merge: fefaf0fa6 554b2b4a5 Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Tue May 1 20:52:30 2018 -0400 Merge branch 'EOL-handling' into prestine Co-Authored-By: Matthew Patton <pattonme@yahoo.com>
2018-12-12 09:45:00 -05:00
"github.com/hashicorp/packer/packer/tmp"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
2013-05-20 18:47:41 -04:00
)
// ErrHandshakeTimeout is returned from New() whenever we're unable to establish
// an ssh connection within a certain timeframe. By default the handshake time-
// out period is 1 minute. You can change it with Config.HandshakeTimeout.
var ErrHandshakeTimeout = fmt.Errorf("Timeout during SSH handshake")
2013-05-20 18:47:41 -04:00
type comm struct {
client *ssh.Client
config *Config
conn net.Conn
address string
}
// Config is the structure used to configure the SSH communicator.
type Config struct {
// The configuration of the Go SSH connection
SSHConfig *ssh.ClientConfig
// Connection returns a new connection. The current connection
// in use will be closed as part of the Close method, or in the
// case an error occurs.
Connection func() (net.Conn, error)
// Pty, if true, will request a pty from the remote end.
Pty bool
// DisableAgentForwarding, if true, will not forward the SSH agent.
DisableAgentForwarding bool
// HandshakeTimeout limits the amount of time we'll wait to handshake before
// saying the connection failed.
HandshakeTimeout time.Duration
// UseSftp, if true, sftp will be used instead of scp for file transfers
UseSftp bool
// KeepAliveInterval sets how often we send a channel request to the
// server. A value < 0 disables.
KeepAliveInterval time.Duration
2018-01-31 02:09:12 -05:00
// Timeout is how long to wait for a read or write to succeed.
Timeout time.Duration
2013-05-20 18:47:41 -04:00
}
// Creates a new packer.Communicator implementation over SSH. This takes
// an already existing TCP connection and SSH configuration.
func New(address string, config *Config) (result *comm, err error) {
// Establish an initial connection and connect
result = &comm{
config: config,
address: address,
}
if err = result.reconnect(); err != nil {
result = nil
return
}
2013-05-20 18:47:41 -04:00
return
}
func (c *comm) Start(ctx context.Context, cmd *packer.RemoteCmd) (err error) {
session, err := c.newSession()
2013-05-20 18:47:41 -04:00
if err != nil {
return
}
// Setup our session
session.Stdin = cmd.Stdin
session.Stdout = cmd.Stdout
session.Stderr = cmd.Stderr
2013-05-20 18:47:41 -04:00
if c.config.Pty {
// Request a PTY
termModes := ssh.TerminalModes{
ssh.ECHO: 0, // do not echo
ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
}
2013-06-23 20:36:45 -04:00
if err = session.RequestPty("xterm", 40, 80, termModes); err != nil {
return
}
2013-06-23 20:36:45 -04:00
}
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] starting remote command: %s", cmd.Command)
err = session.Start(cmd.Command + "\n")
2013-05-20 18:47:41 -04:00
if err != nil {
return
}
go func() {
if c.config.KeepAliveInterval <= 0 {
return
}
c := time.NewTicker(c.config.KeepAliveInterval)
defer c.Stop()
for range c.C {
_, err := session.SendRequest("keepalive@packer.io", true, nil)
if err != nil {
return
}
}
}()
2013-05-20 18:47:41 -04:00
// Start a goroutine to wait for the session to end and set the
// exit boolean and status.
go func() {
defer session.Close()
2013-05-20 18:47:41 -04:00
err := session.Wait()
2013-07-29 15:12:42 -04:00
exitStatus := 0
2013-05-20 18:47:41 -04:00
if err != nil {
switch err.(type) {
case *ssh.ExitError:
exitStatus = err.(*ssh.ExitError).ExitStatus()
2018-03-20 19:28:23 -04:00
log.Printf("[ERROR] Remote command exited with '%d': %s", exitStatus, cmd.Command)
case *ssh.ExitMissingError:
2018-03-20 19:28:23 -04:00
log.Printf("[ERROR] Remote command exited without exit status or exit signal.")
exitStatus = packer.CmdDisconnect
default:
2018-03-20 19:28:23 -04:00
log.Printf("[ERROR] Error occurred waiting for ssh session: %s", err.Error())
2013-05-20 18:47:41 -04:00
}
}
2013-07-29 15:12:42 -04:00
cmd.SetExited(exitStatus)
}()
2013-05-20 18:47:41 -04:00
return
}
func (c *comm) Upload(path string, input io.Reader, fi *os.FileInfo) error {
if c.config.UseSftp {
return c.sftpUploadSession(path, input, fi)
} else {
return c.scpUploadSession(path, input, fi)
2013-05-24 12:06:13 -04:00
}
}
func (c *comm) UploadDir(dst string, src string, excl []string) error {
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] Upload dir '%s' to '%s'", src, dst)
if c.config.UseSftp {
return c.sftpUploadDirSession(dst, src, excl)
} else {
return c.scpUploadDirSession(dst, src, excl)
}
}
func (c *comm) DownloadDir(src string, dst string, excl []string) error {
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] Download dir '%s' to '%s'", src, dst)
scpFunc := func(w io.Writer, stdoutR *bufio.Reader) error {
dirStack := []string{dst}
for {
fmt.Fprint(w, "\x00")
// read file info
fi, err := stdoutR.ReadString('\n')
if err != nil {
return err
}
if len(fi) < 0 {
return fmt.Errorf("empty response from server")
}
switch fi[0] {
case '\x01', '\x02':
2016-11-01 17:08:04 -04:00
return fmt.Errorf("%s", fi[1:])
case 'C', 'D':
break
case 'E':
dirStack = dirStack[:len(dirStack)-1]
if len(dirStack) == 0 {
fmt.Fprint(w, "\x00")
return nil
}
continue
default:
return fmt.Errorf("unexpected server response (%x)", fi[0])
}
var mode int64
var size int64
var name string
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] Download dir str:%s", fi)
n, err := fmt.Sscanf(fi[1:], "%o %d %s", &mode, &size, &name)
if err != nil || n != 3 {
return fmt.Errorf("can't parse server response (%s)", fi)
}
if size < 0 {
return fmt.Errorf("negative file size")
}
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] Download dir mode:%0o size:%d name:%s", mode, size, name)
dst = filepath.Join(dirStack...)
switch fi[0] {
case 'D':
err = os.MkdirAll(filepath.Join(dst, name), os.FileMode(mode))
if err != nil {
return err
}
dirStack = append(dirStack, name)
continue
case 'C':
fmt.Fprint(w, "\x00")
err = scpDownloadFile(filepath.Join(dst, name), stdoutR, size, os.FileMode(mode))
if err != nil {
return err
}
}
if err := checkSCPStatus(stdoutR); err != nil {
return err
}
}
}
return c.scpSession("scp -vrf "+src, scpFunc)
}
2015-02-08 20:24:31 -05:00
func (c *comm) Download(path string, output io.Writer) error {
if c.config.UseSftp {
return c.sftpDownloadSession(path, output)
2015-02-08 20:24:31 -05:00
}
return c.scpDownloadSession(path, output)
}
func (c *comm) newSession() (session *ssh.Session, err error) {
2018-03-20 19:28:23 -04:00
log.Println("[DEBUG] Opening new ssh session")
if c.client == nil {
err = errors.New("client not available")
} else {
session, err = c.client.NewSession()
}
if err != nil {
2018-03-20 19:28:23 -04:00
log.Printf("[ERROR] ssh session open error: '%s', attempting reconnect", err)
if err := c.reconnect(); err != nil {
return nil, err
}
2016-10-14 14:23:35 -04:00
if c.client == nil {
return nil, errors.New("client not available")
2016-10-14 14:23:35 -04:00
} else {
return c.client.NewSession()
2016-10-14 14:23:35 -04:00
}
}
return session, nil
}
func (c *comm) reconnect() (err error) {
if c.conn != nil {
// Ignore errors here because we don't care if it fails
c.conn.Close()
}
// Set the conn and client to nil since we'll recreate it
c.conn = nil
c.client = nil
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] reconnecting to TCP connection for SSH")
c.conn, err = c.config.Connection()
if err != nil {
// Explicitly set this to the REAL nil. Connection() can return
// a nil implementation of net.Conn which will make the
// "if c.conn == nil" check fail above. Read here for more information
// on this psychotic language feature:
//
// http://golang.org/doc/faq#nil_error
c.conn = nil
2018-03-20 19:28:23 -04:00
log.Printf("[ERROR] reconnection error: %s", err)
return
}
2018-01-31 02:09:12 -05:00
if c.config.Timeout > 0 {
c.conn = &timeoutConn{c.conn, c.config.Timeout, c.config.Timeout}
}
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] handshaking with SSH")
// Default timeout to 1 minute if it wasn't specified (zero value). For
// when you need to handshake from low orbit.
var duration time.Duration
if c.config.HandshakeTimeout == 0 {
duration = 1 * time.Minute
} else {
duration = c.config.HandshakeTimeout
}
connectionEstablished := make(chan struct{}, 1)
var sshConn ssh.Conn
var sshChan <-chan ssh.NewChannel
var req <-chan *ssh.Request
go func() {
sshConn, sshChan, req, err = ssh.NewClientConn(c.conn, c.address, c.config.SSHConfig)
close(connectionEstablished)
}()
select {
case <-connectionEstablished:
// We don't need to do anything here. We just want select to block until
// we connect or timeout.
case <-time.After(duration):
if c.conn != nil {
c.conn.Close()
}
if sshConn != nil {
sshConn.Close()
}
return ErrHandshakeTimeout
}
if err != nil {
return
}
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] handshake complete!")
if sshConn != nil {
c.client = ssh.NewClient(sshConn, sshChan, req)
}
2015-06-13 20:15:49 -04:00
c.connectToAgent()
return
}
2015-06-13 20:15:49 -04:00
func (c *comm) connectToAgent() {
if c.client == nil {
return
}
if c.config.DisableAgentForwarding {
2015-06-26 13:52:21 -04:00
log.Printf("[INFO] SSH agent forwarding is disabled.")
return
}
2015-06-13 20:15:49 -04:00
// open connection to the local agent
socketLocation := os.Getenv("SSH_AUTH_SOCK")
if socketLocation == "" {
2015-06-13 22:00:28 -04:00
log.Printf("[INFO] no local agent socket, will not connect agent")
2015-06-13 20:15:49 -04:00
return
}
agentConn, err := net.Dial("unix", socketLocation)
if err != nil {
2015-06-13 22:00:28 -04:00
log.Printf("[ERROR] could not connect to local agent socket: %s", socketLocation)
2015-06-13 20:15:49 -04:00
return
}
// create agent and add in auth
forwardingAgent := agent.NewClient(agentConn)
if forwardingAgent == nil {
2015-06-13 22:00:28 -04:00
log.Printf("[ERROR] Could not create agent client")
2015-06-13 20:15:49 -04:00
agentConn.Close()
return
}
// add callback for forwarding agent to SSH config
// XXX - might want to handle reconnects appending multiple callbacks
auth := ssh.PublicKeysCallback(forwardingAgent.Signers)
c.config.SSHConfig.Auth = append(c.config.SSHConfig.Auth, auth)
agent.ForwardToAgent(c.client, forwardingAgent)
// Setup a session to request agent forwarding
session, err := c.newSession()
if err != nil {
return
}
defer session.Close()
err = agent.RequestAgentForwarding(session)
if err != nil {
2015-06-13 22:00:28 -04:00
log.Printf("[ERROR] RequestAgentForwarding: %#v", err)
2015-06-13 20:15:49 -04:00
return
}
2015-06-13 22:00:28 -04:00
log.Printf("[INFO] agent forwarding enabled")
2015-06-13 20:15:49 -04:00
return
}
func (c *comm) sftpUploadSession(path string, input io.Reader, fi *os.FileInfo) error {
sftpFunc := func(client *sftp.Client) error {
2017-10-05 13:44:18 -04:00
return c.sftpUploadFile(path, input, client, fi)
}
return c.sftpSession(sftpFunc)
}
2017-10-05 13:44:18 -04:00
func (c *comm) sftpUploadFile(path string, input io.Reader, client *sftp.Client, fi *os.FileInfo) error {
log.Printf("[DEBUG] sftp: uploading %s", path)
f, err := client.Create(path)
if err != nil {
return err
}
defer f.Close()
if _, err = io.Copy(f, input); err != nil {
return err
}
if fi != nil && (*fi).Mode().IsRegular() {
mode := (*fi).Mode().Perm()
err = client.Chmod(path, mode)
if err != nil {
return err
}
}
return nil
}
func (c *comm) sftpUploadDirSession(dst string, src string, excl []string) error {
sftpFunc := func(client *sftp.Client) error {
rootDst := dst
if src[len(src)-1] != '/' {
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] No trailing slash, creating the source directory name")
rootDst = filepath.Join(dst, filepath.Base(src))
}
walkFunc := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Calculate the final destination using the
// base source and root destination
relSrc, err := filepath.Rel(src, path)
if err != nil {
return err
}
finalDst := filepath.Join(rootDst, relSrc)
// In Windows, Join uses backslashes which we don't want to get
// to the sftp server
finalDst = filepath.ToSlash(finalDst)
// Skip the creation of the target destination directory since
// it should exist and we might not even own it
if finalDst == dst {
return nil
}
2017-10-05 13:44:18 -04:00
return c.sftpVisitFile(finalDst, path, info, client)
}
return filepath.Walk(src, walkFunc)
}
return c.sftpSession(sftpFunc)
}
2017-10-05 13:44:18 -04:00
func (c *comm) sftpMkdir(path string, client *sftp.Client, fi os.FileInfo) error {
log.Printf("[DEBUG] sftp: creating dir %s", path)
if err := client.Mkdir(path); err != nil {
// Do not consider it an error if the directory existed
remoteFi, fiErr := client.Lstat(path)
if fiErr != nil || !remoteFi.IsDir() {
return err
}
}
mode := fi.Mode().Perm()
if err := client.Chmod(path, mode); err != nil {
return err
}
return nil
}
2017-10-05 13:44:18 -04:00
func (c *comm) sftpVisitFile(dst string, src string, fi os.FileInfo, client *sftp.Client) error {
if !fi.IsDir() {
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
2017-10-05 13:44:18 -04:00
return c.sftpUploadFile(dst, f, client, &fi)
} else {
2017-10-05 13:44:18 -04:00
err := c.sftpMkdir(dst, client, fi)
return err
}
}
func (c *comm) sftpDownloadSession(path string, output io.Writer) error {
sftpFunc := func(client *sftp.Client) error {
f, err := client.Open(path)
if err != nil {
return err
}
defer f.Close()
if _, err = io.Copy(output, f); err != nil {
return err
}
return nil
}
return c.sftpSession(sftpFunc)
}
func (c *comm) sftpSession(f func(*sftp.Client) error) error {
client, err := c.newSftpClient()
if err != nil {
return fmt.Errorf("sftpSession error: %s", err.Error())
}
defer client.Close()
return f(client)
}
func (c *comm) newSftpClient() (*sftp.Client, error) {
session, err := c.newSession()
if err != nil {
return nil, err
}
if err := session.RequestSubsystem("sftp"); err != nil {
return nil, err
}
pw, err := session.StdinPipe()
if err != nil {
return nil, err
}
pr, err := session.StdoutPipe()
if err != nil {
return nil, err
}
// Capture stdout so we can return errors to the user
var stdout bytes.Buffer
tee := io.TeeReader(pr, &stdout)
client, err := sftp.NewClientPipe(tee, pw)
if err != nil && stdout.Len() > 0 {
log.Printf("[ERROR] Upload failed: %s", stdout.Bytes())
}
return client, err
}
func (c *comm) scpUploadSession(path string, input io.Reader, fi *os.FileInfo) error {
// The target directory and file for talking the SCP protocol
target_dir := filepath.Dir(path)
target_file := filepath.Base(path)
2018-03-13 23:21:38 -04:00
// On windows, filepath.Dir uses backslash separators (ie. "\tmp").
// This does not work when the target host is unix. Switch to forward slash
// which works for unix and windows
target_dir = filepath.ToSlash(target_dir)
// Escape spaces in remote directory
target_dir = strings.Replace(target_dir, " ", "\\ ", -1)
scpFunc := func(w io.Writer, stdoutR *bufio.Reader) error {
return scpUploadFile(target_file, input, w, stdoutR, fi)
}
return c.scpSession("scp -vt "+target_dir, scpFunc)
}
func (c *comm) scpUploadDirSession(dst string, src string, excl []string) error {
scpFunc := func(w io.Writer, r *bufio.Reader) error {
uploadEntries := func() error {
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
entries, err := f.Readdir(-1)
if err != nil {
return err
}
return scpUploadDir(src, entries, w, r)
}
if src[len(src)-1] != '/' {
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] No trailing slash, creating the source directory name")
fi, err := os.Stat(src)
if err != nil {
return err
}
return scpUploadDirProtocol(filepath.Base(src), w, r, uploadEntries, fi)
} else {
// Trailing slash, so only upload the contents
return uploadEntries()
}
}
return c.scpSession("scp -rvt "+dst, scpFunc)
}
func (c *comm) scpDownloadSession(path string, output io.Writer) error {
scpFunc := func(w io.Writer, stdoutR *bufio.Reader) error {
fmt.Fprint(w, "\x00")
// read file info
fi, err := stdoutR.ReadString('\n')
if err != nil {
return err
}
if len(fi) < 0 {
return fmt.Errorf("empty response from server")
}
switch fi[0] {
case '\x01', '\x02':
2016-11-01 17:08:04 -04:00
return fmt.Errorf("%s", fi[1:])
case 'C':
case 'D':
return fmt.Errorf("remote file is directory")
default:
return fmt.Errorf("unexpected server response (%x)", fi[0])
}
var mode string
var size int64
n, err := fmt.Sscanf(fi, "%6s %d ", &mode, &size)
if err != nil || n != 2 {
return fmt.Errorf("can't parse server response (%s)", fi)
}
if size < 0 {
return fmt.Errorf("negative file size")
}
fmt.Fprint(w, "\x00")
if _, err := io.CopyN(output, stdoutR, size); err != nil {
return err
}
fmt.Fprint(w, "\x00")
2017-03-28 21:29:55 -04:00
return checkSCPStatus(stdoutR)
}
2017-03-28 21:29:55 -04:00
if !strings.Contains(path, " ") {
return c.scpSession("scp -vf "+path, scpFunc)
}
return c.scpSession("scp -vf "+strconv.Quote(path), scpFunc)
}
func (c *comm) scpSession(scpCommand string, f func(io.Writer, *bufio.Reader) error) error {
session, err := c.newSession()
if err != nil {
return err
}
defer session.Close()
// Get a pipe to stdin so that we can send data down
stdinW, err := session.StdinPipe()
if err != nil {
return err
}
// We only want to close once, so we nil w after we close it,
// and only close in the defer if it hasn't been closed already.
defer func() {
if stdinW != nil {
stdinW.Close()
}
}()
// Get a pipe to stdout so that we can get responses back
stdoutPipe, err := session.StdoutPipe()
if err != nil {
return err
}
stdoutR := bufio.NewReader(stdoutPipe)
// Set stderr to a bytes buffer
stderr := new(bytes.Buffer)
session.Stderr = stderr
// Start the sink mode on the other side
// TODO(mitchellh): There are probably issues with shell escaping the path
2018-03-20 19:28:23 -04:00
log.Println("[DEBUG] Starting remote scp process: ", scpCommand)
if err := session.Start(scpCommand); err != nil {
return err
}
// Call our callback that executes in the context of SCP. We ignore
// EOF errors if they occur because it usually means that SCP prematurely
// ended on the other side.
2018-03-20 19:28:23 -04:00
log.Println("[DEBUG] Started SCP session, beginning transfers...")
if err := f(stdinW, stdoutR); err != nil && err != io.EOF {
return err
}
// Close the stdin, which sends an EOF, and then set w to nil so that
// our defer func doesn't close it again since that is unsafe with
// the Go SSH package.
2018-03-20 19:28:23 -04:00
log.Println("[DEBUG] SCP session complete, closing stdin pipe.")
stdinW.Close()
stdinW = nil
// Wait for the SCP connection to close, meaning it has consumed all
// our data and has completed. Or has errored.
2018-03-20 19:28:23 -04:00
log.Println("[DEBUG] Waiting for SSH session to complete.")
err = session.Wait()
if err != nil {
if exitErr, ok := err.(*ssh.ExitError); ok {
2018-03-13 03:45:13 -04:00
// Otherwise, we have an ExitError, meaning we can just read
// the exit status
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] non-zero exit status: %d", exitErr.ExitStatus())
2017-08-10 14:46:38 -04:00
stdoutB, err := ioutil.ReadAll(stdoutR)
if err != nil {
return err
}
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] scp output: %s", stdoutB)
// If we exited with status 127, it means SCP isn't available.
// Return a more descriptive error for that.
if exitErr.ExitStatus() == 127 {
return errors.New(
"SCP failed to start. This usually means that SCP is not\n" +
"properly installed on the remote system.")
}
}
return err
}
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] scp stderr (length %d): %s", stderr.Len(), stderr.String())
return nil
}
// checkSCPStatus checks that a prior command sent to SCP completed
// successfully. If it did not complete successfully, an error will
// be returned.
func checkSCPStatus(r *bufio.Reader) error {
code, err := r.ReadByte()
if err != nil {
return err
}
if code != 0 {
// Treat any non-zero (really 1 and 2) as fatal errors
message, _, err := r.ReadLine()
if err != nil {
return fmt.Errorf("Error reading error message: %s", err)
}
return errors.New(string(message))
}
return nil
}
func scpDownloadFile(dst string, src io.Reader, size int64, mode os.FileMode) error {
f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
return err
}
defer f.Close()
if _, err := io.CopyN(f, src, size); err != nil {
return err
}
return nil
}
func scpUploadFile(dst string, src io.Reader, w io.Writer, r *bufio.Reader, fi *os.FileInfo) error {
var mode os.FileMode
var size int64
if fi != nil && (*fi).Mode().IsRegular() {
mode = (*fi).Mode().Perm()
size = (*fi).Size()
} else {
// Create a temporary file where we can copy the contents of the src
// so that we can determine the length, since SCP is length-prefixed.
Add tmp package that offers Dir & File funcs this regroups the calls of tmp func calls into one package. the tmp pkg allows to store tmp files under a common directory for easier managment Squashed commit of the following: commit 74b674c01560c6b547e9cab5e8afb486f0fe7d6c Merge: fc94b5714 66001525d Author: Matthew Patton <mpatton@enquizit.com> Date: Fri Nov 2 02:02:38 2018 -0400 Merge branch 'remove_TMPDIR_docs' of github.com:tb3088/packer into remove_TMPDIR_docs commit fc94b571428447aaf2e5171967c95cdf9e5c73ff Author: Matthew Patton <mpatton@enquizit.com> Date: Fri Nov 2 01:42:38 2018 -0400 update tests to honor configured temporary directory commit 5a562359f4937af506607dd96ed1c46426093f52 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 18:52:38 2018 -0400 sync straggler to refactor. implement tests at packer/configfile instead of buried in docker. commit 9d5a4a0935bfd049e2f8f50ad73e4455b42c9410 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 15:39:46 2018 -0400 insert URL to GoLang API for os.TempDir() commit fbae4c2e051898aecd8705e9d8a20353430ede23 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 07:41:29 2018 -0400 revise documentation to match code commit 3590fae8bd0afa92a161a49ed794ee32e357d583 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 07:21:59 2018 -0400 refacter config_file. replace all hard-coded os.TempDir with wrapper commit d5c5306a97b648ded06a8eb76ce728f07c9924f0 Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 20:11:36 2018 -0400 close massive file overwrite hole with TempDir commit 0a72297da86089fbd07e4f7d9472a92fa4d1c733 Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 01:06:00 2018 -0400 adjust var declaration scope commit 20f68228b6c372d984ea5b055cfc8bda9b110ac5 Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 00:54:35 2018 -0400 use mktemp() equivalent to create temporary directory commit c73ebe3d8321664a2685a0baa8e441dd895b0db4 Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 20:40:58 2018 -0400 remove extraneous variable declaration, fix FOR loop commit 63549b8bd745a0090b15ed0b0ebf644162db27db Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 19:30:44 2018 -0400 match styistic convension with rest of docs commit 976101074942db36f10d3a686d6d29ddb7c01926 Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 19:08:39 2018 -0400 revert dangling config file change commit 95159afbc05eac96ed11c3183af5fb0abe2f2d5c Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Fri Dec 29 23:53:43 2017 -0500 replace invalid TMPDIR variable wth PACKER_TMP_DIR. update ConfigTmpDir() to try common temporary paths first and only write to configDir() as a last resort. commit 66001525d72de56a4cf9339b900c46f59bc6e91a Author: Matthew Patton <mpatton@enquizit.com> Date: Fri Nov 2 01:42:38 2018 -0400 update tests to honor configured temporary directory commit e9b6adefeae8c65eb8aa47fef38cbf0aa424338c Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 18:52:38 2018 -0400 sync straggler to refactor. implement tests at packer/configfile instead of buried in docker. commit 852113ed076e2d14e5dca6815ea680da1e2896bb Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 15:39:46 2018 -0400 insert URL to GoLang API for os.TempDir() commit 54add38d1d1c6e283cd444b367ed8bd49a5f3699 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 07:41:29 2018 -0400 revise documentation to match code commit 6b5b8f6d4edad0e187ca13d64ac4118f34eee643 Author: Matthew Patton <mpatton@enquizit.com> Date: Thu Nov 1 07:21:59 2018 -0400 refacter config_file. replace all hard-coded os.TempDir with wrapper commit c22092c601c33484327674f322c7379fa41506d7 Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 20:11:36 2018 -0400 close massive file overwrite hole with TempDir commit 7a730450916bf8e5dbc2a741ec233a49466ab7cc Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 01:06:00 2018 -0400 adjust var declaration scope commit 0f2933adb6f6922dfeab78a95371a444ec8918ab Author: Matthew Patton <mpatton@enquizit.com> Date: Wed Oct 31 00:54:35 2018 -0400 use mktemp() equivalent to create temporary directory commit d74839ede05dacf712b55a7bb48aec19fe6b007f Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 20:40:58 2018 -0400 remove extraneous variable declaration, fix FOR loop commit eb65416619437e4a3dec90277770803dd5b2281c Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 19:30:44 2018 -0400 match styistic convension with rest of docs commit acaa2b31ed463219c4ef099f351eec72406e2989 Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 19:08:39 2018 -0400 revert dangling config file change commit e573fde668dcb418561e61535c1d68b2888f5b0f Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Fri Dec 29 23:53:43 2017 -0500 replace invalid TMPDIR variable wth PACKER_TMP_DIR. update ConfigTmpDir() to try common temporary paths first and only write to configDir() as a last resort. commit 39a9874afc9dea71b36753375fb7feb61ffecf69 Merge: 8a413cfe8 3315812c2 Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Oct 30 18:19:26 2018 -0400 Merge branch 'master' of https://github.com/hashicorp/packer into prestine commit 8a413cfe83b0a70dbd99c32d936334ca5788ca9b Merge: e07491de5 4e14710a6 Author: Matthew Patton <mpatton@enquizit.com> Date: Mon Oct 1 20:18:10 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit e07491de59cead0b337a7f57f4a6c625e1f560ab Merge: 42610a35d a1fa35dff Author: Matthew Patton <mpatton@enquizit.com> Date: Tue Aug 21 13:26:19 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit 42610a35d5ef65af8844a58ad70e2ec75262da6a Merge: 5298142da 0d63cf7bc Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Tue Jun 19 22:45:05 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit 5298142da6da95354844f201deeeef3c599e48b2 Merge: 7bb110bc7 9d9736552 Author: Matthew Patton <mpatton@enquizit.com> Date: Mon Jun 11 15:10:09 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit 7bb110bc744c9797fcaec25a265d9b85a22d6d6b Merge: a61c869ca 6189d66e7 Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Wed May 9 23:41:22 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit a61c869ca7268acf0e3b1e1fa7a8ee7feea65984 Merge: 098101dd9 4be5f072c Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Fri May 4 10:13:04 2018 -0400 Merge remote-tracking branch 'upstream/master' into prestine commit 098101dd99f08b0ca110d33eff6904537c6d21de Merge: fefaf0fa6 554b2b4a5 Author: Matthew Patton <matthew.patton@itgfirm.com> Date: Tue May 1 20:52:30 2018 -0400 Merge branch 'EOL-handling' into prestine Co-Authored-By: Matthew Patton <pattonme@yahoo.com>
2018-12-12 09:45:00 -05:00
tf, err := tmp.File("packer-upload")
if err != nil {
return fmt.Errorf("Error creating temporary file for upload: %s", err)
}
defer os.Remove(tf.Name())
defer tf.Close()
mode = 0644
2018-03-20 19:28:23 -04:00
log.Println("[DEBUG] Copying input data into temporary file so we can read the length")
if _, err := io.Copy(tf, src); err != nil {
return fmt.Errorf("Error copying input data into local temporary "+
"file. Check that TEMPDIR has enough space. Please see "+
"https://www.packer.io/docs/other/environment-variables.html#tmpdir"+
"for more info. Error: %s", err)
}
// Sync the file so that the contents are definitely on disk, then
// read the length of it.
if err := tf.Sync(); err != nil {
return fmt.Errorf("Error creating temporary file for upload: %s", err)
}
// Seek the file to the beginning so we can re-read all of it
if _, err := tf.Seek(0, 0); err != nil {
return fmt.Errorf("Error creating temporary file for upload: %s", err)
}
tfi, err := tf.Stat()
if err != nil {
return fmt.Errorf("Error creating temporary file for upload: %s", err)
}
size = tfi.Size()
src = tf
}
// Start the protocol
perms := fmt.Sprintf("C%04o", mode)
log.Printf("[DEBUG] scp: Uploading %s: perms=%s size=%d", dst, perms, size)
fmt.Fprintln(w, perms, size, dst)
if err := checkSCPStatus(r); err != nil {
return err
}
if _, err := io.CopyN(w, src, size); err != nil {
return err
}
fmt.Fprint(w, "\x00")
2017-03-28 21:29:55 -04:00
return checkSCPStatus(r)
}
func scpUploadDirProtocol(name string, w io.Writer, r *bufio.Reader, f func() error, fi os.FileInfo) error {
2018-03-20 19:28:23 -04:00
log.Printf("[DEBUG] SCP: starting directory upload: %s", name)
mode := fi.Mode().Perm()
perms := fmt.Sprintf("D%04o 0", mode)
fmt.Fprintln(w, perms, name)
err := checkSCPStatus(r)
if err != nil {
return err
}
if err := f(); err != nil {
return err
}
fmt.Fprintln(w, "E")
2017-03-28 21:29:55 -04:00
return err
}
func scpUploadDir(root string, fs []os.FileInfo, w io.Writer, r *bufio.Reader) error {
for _, fi := range fs {
realPath := filepath.Join(root, fi.Name())
// Track if this is actually a symlink to a directory. If it is
// a symlink to a file we don't do any special behavior because uploading
// a file just works. If it is a directory, we need to know so we
// treat it as such.
isSymlinkToDir := false
2013-09-30 14:39:52 -04:00
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
symPath, err := filepath.EvalSymlinks(realPath)
if err != nil {
return err
}
symFi, err := os.Lstat(symPath)
if err != nil {
return err
}
isSymlinkToDir = symFi.IsDir()
}
if !fi.IsDir() && !isSymlinkToDir {
// It is a regular file (or symlink to a file), just upload it
f, err := os.Open(realPath)
if err != nil {
return err
}
err = func() error {
defer f.Close()
return scpUploadFile(fi.Name(), f, w, r, &fi)
}()
if err != nil {
return err
}
continue
}
// It is a directory, recursively upload
err := scpUploadDirProtocol(fi.Name(), w, r, func() error {
f, err := os.Open(realPath)
if err != nil {
return err
}
defer f.Close()
entries, err := f.Readdir(-1)
if err != nil {
return err
}
return scpUploadDir(realPath, entries, w, r)
}, fi)
if err != nil {
return err
}
}
return nil
}