packer-cn/builder/vmware/common/driver_esx5.go

package common

import (
	"bufio"
	"bytes"
	"context"
	"encoding/csv"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/go-getter/v2"
	"github.com/hashicorp/packer/communicator/ssh"
	"github.com/hashicorp/packer/helper/communicator"
	"github.com/hashicorp/packer/helper/multistep"
	helperssh "github.com/hashicorp/packer/helper/ssh"
	"github.com/hashicorp/packer/packer"
	gossh "golang.org/x/crypto/ssh"
)

// ESX5 driver talks to an ESXi5 hypervisor remotely over SSH to build
// virtual machines. This driver can only manage one machine at a time.
type ESX5Driver struct {
	base VmwareDriver

	Host           string
	Port           int
	Username       string
	Password       string
	PrivateKeyFile string
	Datastore      string
	CacheDatastore string
	CacheDirectory string
	VMName         string
	CommConfig     communicator.Config

	comm      packer.Communicator
	outputDir string
	vmId      string
}

func (d *ESX5Driver) Clone(dst, src string, linked bool) error {

	linesToArray := func(lines string) []string { return strings.Split(strings.Trim(lines, "\r\n"), "\n") }

	d.SetOutputDir(path.Dir(filepath.ToSlash(dst)))
	srcVmx := d.datastorePath(src)
	dstVmx := d.datastorePath(dst)
	srcDir := path.Dir(srcVmx)
	dstDir := path.Dir(dstVmx)

	log.Printf("Source: %s\n", srcVmx)
	log.Printf("Dest: %s\n", dstVmx)

	err := d.MkdirAll()
	if err != nil {
		return fmt.Errorf("Failed to create the destination directory %s: %s", d.outputDir, err)
	}

	err = d.sh("cp", strconv.Quote(srcVmx), strconv.Quote(dstVmx))
	if err != nil {
		return fmt.Errorf("Failed to copy the vmx file %s: %s", srcVmx, err)
	}

	filesToClone, err := d.run(nil, "find", strconv.Quote(srcDir), "! -name '*.vmdk' ! -name '*.vmx' ! -name '*.vmxf' -type f ! -size 0")
	if err != nil {
		return fmt.Errorf("Failed to get the file list to copy: %s", err)
	}

	for _, f := range linesToArray(filesToClone) {
		// TODO: linesToArray should really return [] if the string is empty. Instead it returns [""]
		if f == "" {
			continue
		}
		err := d.sh("cp", strconv.Quote(f), strconv.Quote(dstDir))
		if err != nil {
			return fmt.Errorf("Failing to copy %s to %s: %s", f, dstDir, err)
		}
	}

	disksToClone, err := d.run(nil, "sed -ne 's/.*file[Nn]ame = \"\\(.*vmdk\\)\"/\\1/p'", strconv.Quote(srcVmx))
	if err != nil {
		return fmt.Errorf("Failing to get the vmdk list to clone %s", err)
	}
	for _, disk := range linesToArray(disksToClone) {
		srcDisk := path.Join(srcDir, disk)
		if path.IsAbs(disk) {
			srcDisk = disk
		}
		destDisk := path.Join(dstDir, path.Base(disk))
		err = d.sh("vmkfstools", "-d thin", "-i", strconv.Quote(srcDisk), strconv.Quote(destDisk))
		if err != nil {
			return fmt.Errorf("Failing to clone disk %s: %s", srcDisk, err)
		}
	}
	log.Printf("Successfully cloned %s to %s\n", src, dst)
	return nil
}

func (d *ESX5Driver) CompactDisk(diskPathLocal string) error {
	diskPath := d.datastorePath(diskPathLocal)
	return d.sh("vmkfstools", "--punchzero", strconv.Quote(diskPath))
}

func (d *ESX5Driver) CreateDisk(diskPathLocal string, size string, adapter_type string, typeId string) error {
	diskPath := strconv.Quote(d.datastorePath(diskPathLocal))
	return d.sh("vmkfstools", "-c", size, "-d", typeId, "-a", adapter_type, diskPath)
}

func (d *ESX5Driver) IsRunning(string) (bool, error) {
	state, err := d.run(nil, "vim-cmd", "vmsvc/power.getstate", d.vmId)
	if err != nil {
		return false, err
	}
	return strings.Contains(state, "Powered on"), nil
}

func (d *ESX5Driver) ReloadVM() error {
	if d.vmId != "" {
		return d.sh("vim-cmd", "vmsvc/reload", d.vmId)
	} else {
		return nil
	}
}

func (d *ESX5Driver) Start(vmxPathLocal string, headless bool) error {
	for i := 0; i < 20; i++ {
		//intentionally not checking for error since poweron may fail specially after initial VM registration
		d.sh("vim-cmd", "vmsvc/power.on", d.vmId)
		time.Sleep((time.Duration(i) * time.Second) + 1)
		running, err := d.IsRunning(vmxPathLocal)
		if err != nil {
			return err
		}
		if running {
			return nil
		}
	}
	return errors.New("Retry limit exceeded")
}

func (d *ESX5Driver) Stop(vmxPathLocal string) error {
	return d.sh("vim-cmd", "vmsvc/power.off", d.vmId)
}

func (d *ESX5Driver) Register(vmxPathLocal string) error {
	vmxPath := filepath.ToSlash(filepath.Join(d.outputDir, filepath.Base(vmxPathLocal)))
	if err := d.upload(vmxPath, vmxPathLocal); err != nil {
		return err
	}
	r, err := d.run(nil, "vim-cmd", "solo/registervm", strconv.Quote(vmxPath))
	if err != nil {
		return err
	}
	d.vmId = strings.TrimRight(r, "\n")
	return nil
}

func (d *ESX5Driver) SuppressMessages(vmxPath string) error {
	return nil
}

func (d *ESX5Driver) Unregister(vmxPathLocal string) error {
	return d.sh("vim-cmd", "vmsvc/unregister", d.vmId)
}

func (d *ESX5Driver) Destroy() error {
	return d.sh("vim-cmd", "vmsvc/destroy", d.vmId)
}

func (d *ESX5Driver) IsDestroyed() (bool, error) {
	err := d.sh("test", "!", "-e", strconv.Quote(d.outputDir))
	if err != nil {
		return false, err
	}
	return true, err
}

func (d *ESX5Driver) UploadISO(localPath string, checksum string) (string, error) {
	finalPath := d.CachePath(localPath)
	if err := d.mkdir(filepath.ToSlash(filepath.Dir(finalPath))); err != nil {
		return "", err
	}

	log.Printf("Verifying checksum of %s", finalPath)
	if d.VerifyChecksum(checksum, finalPath) {
		log.Println("Initial checksum matched, no upload needed.")
		return finalPath, nil
	}
	log.Println("Initial checksum did not match, uploading.")

	if err := d.upload(finalPath, localPath); err != nil {
		return "", err
	}

	if !d.VerifyChecksum(checksum, finalPath) {
		e := fmt.Errorf("Checksum did not match after upload.")
		log.Println(e)
		return "", e
	}

	return finalPath, nil
}

func (d *ESX5Driver) RemoveCache(localPath string) error {
	finalPath := d.CachePath(localPath)
	log.Printf("Removing remote cache path %s (local %s)", finalPath, localPath)
	return d.sh("rm", "-f", strconv.Quote(finalPath))
}

func (d *ESX5Driver) ToolsIsoPath(string) string {
	return ""
}

func (d *ESX5Driver) ToolsInstall() error {
	return d.sh("vim-cmd", "vmsvc/tools.install", d.vmId)
}

func (d *ESX5Driver) Verify() error {
	// Ensure that NetworkMapper is nil, since the mapping of device<->network
	// is handled by ESX and thus can't be performed by packer unless we
	// query things.

	// FIXME: If we want to expose the network devices to the user, then we can
	// probably use esxcli to enumerate the portgroup and switchId
	d.base.NetworkMapper = nil

	// Be safe/friendly and overwrite the rest of the utility functions with
	// log functions despite the fact that these shouldn't be called anyways.
	d.base.DhcpLeasesPath = func(device string) string {
		log.Printf("Unexpected error, ESX5 driver attempted to call DhcpLeasesPath(%#v)\n", device)
		return ""
	}
	d.base.DhcpConfPath = func(device string) string {
		log.Printf("Unexpected error, ESX5 driver attempted to call DhcpConfPath(%#v)\n", device)
		return ""
	}
	d.base.VmnetnatConfPath = func(device string) string {
		log.Printf("Unexpected error, ESX5 driver attempted to call VmnetnatConfPath(%#v)\n", device)
		return ""
	}

	checks := []func() error{
		d.connect,
		d.checkSystemVersion,
		d.checkGuestIPHackEnabled,
	}

	for _, check := range checks {
		if err := check(); err != nil {
			return err
		}
	}
	return nil
}

func (d *ESX5Driver) HostIP(multistep.StateBag) (string, error) {
	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", d.Host, d.Port))
	if err != nil {
		return "", err
	}
	defer conn.Close()

	host, _, err := net.SplitHostPort(conn.LocalAddr().String())
	return host, err
}

func (d *ESX5Driver) PotentialGuestIP(multistep.StateBag) ([]string, error) {
	// GuestIP is defined by the user as d.Host..but let's validate it just to be sure
	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", d.Host, d.Port))
	if err != nil {
		return []string{}, err
	}
	defer conn.Close()

	host, _, err := net.SplitHostPort(conn.RemoteAddr().String())
	return []string{host}, err
}

func (d *ESX5Driver) HostAddress(multistep.StateBag) (string, error) {
	// make a connection
	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", d.Host, d.Port))
	if err != nil {
		return "", err
	}
	defer conn.Close()

	// get the local address (the host)
	host, _, err := net.SplitHostPort(conn.LocalAddr().String())
	if err != nil {
		return "", fmt.Errorf("Unable to determine host address for ESXi: %v", err)
	}

	// iterate through all the interfaces..
	interfaces, err := net.Interfaces()
	if err != nil {
		return "", fmt.Errorf("Unable to enumerate host interfaces : %v", err)
	}

	for _, intf := range interfaces {
		addrs, err := intf.Addrs()
		if err != nil {
			continue
		}

		// ..checking to see if any if it's addrs match the host address
		for _, addr := range addrs {
			if addr.String() == host { // FIXME: Is this the proper way to compare two HardwareAddrs?
				return intf.HardwareAddr.String(), nil
			}
		}
	}

	// ..unfortunately nothing was found
	return "", fmt.Errorf("Unable to locate interface matching host address in ESXi: %v", host)
}

func (d *ESX5Driver) GuestAddress(multistep.StateBag) (string, error) {
	// list all the interfaces on the esx host
	r, err := d.esxcli("network", "ip", "interface", "list")
	if err != nil {
		return "", fmt.Errorf("Could not retrieve network interfaces for ESXi: %v", err)
	}

	// rip out the interface name and the MAC address from the csv output
	addrs := make(map[string]string)
	for record, err := r.read(); record != nil && err == nil; record, err = r.read() {
		if strings.ToUpper(record["Enabled"]) != "TRUE" {
			continue
		}
		addrs[record["Name"]] = record["MAC Address"]
	}

	// list all the addresses on the esx host
	r, err = d.esxcli("network", "ip", "interface", "ipv4", "get")
	if err != nil {
		return "", fmt.Errorf("Could not retrieve network addresses for ESXi: %v", err)
	}

	// figure out the interface name that matches the specified d.Host address
	var intf string
	intf = ""
	for record, err := r.read(); record != nil && err == nil; record, err = r.read() {
		if record["IPv4 Address"] == d.Host && record["Name"] != "" {
			intf = record["Name"]
			break
		}
	}
	if intf == "" {
		return "", fmt.Errorf("Unable to find matching address for ESXi guest")
	}

	// find the MAC address according to the interface name
	result, ok := addrs[intf]
	if !ok {
		return "", fmt.Errorf("Unable to find address for ESXi guest interface")
	}

	// ..and we're good
	return result, nil
}

func (d *ESX5Driver) VNCAddress(ctx context.Context, _ string, portMin, portMax int) (string, int, error) {
	var vncPort int

	//Process ports ESXi is listening on to determine which are available
	//This process does best effort to detect ports that are unavailable,
	//it will ignore any ports listened to by only localhost
	r, err := d.esxcli("network", "ip", "connection", "list")
	if err != nil {
		err = fmt.Errorf("Could not retrieve network information for ESXi: %v", err)
		return "", 0, err
	}

	listenPorts := make(map[string]bool)
	for record, err := r.read(); record != nil && err == nil; record, err = r.read() {
		if record["State"] == "LISTEN" {
			splitAddress := strings.Split(record["LocalAddress"], ":")
			if splitAddress[0] != "127.0.0.1" {
				port := splitAddress[len(splitAddress)-1]
				log.Printf("ESXi listening on address %s, port %s unavailable for VNC", record["LocalAddress"], port)
				listenPorts[port] = true
			}
		}
	}

	vncTimeout := time.Duration(15 * time.Second)
	envTimeout := os.Getenv("PACKER_ESXI_VNC_PROBE_TIMEOUT")
	if envTimeout != "" {
		if parsedTimeout, err := time.ParseDuration(envTimeout); err != nil {
			log.Printf("Error parsing PACKER_ESXI_VNC_PROBE_TIMEOUT. Falling back to default (15s). %s", err)
		} else {
			vncTimeout = parsedTimeout
		}
	}

	for port := portMin; port <= portMax; port++ {
		if _, ok := listenPorts[fmt.Sprintf("%d", port)]; ok {
			log.Printf("Port %d in use", port)
			continue
		}
		address := fmt.Sprintf("%s:%d", d.Host, port)
		log.Printf("Trying address: %s...", address)
		l, err := net.DialTimeout("tcp", address, vncTimeout)

		if err != nil {
			if e, ok := err.(*net.OpError); ok {
				if e.Timeout() {
					log.Printf("Timeout connecting to: %s (check firewall rules)", address)
				} else {
					vncPort = port
					break
				}
			}
		} else {
			defer l.Close()
		}
	}

	if vncPort == 0 {
		err := fmt.Errorf("Unable to find available VNC port between %d and %d",
			portMin, portMax)
		return d.Host, vncPort, err
	}

	return d.Host, vncPort, nil
}

// UpdateVMX, adds the VNC port to the VMX data.
func (ESX5Driver) UpdateVMX(_, password string, port int, data map[string]string) {
	// Do not set remotedisplay.vnc.ip - this breaks ESXi.
	data["remotedisplay.vnc.enabled"] = "TRUE"
	data["remotedisplay.vnc.port"] = fmt.Sprintf("%d", port)
	if len(password) > 0 {
		data["remotedisplay.vnc.password"] = password
	}
}

func (d *ESX5Driver) CommHost(state multistep.StateBag) (string, error) {
	sshc := state.Get("sshConfig").(*SSHConfig).Comm
	port := sshc.Port()

	if address, ok := state.GetOk("vm_address"); ok {
		return address.(string), nil
	}

	if address := d.CommConfig.Host(); address != "" {
		state.Put("vm_address", address)
		return address, nil
	}

	r, err := d.esxcli("network", "vm", "list")
	if err != nil {
		return "", err
	}

	// The value in the Name field returned by 'esxcli network vm list'
	// corresponds directly to the value of displayName set in the VMX file
	var displayName string
	if v, ok := state.GetOk("display_name"); ok {
		displayName = v.(string)
	} else {
		displayName = strings.Replace(d.VMName, " ", "_", -1)
		log.Printf("No display_name set; falling back to using VMName %s "+
			"to look for SSH IP", displayName)
	}

	record, err := r.find("Name", displayName)
	if err != nil {
		return "", err
	}
	wid := record["WorldID"]
	if wid == "" {
		return "", errors.New("VM WorldID not found")
	}

	r, err = d.esxcli("network", "vm", "port", "list", "-w", wid)
	if err != nil {
		return "", err
	}

	// Loop through interfaces
	for {
		record, err = r.read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return "", err
		}

		if record["IPAddress"] == "0.0.0.0" {
			continue
		}

		// if ssh is going through a bastion, we can't easily check if the nic is reachable on the network
		// so just pick the first one that is not 0.0.0.0
		if sshc.SSHBastionHost != "" {
			address := record["IPAddress"]
			state.Put("vm_address", address)
			return address, nil
		}

		// When multiple NICs are connected to the same network, choose
		// one that has a route back. This Dial should ensure that.
		conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", record["IPAddress"], port), 2*time.Second)
		if err != nil {
			if e, ok := err.(*net.OpError); ok {
				if e.Timeout() {
					log.Printf("Timeout connecting to %s", record["IPAddress"])
					continue
				}
			}
		} else {
			defer conn.Close()
			address := record["IPAddress"]
			state.Put("vm_address", address)
			return address, nil
		}
	}
	return "", errors.New("No interface on the VM has an IP address ready")
}

//-------------------------------------------------------------------
// OutputDir implementation
//-------------------------------------------------------------------

func (d *ESX5Driver) DirExists() (bool, error) {
	err := d.sh("test", "-e", strconv.Quote(d.outputDir))
	return err == nil, nil
}

func (d *ESX5Driver) ListFiles() ([]string, error) {
	stdout, err := d.ssh("ls -1p "+strconv.Quote(d.outputDir), nil)
	if err != nil {
		return nil, err
	}

	files := make([]string, 0, 10)
	reader := bufio.NewReader(stdout)
	for {
		line, _, err := reader.ReadLine()
		if err == io.EOF {
			break
		}
		if line[len(line)-1] == '/' {
			continue
		}

		files = append(files, filepath.ToSlash(filepath.Join(d.outputDir, string(line))))
	}

	return files, nil
}

func (d *ESX5Driver) MkdirAll() error {
	return d.mkdir(d.outputDir)
}

func (d *ESX5Driver) Remove(path string) error {
	return d.sh("rm", strconv.Quote(path))
}

func (d *ESX5Driver) RemoveAll() error {
	return d.sh("rm", "-rf", strconv.Quote(d.outputDir))
}

func (d *ESX5Driver) SetOutputDir(path string) {
	d.outputDir = d.datastorePath(path)
}

func (d *ESX5Driver) String() string {
	return d.outputDir
}

func (d *ESX5Driver) datastorePath(path string) string {
	dirPath := filepath.Dir(path)
	return filepath.ToSlash(filepath.Join("/vmfs/volumes", d.Datastore, dirPath, filepath.Base(path)))
}

func (d *ESX5Driver) CachePath(path string) string {
	return filepath.ToSlash(filepath.Join("/vmfs/volumes", d.CacheDatastore, d.CacheDirectory, filepath.Base(path)))
}

func (d *ESX5Driver) connect() error {
	address := fmt.Sprintf("%s:%d", d.Host, d.Port)

	auth := []gossh.AuthMethod{
		gossh.Password(d.Password),
		gossh.KeyboardInteractive(
			ssh.PasswordKeyboardInteractive(d.Password)),
	}

	if d.PrivateKeyFile != "" {
		signer, err := helperssh.FileSigner(d.PrivateKeyFile)
		if err != nil {
			return err
		}

		auth = append(auth, gossh.PublicKeys(signer))
	}

	sshConfig := &ssh.Config{
		Connection: ssh.ConnectFunc("tcp", address),
		SSHConfig: &gossh.ClientConfig{
			User:            d.Username,
			Auth:            auth,
			HostKeyCallback: gossh.InsecureIgnoreHostKey(),
		},
	}

	comm, err := ssh.New(address, sshConfig)
	if err != nil {
		return err
	}

	d.comm = comm
	return nil
}

func (d *ESX5Driver) checkSystemVersion() error {
	r, err := d.esxcli("system", "version", "get")
	if err != nil {
		return err
	}

	record, err := r.read()
	if err != nil {
		return err
	}

	log.Printf("Connected to %s %s %s", record["Product"],
		record["Version"], record["Build"])
	return nil
}

func (d *ESX5Driver) checkGuestIPHackEnabled() error {
	r, err := d.esxcli("system", "settings", "advanced", "list", "-o", "/Net/GuestIPHack")
	if err != nil {
		return err
	}

	record, err := r.read()
	if err != nil {
		return err
	}

	if record["IntValue"] != "1" {
		return errors.New(
			"GuestIPHack is required, enable by running this on the ESX machine:\n" +
				"esxcli system settings advanced set -o /Net/GuestIPHack -i 1")
	}

	return nil
}

func (d *ESX5Driver) mkdir(path string) error {
	return d.sh("mkdir", "-p", strconv.Quote(path))
}

func (d *ESX5Driver) upload(dst, src string) error {
	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()
	return d.comm.Upload(dst, f, nil)
}

func (d *ESX5Driver) Download(src, dst string) error {
	file, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer file.Close()
	return d.comm.Download(d.datastorePath(src), file)
}

// VerifyChecksum checks that file on the esxi instance matches hash
func (d *ESX5Driver) VerifyChecksum(hash string, file string) bool {
	if hash == "none" {
		if err := d.sh("stat", strconv.Quote(file)); err != nil {
			return false
		}
		return true
	}

	// parse user checksum
	fcksum, err := getter.DefaultClient.GetChecksum(context.TODO(), &getter.Request{
		Src: file + "?checksum=" + hash,
	})
	if err != nil {
		log.Printf("coulnd't parse the checksum: %v", err)
		return false
	}

	checksumEntry := fmt.Sprintf("%s  %s", hex.EncodeToString(fcksum.Value), file)
	checksumCommand := []string{fmt.Sprintf("%ssum", fcksum.Type), "-c"}

	log.Printf("running: %s | %s", checksumEntry, checksumCommand)

	_, err = d.run(bytes.NewBufferString(checksumEntry), checksumCommand...)
	if err != nil {
		log.Printf("checksum failed: %s", err)
	}

	return err == nil
}

func (d *ESX5Driver) ssh(command string, stdin io.Reader) (*bytes.Buffer, error) {
	ctx := context.TODO()
	var stdout, stderr bytes.Buffer

	cmd := &packer.RemoteCmd{
		Command: command,
		Stdout:  &stdout,
		Stderr:  &stderr,
		Stdin:   stdin,
	}

	err := d.comm.Start(ctx, cmd)
	if err != nil {
		return nil, err
	}

	cmd.Wait()

	if cmd.ExitStatus() != 0 {
		err = fmt.Errorf("'%s'\n\nStdout: %s\n\nStderr: %s",
			cmd.Command, stdout.String(), stderr.String())
		return nil, err
	}

	return &stdout, nil
}

func (d *ESX5Driver) run(stdin io.Reader, args ...string) (string, error) {
	stdout, err := d.ssh(strings.Join(args, " "), stdin)
	if err != nil {
		return "", err
	}
	return stdout.String(), nil
}

func (d *ESX5Driver) sh(args ...string) error {
	_, err := d.run(nil, args...)
	return err
}

func (d *ESX5Driver) esxcli(args ...string) (*esxcliReader, error) {
	stdout, err := d.ssh("esxcli --formatter csv "+strings.Join(args, " "), nil)
	if err != nil {
		return nil, err
	}
	r := csv.NewReader(bytes.NewReader(stdout.Bytes()))
	r.TrailingComma = true
	header, err := r.Read()
	if err != nil {
		return nil, err
	}
	return &esxcliReader{r, header}, nil
}

func (d *ESX5Driver) GetVmwareDriver() VmwareDriver {
	return d.base
}

type esxcliReader struct {
	cr     *csv.Reader
	header []string
}

func (r *esxcliReader) read() (map[string]string, error) {
	fields, err := r.cr.Read()

	if err != nil {
		return nil, err
	}

	record := map[string]string{}
	for i, v := range fields {
		record[r.header[i]] = v
	}

	return record, nil
}

func (r *esxcliReader) find(key, val string) (map[string]string, error) {
	for {
		record, err := r.read()
		if err != nil {
			return nil, err
		}
		if record[key] == val {
			return record, nil
		}
	}
}