builder/vmware: new driver to support building images directly on ESX

This driver talks directly to ESX over ssh, using vim-cmd, esxcli and sh;
no vCenter or VIM api required.

Remote* config properties added to support a remote driver

RemoteDriver interface extends Driver:
* SSHAddress - esx flavor uses esxcli to find the VM's ip address
* Download - esx flavor downloads iso files to a vmfs datastore

Driver can optionally implement the following interfaces:
* VNCAddressFinder - esx flavor needs to check remote ports
* OutputDir - esx driver needs a local and remote OutputDir
* Inventory - esx driver needs to register/unregister VMs
* HostIPFinder - esx flavor needs an address on the same network as esx itself
This commit is contained in:
Doug MacEachern 2013-09-19 17:07:04 -07:00 committed by Mitchell Hashimoto
parent b55252b332
commit a828a9a064
8 changed files with 610 additions and 37 deletions

View File

@ -55,6 +55,13 @@ type config struct {
VNCPortMin uint `mapstructure:"vnc_port_min"`
VNCPortMax uint `mapstructure:"vnc_port_max"`
RemoteType string `mapstructure:"remote_type"`
RemoteDatastore string `mapstructure:"remote_datastore"`
RemoteHost string `mapstructure:"remote_host"`
RemotePort uint `mapstructure:"remote_port"`
RemoteUser string `mapstructure:"remote_username"`
RemotePassword string `mapstructure:"remote_password"`
RawBootWait string `mapstructure:"boot_wait"`
RawSingleISOUrl string `mapstructure:"iso_url"`
RawShutdownTimeout string `mapstructure:"shutdown_timeout"`
@ -158,6 +165,10 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
"shutdown_timeout": &b.config.RawShutdownTimeout,
"ssh_wait_timeout": &b.config.RawSSHWaitTimeout,
"vmx_template_path": &b.config.VMXTemplatePath,
"remote_host": &b.config.RemoteHost,
"remote_datastore": &b.config.RemoteDatastore,
"remote_user": &b.config.RemoteUser,
"remote_password": &b.config.RemotePassword,
}
for n, ptr := range templates {
@ -343,7 +354,19 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
// Initialize the driver that will handle our interaction with VMware
driver, err := NewDriver()
var driver Driver
var err error
var sshAddressFunc func(multistep.StateBag) (string, error) = sshAddress
var downloadFunc func(*common.DownloadConfig, multistep.StateBag) (string, error, bool)
if b.config.RemoteType == "" {
driver, err = NewDriver()
} else {
driver, err = NewRemoteDriver(&b.config)
sshAddressFunc = driver.(RemoteDriver).SSHAddress()
downloadFunc = driver.(RemoteDriver).Download()
}
if err != nil {
return nil, fmt.Errorf("Failed creating VMware driver: %s", err)
}
@ -359,6 +382,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
Description: "ISO",
ResultKey: "iso_path",
Url: b.config.ISOUrls,
Download: downloadFunc,
},
&stepPrepareOutputDir{},
&common.StepCreateFloppy{
@ -371,7 +395,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
&stepRun{},
&stepTypeBootCommand{},
&common.StepConnectSSH{
SSHAddress: sshAddress,
SSHAddress: sshAddressFunc,
SSHConfig: sshConfig,
SSHWaitTimeout: b.config.sshWaitTimeout,
NoPty: b.config.SSHSkipRequestPty,

View File

@ -0,0 +1,418 @@
package vmware
import (
"bytes"
gossh "code.google.com/p/go.crypto/ssh"
"encoding/csv"
"errors"
"fmt"
"github.com/mitchellh/multistep"
"github.com/mitchellh/packer/common"
"github.com/mitchellh/packer/communicator/ssh"
"github.com/mitchellh/packer/packer"
"io"
"log"
"net"
"os"
"path/filepath"
"strings"
"syscall"
"time"
)
type ESX5Driver struct {
comm packer.Communicator
config *config
}
func (d *ESX5Driver) CompactDisk(diskPathLocal string) error {
return nil
}
func (d *ESX5Driver) CreateDisk(diskPathLocal string, size string, typeId string) error {
diskPath := d.datastorePath(diskPathLocal)
return d.sh("vmkfstools", "-c", size, "-d", typeId, "-a", "lsilogic", diskPath)
}
func (d *ESX5Driver) IsRunning(vmxPathLocal string) (bool, error) {
vmxPath := d.datastorePath(vmxPathLocal)
state, err := d.run(nil, "vim-cmd", "vmsvc/power.getstate", vmxPath)
if err != nil {
return false, err
}
return strings.Contains(state, "Powered on"), nil
}
func (d *ESX5Driver) Start(vmxPathLocal string, headless bool) error {
return d.sh("vim-cmd", "vmsvc/power.on", d.datastorePath(vmxPathLocal))
}
func (d *ESX5Driver) Stop(vmxPathLocal string) error {
return d.sh("vim-cmd", "vmsvc/power.off", d.datastorePath(vmxPathLocal))
}
func (d *ESX5Driver) Register(vmxPathLocal string) error {
vmxPath := d.datastorePath(vmxPathLocal)
if err := d.upload(vmxPathLocal, vmxPath); err != nil {
return err
}
return d.sh("vim-cmd", "solo/registervm", vmxPath)
}
func (d *ESX5Driver) Unregister(vmxPathLocal string) error {
return d.sh("vim-cmd", "vmsvc/unregister", d.datastorePath(vmxPathLocal))
}
func (d *ESX5Driver) ToolsIsoPath(string) string {
return ""
}
func (d *ESX5Driver) DhcpLeasesPath(string) string {
return ""
}
func (d *ESX5Driver) Verify() error {
checks := []func() error{
d.connect,
d.checkSystemVersion,
d.checkGuestIPHackEnabled,
d.checkOutputFolder,
}
for _, check := range checks {
err := check()
if err != nil {
return err
}
}
return nil
}
func (d *ESX5Driver) HostIP() (string, error) {
ip := net.ParseIP(d.config.RemoteHost)
interfaces, err := net.Interfaces()
if err != nil {
return "", err
}
for _, dev := range interfaces {
addrs, err := dev.Addrs()
if err != nil {
continue
}
for _, addr := range addrs {
if ipnet, ok := addr.(*net.IPNet); ok {
if ipnet.Contains(ip) {
return ipnet.IP.String(), nil
}
}
}
}
return "", errors.New("Unable to determine Host IP")
}
func (d *ESX5Driver) VNCAddress(portMin, portMax uint) (string, uint) {
var vncPort uint
// TODO(dougm) use esxcli network ip connection list
for port := portMin; port <= portMax; port++ {
address := fmt.Sprintf("%s:%d", d.config.RemoteHost, port)
log.Printf("Trying address: %s...", address)
l, err := net.DialTimeout("tcp", address, 1*time.Second)
if err == nil {
log.Printf("%s in use", address)
l.Close()
} else if e, ok := err.(*net.OpError); ok {
if e.Err == syscall.ECONNREFUSED {
// then port should be available for listening
vncPort = port
break
} else if e.Timeout() {
log.Printf("Timeout connecting to: %s (check firewall rules)", address)
}
}
}
return d.config.RemoteHost, vncPort
}
func (d *ESX5Driver) SSHAddress() func(multistep.StateBag) (string, error) {
return d.sshAddress
}
func (d *ESX5Driver) Download() func(*common.DownloadConfig, multistep.StateBag) (string, error, bool) {
return d.download
}
func (d *ESX5Driver) FileExists(path string) bool {
err := d.sh("test", "-e", d.datastorePath(path))
if err != nil {
return false
}
return true
}
func (d *ESX5Driver) MkdirAll(path string) error {
return d.sh("mkdir", "-p", d.datastorePath(path))
}
func (d *ESX5Driver) RemoveAll(path string) error {
return d.sh("rm", "-rf", d.datastorePath(path))
}
func (d *ESX5Driver) DirType() string {
return "datastore"
}
func (d *ESX5Driver) datastorePath(path string) string {
return filepath.Join("/vmfs/volumes", d.config.RemoteDatastore, path)
}
func (d *ESX5Driver) connect() error {
if d.config.RemoteHost == "" {
return errors.New("A remote_host must be specified.")
}
if d.config.RemotePort == 0 {
d.config.RemotePort = 22
}
address := fmt.Sprintf("%s:%d", d.config.RemoteHost, d.config.RemotePort)
auth := []gossh.ClientAuth{
gossh.ClientAuthPassword(ssh.Password(d.config.RemotePassword)),
gossh.ClientAuthKeyboardInteractive(
ssh.PasswordKeyboardInteractive(d.config.RemotePassword)),
}
// TODO(dougm) KeyPath support
sshConfig := &ssh.Config{
Connection: ssh.ConnectFunc("tcp", address),
SSHConfig: &gossh.ClientConfig{
User: d.config.RemoteUser,
Auth: auth,
},
NoPty: true,
}
comm, err := ssh.New(sshConfig)
if err != nil {
return err
}
d.comm = comm
return nil
}
func (d *ESX5Driver) checkSystemVersion() error {
r, err := d.esxcli("system", "version", "get")
if err != nil {
return err
}
record, err := r.read()
if err != nil {
return err
}
log.Printf("Connected to %s %s %s", record["Product"], record["Version"], record["Build"])
return nil
}
func (d *ESX5Driver) checkGuestIPHackEnabled() error {
r, err := d.esxcli("system", "settings", "advanced", "list", "-o", "/Net/GuestIPHack")
if err != nil {
return err
}
record, err := r.read()
if err != nil {
return err
}
if record["IntValue"] != "1" {
return errors.New("GuestIPHack is required, enable with:\n" +
"esxcli system settings advanced set -o /Net/GuestIPHack -i 1")
}
return nil
}
func (d *ESX5Driver) checkOutputFolder() error {
if d.config.RemoteDatastore == "" {
d.config.RemoteDatastore = "datastore1"
}
if !d.config.PackerForce && d.FileExists(d.config.OutputDir) {
return fmt.Errorf("Output folder '%s' already exists. It must not exist.",
d.config.OutputDir)
}
return nil
}
func (d *ESX5Driver) download(config *common.DownloadConfig, state multistep.StateBag) (string, error, bool) {
cacheRoot, _ := filepath.Abs(".")
targetFile, err := filepath.Rel(cacheRoot, config.TargetPath)
if err != nil {
return "", err, false
}
path := d.datastorePath(targetFile)
err = d.MkdirAll(filepath.Dir(targetFile))
if err != nil {
return "", err, false
}
if d.verifyChecksum(d.config.ISOChecksumType, d.config.ISOChecksum, path) {
log.Println("Initial checksum matched, no download needed.")
return path, nil, true
}
// TODO(dougm) progress and handle interrupt
err = d.sh("wget", config.Url, "-O", path)
return path, err, true
}
func (d *ESX5Driver) upload(src, dst string) error {
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
return d.comm.Upload(dst, f)
}
func (d *ESX5Driver) verifyChecksum(ctype string, hash string, file string) bool {
stdin := bytes.NewBufferString(fmt.Sprintf("%s %s", hash, file))
_, err := d.run(stdin, fmt.Sprintf("%ssum", ctype), "-c")
if err != nil {
return false
}
return true
}
func (d *ESX5Driver) ssh(command string, stdin io.Reader) (*bytes.Buffer, error) {
var stdout, stderr bytes.Buffer
cmd := &packer.RemoteCmd{
Command: command,
Stdout: &stdout,
Stderr: &stderr,
Stdin: stdin,
}
err := d.comm.Start(cmd)
if err != nil {
return nil, err
}
cmd.Wait()
if cmd.ExitStatus != 0 {
err = fmt.Errorf("'%s'\n\nStdout: %s\n\nStderr: %s",
cmd.Command, stdout.String(), stderr.String())
return nil, err
}
return &stdout, nil
}
func (d *ESX5Driver) run(stdin io.Reader, args ...string) (string, error) {
stdout, err := d.ssh(strings.Join(args, " "), stdin)
if err != nil {
return "", err
}
return stdout.String(), nil
}
func (d *ESX5Driver) sh(args ...string) error {
_, err := d.run(nil, args...)
return err
}
func (d *ESX5Driver) esxcli(args ...string) (*esxcliReader, error) {
stdout, err := d.ssh("esxcli --formatter csv "+strings.Join(args, " "), nil)
if err != nil {
return nil, err
}
r := csv.NewReader(bytes.NewReader(stdout.Bytes()))
r.TrailingComma = true
header, err := r.Read()
if err != nil {
return nil, err
}
return &esxcliReader{r, header}, nil
}
type esxcliReader struct {
cr *csv.Reader
header []string
}
func (r *esxcliReader) read() (map[string]string, error) {
fields, err := r.cr.Read()
if err != nil {
return nil, err
}
record := map[string]string{}
for i, v := range fields {
record[r.header[i]] = v
}
return record, nil
}
func (r *esxcliReader) find(key, val string) (map[string]string, error) {
for {
record, err := r.read()
if err != nil {
return nil, err
}
if record[key] == val {
return record, nil
}
}
}
func (d *ESX5Driver) sshAddress(state multistep.StateBag) (string, error) {
if address, ok := state.GetOk("vm_address"); ok {
return address.(string), nil
}
r, err := d.esxcli("network", "vm", "list")
if err != nil {
return "", err
}
record, err := r.find("Name", d.config.VMName)
if err != nil {
return "", err
}
wid := record["WorldID"]
if wid == "" {
return "", errors.New("VM WorldID not found")
}
r, err = d.esxcli("network", "vm", "port", "list", "-w", wid)
if err != nil {
return "", err
}
record, err = r.read()
if err != nil {
return "", err
}
if record["IPAddress"] == "0.0.0.0" {
return "", errors.New("VM network port found, but no IP address")
}
address := fmt.Sprintf("%s:%d", record["IPAddress"], d.config.SSHPort)
state.Put("vm_address", address)
return address, nil
}

View File

@ -0,0 +1,28 @@
package vmware
import (
"fmt"
"github.com/mitchellh/multistep"
"github.com/mitchellh/packer/common"
)
type RemoteDriver interface {
Driver
SSHAddress() func(multistep.StateBag) (string, error)
Download() func(*common.DownloadConfig, multistep.StateBag) (string, error, bool)
}
func NewRemoteDriver(config *config) (Driver, error) {
var driver Driver
switch config.RemoteType {
case "esx5":
driver = &ESX5Driver{
config: config,
}
default:
return nil, fmt.Errorf("Unknown product type: '%s'", config.RemoteType)
}
return driver, driver.Verify()
}

View File

@ -22,8 +22,31 @@ import (
// vnc_port uint - The port that VNC is configured to listen on.
type stepConfigureVNC struct{}
func (stepConfigureVNC) Run(state multistep.StateBag) multistep.StepAction {
type VNCAddressFinder interface {
VNCAddress(uint, uint) (string, uint)
}
func (stepConfigureVNC) VNCAddress(portMin, portMax uint) (string, uint) {
// Find an open VNC port. Note that this can still fail later on
// because we have to release the port at some point. But this does its
// best.
var vncPort uint
portRange := int(portMax - portMin)
for {
vncPort = uint(rand.Intn(portRange)) + portMin
log.Printf("Trying port: %d", vncPort)
l, err := net.Listen("tcp", fmt.Sprintf(":%d", vncPort))
if err == nil {
defer l.Close()
break
}
}
return "127.0.0.1", vncPort
}
func (s *stepConfigureVNC) Run(state multistep.StateBag) multistep.StepAction {
config := state.Get("config").(*config)
driver := state.Get("driver").(Driver)
ui := state.Get("ui").(packer.Ui)
vmxPath := state.Get("vmx_path").(string)
@ -43,20 +66,20 @@ func (stepConfigureVNC) Run(state multistep.StateBag) multistep.StepAction {
return multistep.ActionHalt
}
// Find an open VNC port. Note that this can still fail later on
// because we have to release the port at some point. But this does its
// best.
var vncFinder VNCAddressFinder
if finder, ok := driver.(VNCAddressFinder); ok {
vncFinder = finder
} else {
vncFinder = s
}
log.Printf("Looking for available port between %d and %d", config.VNCPortMin, config.VNCPortMax)
var vncPort uint
portRange := int(config.VNCPortMax - config.VNCPortMin)
for {
vncPort = uint(rand.Intn(portRange)) + config.VNCPortMin
log.Printf("Trying port: %d", vncPort)
l, err := net.Listen("tcp", fmt.Sprintf(":%d", vncPort))
if err == nil {
defer l.Close()
break
}
vncIp, vncPort := vncFinder.VNCAddress(config.VNCPortMin, config.VNCPortMax)
if vncPort == 0 {
err := fmt.Errorf("Unable to find available VNC port between %d and %d",
config.VNCPortMin, config.VNCPortMax)
state.Put("error", err)
ui.Error(err.Error())
return multistep.ActionHalt
}
log.Printf("Found available VNC port: %d", vncPort)
@ -73,6 +96,7 @@ func (stepConfigureVNC) Run(state multistep.StateBag) multistep.StepAction {
}
state.Put("vnc_port", vncPort)
state.Put("vnc_ip", vncIp)
return multistep.ActionContinue
}

View File

@ -1,6 +1,7 @@
package vmware
import (
"fmt"
"github.com/mitchellh/multistep"
"github.com/mitchellh/packer/packer"
"log"
@ -8,26 +9,54 @@ import (
"time"
)
type OutputDir interface {
FileExists(path string) bool
MkdirAll(path string) error
RemoveAll(path string) error
DirType() string
}
type localOutputDir struct{}
func (localOutputDir) FileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func (localOutputDir) MkdirAll(path string) error {
return os.MkdirAll(path, 0755)
}
func (localOutputDir) RemoveAll(path string) error {
return os.RemoveAll(path)
}
func (localOutputDir) DirType() string {
return "local"
}
type stepPrepareOutputDir struct{}
func (stepPrepareOutputDir) Run(state multistep.StateBag) multistep.StepAction {
func (s *stepPrepareOutputDir) Run(state multistep.StateBag) multistep.StepAction {
config := state.Get("config").(*config)
ui := state.Get("ui").(packer.Ui)
if _, err := os.Stat(config.OutputDir); err == nil && config.PackerForce {
ui.Say("Deleting previous output directory...")
os.RemoveAll(config.OutputDir)
}
for _, dir := range s.outputDirs(state) {
if dir.FileExists(config.OutputDir) && config.PackerForce {
ui.Say(fmt.Sprintf("Deleting previous %s output directory...", dir.DirType()))
dir.RemoveAll(config.OutputDir)
}
if err := os.MkdirAll(config.OutputDir, 0755); err != nil {
state.Put("error", err)
return multistep.ActionHalt
if err := dir.MkdirAll(config.OutputDir); err != nil {
state.Put("error", err)
return multistep.ActionHalt
}
}
return multistep.ActionContinue
}
func (stepPrepareOutputDir) Cleanup(state multistep.StateBag) {
func (s *stepPrepareOutputDir) Cleanup(state multistep.StateBag) {
_, cancelled := state.GetOk(multistep.StateCancelled)
_, halted := state.GetOk(multistep.StateHalted)
@ -35,15 +64,30 @@ func (stepPrepareOutputDir) Cleanup(state multistep.StateBag) {
config := state.Get("config").(*config)
ui := state.Get("ui").(packer.Ui)
ui.Say("Deleting output directory...")
for i := 0; i < 5; i++ {
err := os.RemoveAll(config.OutputDir)
if err == nil {
break
}
for _, dir := range s.outputDirs(state) {
ui.Say(fmt.Sprintf("Deleting %s output directory...", dir.DirType()))
for i := 0; i < 5; i++ {
err := dir.RemoveAll(config.OutputDir)
if err == nil {
break
}
log.Printf("Error removing output dir: %s", err)
time.Sleep(2 * time.Second)
log.Printf("Error removing output dir: %s", err)
time.Sleep(2 * time.Second)
}
}
}
}
func (s *stepPrepareOutputDir) outputDirs(state multistep.StateBag) []OutputDir {
driver := state.Get("driver").(Driver)
dirs := []OutputDir{
localOutputDir{},
}
if dir, ok := driver.(OutputDir); ok {
dirs = append(dirs, dir)
}
return dirs
}

View File

@ -22,11 +22,19 @@ type stepRun struct {
vmxPath string
}
type Inventory interface {
// Adds a VM to inventory specified by the path to the VMX given.
Register(string) error
// Removes a VM from inventory specified by the path to the VMX given.
Unregister(string) error
}
func (s *stepRun) Run(state multistep.StateBag) multistep.StepAction {
config := state.Get("config").(*config)
driver := state.Get("driver").(Driver)
ui := state.Get("ui").(packer.Ui)
vmxPath := state.Get("vmx_path").(string)
vncIp := state.Get("vnc_ip").(string)
vncPort := state.Get("vnc_port").(uint)
// Set the VMX path so that we know we started the machine
@ -38,7 +46,16 @@ func (s *stepRun) Run(state multistep.StateBag) multistep.StepAction {
ui.Message(fmt.Sprintf(
"The VM will be run headless, without a GUI. If you want to\n"+
"view the screen of the VM, connect via VNC without a password to\n"+
"127.0.0.1:%d", vncPort))
"%s:%d", vncIp, vncPort))
}
if inv, ok := driver.(Inventory); ok {
if err := inv.Register(vmxPath); err != nil {
err := fmt.Errorf("Error registering VM: %s", err)
state.Put("error", err)
ui.Error(err.Error())
return multistep.ActionHalt
}
}
if err := driver.Start(vmxPath, config.Headless); err != nil {
@ -80,5 +97,12 @@ func (s *stepRun) Cleanup(state multistep.StateBag) {
ui.Error(fmt.Sprintf("Error stopping VM: %s", err))
}
}
if inv, ok := driver.(Inventory); ok {
ui.Say("Unregistering virtual machine...")
if err := inv.Unregister(s.vmxPath); err != nil {
ui.Error(fmt.Sprintf("Error unregistering VM: %s", err))
}
}
}
}

View File

@ -36,13 +36,15 @@ type stepTypeBootCommand struct{}
func (s *stepTypeBootCommand) Run(state multistep.StateBag) multistep.StepAction {
config := state.Get("config").(*config)
driver := state.Get("driver").(Driver)
httpPort := state.Get("http_port").(uint)
ui := state.Get("ui").(packer.Ui)
vncIp := state.Get("vnc_ip").(string)
vncPort := state.Get("vnc_port").(uint)
// Connect to VNC
ui.Say("Connecting to VM via VNC")
nc, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", vncPort))
nc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", vncIp, vncPort))
if err != nil {
err := fmt.Errorf("Error connecting to VNC: %s", err)
state.Put("error", err)
@ -64,7 +66,9 @@ func (s *stepTypeBootCommand) Run(state multistep.StateBag) multistep.StepAction
// Determine the host IP
var ipFinder HostIPFinder
if runtime.GOOS == "windows" {
if finder, ok := driver.(HostIPFinder); ok {
ipFinder = finder
} else if runtime.GOOS == "windows" {
ipFinder = new(VMnetNatConfIPFinder)
} else {
ipFinder = &IfconfigIPFinder{Device: "vmnet8"}

View File

@ -35,6 +35,8 @@ type StepDownload struct {
// A list of URLs to attempt to download this thing.
Url []string
Download func(*DownloadConfig, multistep.StateBag) (string, error, bool)
}
func (s *StepDownload) Run(state multistep.StateBag) multistep.StepAction {
@ -53,6 +55,11 @@ func (s *StepDownload) Run(state multistep.StateBag) multistep.StepAction {
ui.Say(fmt.Sprintf("Downloading or copying %s", s.Description))
downloadFunc := s.Download
if downloadFunc == nil {
downloadFunc = s.download
}
var finalPath string
for _, url := range s.Url {
ui.Message(fmt.Sprintf("Downloading or copying: %s", url))
@ -72,7 +79,7 @@ func (s *StepDownload) Run(state multistep.StateBag) multistep.StepAction {
Checksum: checksum,
}
path, err, retry := s.download(config, state)
path, err, retry := downloadFunc(config, state)
if err != nil {
ui.Message(fmt.Sprintf("Error downloading: %s", err))
}