From 659a0da594eab67dbe20c3086d01958317408050 Mon Sep 17 00:00:00 2001 From: Michele Catalano Date: Tue, 22 Dec 2015 15:56:33 +0100 Subject: [PATCH] Add lxc builder to packer --- builder/lxc/artifact.go | 35 ++++++ builder/lxc/builder.go | 118 ++++++++++++++++++++ builder/lxc/command.go | 15 +++ builder/lxc/communicator.go | 147 +++++++++++++++++++++++++ builder/lxc/config.go | 70 ++++++++++++ builder/lxc/step_export.go | 99 +++++++++++++++++ builder/lxc/step_lxc_create.go | 89 +++++++++++++++ builder/lxc/step_prepare_output_dir.go | 49 +++++++++ builder/lxc/step_provision.go | 36 ++++++ builder/lxc/step_wait_init.go | 123 +++++++++++++++++++++ 10 files changed, 781 insertions(+) create mode 100644 builder/lxc/artifact.go create mode 100644 builder/lxc/builder.go create mode 100644 builder/lxc/command.go create mode 100644 builder/lxc/communicator.go create mode 100644 builder/lxc/config.go create mode 100644 builder/lxc/step_export.go create mode 100644 builder/lxc/step_lxc_create.go create mode 100644 builder/lxc/step_prepare_output_dir.go create mode 100644 builder/lxc/step_provision.go create mode 100644 builder/lxc/step_wait_init.go diff --git a/builder/lxc/artifact.go b/builder/lxc/artifact.go new file mode 100644 index 000000000..9ac57d0cc --- /dev/null +++ b/builder/lxc/artifact.go @@ -0,0 +1,35 @@ +package lxc + +import ( + "fmt" + "os" +) + +type Artifact struct { + dir string + f []string +} + +func (*Artifact) BuilderId() string { + return BuilderId +} + +func (a *Artifact) Files() []string { + return a.f +} + +func (*Artifact) Id() string { + return "VM" +} + +func (a *Artifact) String() string { + return fmt.Sprintf("VM files in directory: %s", a.dir) +} + +func (a *Artifact) State(name string) interface{} { + return nil +} + +func (a *Artifact) Destroy() error { + return os.RemoveAll(a.dir) +} diff --git a/builder/lxc/builder.go b/builder/lxc/builder.go new file mode 100644 index 000000000..7d630fe44 --- /dev/null +++ b/builder/lxc/builder.go @@ -0,0 +1,118 @@ +package lxc + +import ( + "errors" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/template/interpolate" + "log" + "os" + "path/filepath" + "runtime" +) + +// The unique ID for this builder +const BuilderId = "ustream.lxc" + +type wrappedCommandTemplate struct { + Command string +} + +type Builder struct { + config *Config + runner multistep.Runner +} + +func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { + c, errs := NewConfig(raws...) + if errs != nil { + return nil, errs + } + b.config = c + + return nil, nil +} + +func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { + if runtime.GOOS != "linux" { + return nil, errors.New("The lxc builder only works on linux environments.") + } + + wrappedCommand := func(command string) (string, error) { + b.config.ctx.Data = &wrappedCommandTemplate{Command: command} + return interpolate.Render(b.config.CommandWrapper, &b.config.ctx) + } + + steps := []multistep.Step{ + new(stepPrepareOutputDir), + new(stepLxcCreate), + &StepWaitInit{ + WaitTimeout: b.config.InitTimeout, + }, + new(StepProvision), + new(stepExport), + } + + // Setup the state bag + state := new(multistep.BasicStateBag) + state.Put("config", b.config) + state.Put("cache", cache) + state.Put("hook", hook) + state.Put("ui", ui) + state.Put("wrappedCommand", CommandWrapper(wrappedCommand)) + + // Run + if b.config.PackerDebug { + b.runner = &multistep.DebugRunner{ + Steps: steps, + PauseFn: common.MultistepDebugFn(ui), + } + } else { + b.runner = &multistep.BasicRunner{Steps: steps} + } + + b.runner.Run(state) + + // If there was an error, return that + if rawErr, ok := state.GetOk("error"); ok { + return nil, rawErr.(error) + } + + // If we were interrupted or cancelled, then just exit. + if _, ok := state.GetOk(multistep.StateCancelled); ok { + return nil, errors.New("Build was cancelled.") + } + + if _, ok := state.GetOk(multistep.StateHalted); ok { + return nil, errors.New("Build was halted.") + } + + // Compile the artifact list + files := make([]string, 0, 5) + visit := func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + files = append(files, path) + } + + return err + } + + if err := filepath.Walk(b.config.OutputDir, visit); err != nil { + return nil, err + } + + artifact := &Artifact{ + dir: b.config.OutputDir, + f: files, + } + + return artifact, nil +} + +func (b *Builder) Cancel() { + if b.runner != nil { + log.Println("Cancelling the step runner...") + b.runner.Cancel() + } +} diff --git a/builder/lxc/command.go b/builder/lxc/command.go new file mode 100644 index 000000000..af81cff83 --- /dev/null +++ b/builder/lxc/command.go @@ -0,0 +1,15 @@ +package lxc + +import ( + "os/exec" +) + +// CommandWrapper is a type that given a command, will possibly modify that +// command in-flight. This might return an error. +type CommandWrapper func(string) (string, error) + +// ShellCommand takes a command string and returns an *exec.Cmd to execute +// it within the context of a shell (/bin/sh). +func ShellCommand(command string) *exec.Cmd { + return exec.Command("/bin/sh", "-c", command) +} diff --git a/builder/lxc/communicator.go b/builder/lxc/communicator.go new file mode 100644 index 000000000..a1eb50fdb --- /dev/null +++ b/builder/lxc/communicator.go @@ -0,0 +1,147 @@ +package lxc + +import ( + "fmt" + "github.com/mitchellh/packer/packer" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "syscall" + "strings" +) + +type LxcAttachCommunicator struct { + RootFs string + ContainerName string + CmdWrapper CommandWrapper +} + +func (c *LxcAttachCommunicator) Start(cmd *packer.RemoteCmd) error { + localCmd, err := c.Execute(cmd.Command) + + if err != nil { + return err + } + + localCmd.Stdin = cmd.Stdin + localCmd.Stdout = cmd.Stdout + localCmd.Stderr = cmd.Stderr + if err := localCmd.Start(); err != nil { + return err + } + + go func() { + exitStatus := 0 + if err := localCmd.Wait(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + exitStatus = 1 + + // There is no process-independent way to get the REAL + // exit status so we just try to go deeper. + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { + exitStatus = status.ExitStatus() + } + } + } + + log.Printf( + "lxc-attach execution exited with '%d': '%s'", + exitStatus, cmd.Command) + cmd.SetExited(exitStatus) + }() + + return nil +} + +func (c *LxcAttachCommunicator) Upload(dst string, r io.Reader, fi *os.FileInfo) error { + dst = filepath.Join(c.RootFs, dst) + log.Printf("Uploading to rootfs: %s", dst) + tf, err := ioutil.TempFile("", "packer-lxc-attach") + if err != nil { + return fmt.Errorf("Error uploading file to rootfs: %s", err) + } + defer os.Remove(tf.Name()) + io.Copy(tf, r) + + cpCmd, err := c.CmdWrapper(fmt.Sprintf("sudo cp %s %s", tf.Name(), dst)) + if err != nil { + return err + } + + log.Printf("Running copy command: %s", dst) + + return ShellCommand(cpCmd).Run() +} + +func (c *LxcAttachCommunicator) UploadDir(dst string, src string, exclude []string) error { + // TODO: remove any file copied if it appears in `exclude` + dest := filepath.Join(c.RootFs, dst) + log.Printf("Uploading directory '%s' to rootfs '%s'", src, dest) + cpCmd, err := c.CmdWrapper(fmt.Sprintf("sudo cp -R %s/. %s", src, dest)) + if err != nil { + return err + } + + return ShellCommand(cpCmd).Run() +} + +func (c *LxcAttachCommunicator) Download(src string, w io.Writer) error { + src = filepath.Join(c.RootFs, src) + log.Printf("Downloading from rootfs dir: %s", src) + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.Copy(w, f); err != nil { + return err + } + + return nil +} + +func (c *LxcAttachCommunicator) Execute(commandString string) (*exec.Cmd, error) { + log.Printf("Executing with lxc-attach in container: %s %s %s", c.ContainerName, c.RootFs, commandString) + command, err := c.CmdWrapper( + fmt.Sprintf("sudo lxc-attach --name %s -- /bin/sh -c \"%s\"", c.ContainerName, commandString)) + if err != nil { + return nil, err + } + + localCmd := ShellCommand(command) + log.Printf("Executing lxc-attach: %s %#v", localCmd.Path, localCmd.Args) + + return localCmd, nil +} + +func (c *LxcAttachCommunicator) CheckInit() (string, error) { + log.Printf("Debug runlevel exec") + localCmd, err := c.Execute("/sbin/runlevel") + + if err != nil { + return "", err + } + + pr, _ := localCmd.StdoutPipe() + if err = localCmd.Start(); err != nil { + return "", err + } + + output, err := ioutil.ReadAll(pr) + + if err != nil { + return "", err + } + + err = localCmd.Wait() + + if err != nil { + return "", err + } + + return strings.TrimSpace(string(output)), nil +} \ No newline at end of file diff --git a/builder/lxc/config.go b/builder/lxc/config.go new file mode 100644 index 000000000..6c32eeb95 --- /dev/null +++ b/builder/lxc/config.go @@ -0,0 +1,70 @@ +package lxc + +import ( + "fmt" + "github.com/mitchellh/mapstructure" + "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/config" + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/template/interpolate" + "time" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + ConfigFile string `mapstructure:"config_file"` + OutputDir string `mapstructure:"output_directory"` + ContainerName string `mapstructure:"container_name"` + CommandWrapper string `mapstructure:"command_wrapper"` + RawInitTimeout string `mapstructure:"init_timeout"` + Name string `mapstructure:"template_name"` + Parameters []string `mapstructure:"template_parameters"` + EnvVars []string `mapstructure:"template_environment_vars"` + TargetRunlevel int `mapstructure:"target_runlevel"` + InitTimeout time.Duration + + ctx interpolate.Context +} + +func NewConfig(raws ...interface{}) (*Config, error) { + var c Config + + var md mapstructure.Metadata + err := config.Decode(&c, &config.DecodeOpts{ + Metadata: &md, + Interpolate: true, + }, raws...) + if err != nil { + return nil, err + } + + // Accumulate any errors + var errs *packer.MultiError + + if c.OutputDir == "" { + c.OutputDir = fmt.Sprintf("output-%s", c.PackerBuildName) + } + + if c.ContainerName == "" { + c.ContainerName = fmt.Sprintf("packer-%s", c.PackerBuildName) + } + + if c.CommandWrapper == "" { + c.CommandWrapper = "{{.Command}}" + } + + if c.RawInitTimeout == "" { + c.RawInitTimeout = "20s" + } + + c.InitTimeout, err = time.ParseDuration(c.RawInitTimeout) + if err != nil { + errs = packer.MultiErrorAppend(errs, fmt.Errorf("Failed parsing init_timeout: %s", err)) + } + + if errs != nil && len(errs.Errors) > 0 { + return nil, errs + } + + return &c, nil +} diff --git a/builder/lxc/step_export.go b/builder/lxc/step_export.go new file mode 100644 index 000000000..3af9ed7f3 --- /dev/null +++ b/builder/lxc/step_export.go @@ -0,0 +1,99 @@ +package lxc + +import ( + "github.com/mitchellh/multistep" + "fmt" + "github.com/mitchellh/packer/packer" + "bytes" + "os/exec" + "log" + "strings" + "path/filepath" + "os" + "io" +) + +type stepExport struct{} + +func (s *stepExport) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(*Config) + ui := state.Get("ui").(packer.Ui) + + name := config.ContainerName + + containerDir := fmt.Sprintf("/var/lib/lxc/%s", name) + outputPath := filepath.Join(config.OutputDir, "rootfs.tar.gz") + configFilePath := filepath.Join(config.OutputDir, "lxc-config") + + configFile, err := os.Create(configFilePath) + + if err != nil { + err := fmt.Errorf("Error creating config file: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + originalConfigFile, err := os.Open(config.ConfigFile) + + if err != nil { + err := fmt.Errorf("Error opening config file: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + _, err = io.Copy(configFile, originalConfigFile) + + commands := make([][]string, 4) + commands[0] = []string{ + "lxc-stop", "--name", name, + } + commands[1] = []string{ + "tar", "-C", containerDir, "--numeric-owner", "--anchored", "--exclude=./rootfs/dev/log", "-czf", outputPath, "./rootfs", + } + commands[2] = []string{ + "chmod", "+x", configFilePath, + } + commands[3] = []string{ + "sh", "-c", "chown $USER:`id -gn` " + filepath.Join(config.OutputDir, "*"), + } + + ui.Say("Exporting container...") + for _, command := range commands { + err := s.SudoCommand(command...) + if err != nil { + err := fmt.Errorf("Error exporting container: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + } + + return multistep.ActionContinue +} + +func (s *stepExport) Cleanup(state multistep.StateBag) {} + + +func (s *stepExport) SudoCommand(args ...string) error { + var stdout, stderr bytes.Buffer + + log.Printf("Executing sudo command: %#v", args) + cmd := exec.Command("sudo", args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + + stdoutString := strings.TrimSpace(stdout.String()) + stderrString := strings.TrimSpace(stderr.String()) + + if _, ok := err.(*exec.ExitError); ok { + err = fmt.Errorf("Sudo command error: %s", stderrString) + } + + log.Printf("stdout: %s", stdoutString) + log.Printf("stderr: %s", stderrString) + + return err +} \ No newline at end of file diff --git a/builder/lxc/step_lxc_create.go b/builder/lxc/step_lxc_create.go new file mode 100644 index 000000000..070eae680 --- /dev/null +++ b/builder/lxc/step_lxc_create.go @@ -0,0 +1,89 @@ +package lxc + +import ( + "bytes" + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" + "os/exec" + "path/filepath" + "strings" +) + +type stepLxcCreate struct{} + +func (s *stepLxcCreate) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(*Config) + ui := state.Get("ui").(packer.Ui) + + name := config.ContainerName + + // TODO: read from env + lxc_dir := "/var/lib/lxc" + rootfs := filepath.Join(lxc_dir, name, "rootfs") + + if config.PackerForce { + s.Cleanup(state) + } + + commands := make([][]string, 3) + commands[0] = append(config.EnvVars, []string{"lxc-create", "-n", name, "-t", config.Name, "--"}...) + commands[0] = append(commands[0], config.Parameters...) + // prevent tmp from being cleaned on boot, we put provisioning scripts there + // todo: wait for init to finish before moving on to provisioning instead of this + commands[1] = []string{"touch", filepath.Join(rootfs, "tmp", ".tmpfs")} + commands[2] = []string{"lxc-start", "-d", "--name", name} + + ui.Say("Creating container...") + for _, command := range commands { + log.Printf("Executing sudo command: %#v", command) + err := s.SudoCommand(command...) + if err != nil { + err := fmt.Errorf("Error creating container: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + } + + state.Put("mount_path", rootfs) + + return multistep.ActionContinue +} + +func (s *stepLxcCreate) Cleanup(state multistep.StateBag) { + config := state.Get("config").(*Config) + ui := state.Get("ui").(packer.Ui) + + command := []string{ + "lxc-destroy", "-f", "-n", config.ContainerName, + } + + ui.Say("Unregistering and deleting virtual machine...") + if err := s.SudoCommand(command...); err != nil { + ui.Error(fmt.Sprintf("Error deleting virtual machine: %s", err)) + } +} + +func (s *stepLxcCreate) SudoCommand(args ...string) error { + var stdout, stderr bytes.Buffer + + log.Printf("Executing sudo command: %#v", args) + cmd := exec.Command("sudo", args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + + stdoutString := strings.TrimSpace(stdout.String()) + stderrString := strings.TrimSpace(stderr.String()) + + if _, ok := err.(*exec.ExitError); ok { + err = fmt.Errorf("Sudo command error: %s", stderrString) + } + + log.Printf("stdout: %s", stdoutString) + log.Printf("stderr: %s", stderrString) + + return err +} diff --git a/builder/lxc/step_prepare_output_dir.go b/builder/lxc/step_prepare_output_dir.go new file mode 100644 index 000000000..4b66c0c87 --- /dev/null +++ b/builder/lxc/step_prepare_output_dir.go @@ -0,0 +1,49 @@ +package lxc + +import ( + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" + "os" + "time" +) + +type stepPrepareOutputDir struct{} + +func (stepPrepareOutputDir) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(*Config) + ui := state.Get("ui").(packer.Ui) + + if _, err := os.Stat(config.OutputDir); err == nil && config.PackerForce { + ui.Say("Deleting previous output directory...") + os.RemoveAll(config.OutputDir) + } + + if err := os.MkdirAll(config.OutputDir, 0755); err != nil { + state.Put("error", err) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (stepPrepareOutputDir) Cleanup(state multistep.StateBag) { + _, cancelled := state.GetOk(multistep.StateCancelled) + _, halted := state.GetOk(multistep.StateHalted) + + if cancelled || halted { + config := state.Get("config").(*Config) + ui := state.Get("ui").(packer.Ui) + + ui.Say("Deleting output directory...") + for i := 0; i < 5; i++ { + err := os.RemoveAll(config.OutputDir) + if err == nil { + break + } + + log.Printf("Error removing output dir: %s", err) + time.Sleep(2 * time.Second) + } + } +} diff --git a/builder/lxc/step_provision.go b/builder/lxc/step_provision.go new file mode 100644 index 000000000..b8fe6cd04 --- /dev/null +++ b/builder/lxc/step_provision.go @@ -0,0 +1,36 @@ +package lxc + +import ( + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" +) + +// StepProvision provisions the instance within a chroot. +type StepProvision struct {} + +func (s *StepProvision) Run(state multistep.StateBag) multistep.StepAction { + hook := state.Get("hook").(packer.Hook) + config := state.Get("config").(*Config) + mountPath := state.Get("mount_path").(string) + ui := state.Get("ui").(packer.Ui) + wrappedCommand := state.Get("wrappedCommand").(CommandWrapper) + + // Create our communicator + comm := &LxcAttachCommunicator{ + ContainerName: config.ContainerName, + RootFs: mountPath, + CmdWrapper: wrappedCommand, + } + + // Provision + log.Println("Running the provision hook") + if err := hook.Run(packer.HookProvision, ui, comm, nil); err != nil { + state.Put("error", err) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *StepProvision) Cleanup(state multistep.StateBag) {} diff --git a/builder/lxc/step_wait_init.go b/builder/lxc/step_wait_init.go new file mode 100644 index 000000000..180f83f73 --- /dev/null +++ b/builder/lxc/step_wait_init.go @@ -0,0 +1,123 @@ +package lxc + +import ( + "errors" + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" + "strings" + "time" +) + +type StepWaitInit struct { + WaitTimeout time.Duration +} + +func (s *StepWaitInit) Run(state multistep.StateBag) multistep.StepAction { + ui := state.Get("ui").(packer.Ui) + + var err error + + cancel := make(chan struct{}) + waitDone := make(chan bool, 1) + go func() { + ui.Say("Waiting for container to finish init...") + err = s.waitForInit(state, cancel) + waitDone <- true + }() + + log.Printf("Waiting for container to finish init, up to timeout: %s", s.WaitTimeout) + timeout := time.After(s.WaitTimeout) +WaitLoop: + for { + select { + case <-waitDone: + if err != nil { + ui.Error(fmt.Sprintf("Error waiting for container to finish init: %s", err)) + return multistep.ActionHalt + } + + ui.Say("Container finished init!") + break WaitLoop + case <-timeout: + err := fmt.Errorf("Timeout waiting for container to finish init.") + state.Put("error", err) + ui.Error(err.Error()) + close(cancel) + return multistep.ActionHalt + case <-time.After(1 * time.Second): + if _, ok := state.GetOk(multistep.StateCancelled); ok { + close(cancel) + log.Println("Interrupt detected, quitting waiting for container to finish init.") + return multistep.ActionHalt + } + } + } + + return multistep.ActionContinue +} + +func (s *StepWaitInit) Cleanup(multistep.StateBag) { +} + +func (s *StepWaitInit) waitForInit(state multistep.StateBag, cancel <-chan struct{}) error { + config := state.Get("config").(*Config) + mountPath := state.Get("mount_path").(string) + wrappedCommand := state.Get("wrappedCommand").(CommandWrapper) + + for { + select { + case <-cancel: + log.Println("Cancelled. Exiting loop.") + return errors.New("Wait cancelled") + case <-time.After(1 * time.Second): + } + + comm := &LxcAttachCommunicator{ + ContainerName: config.ContainerName, + RootFs: mountPath, + CmdWrapper: wrappedCommand, + } + + runlevel, _ := comm.CheckInit() + currentRunlevel := "unknown" + if arr := strings.Split(runlevel, " "); len(arr) >= 2 { + currentRunlevel = arr[1] + } + + log.Printf("Current runlevel in container: '%s'", runlevel) + + targetRunlevel := fmt.Sprintf("%d", config.TargetRunlevel) + if currentRunlevel == targetRunlevel { + log.Printf("Container finished init.") + break + } + + /*log.Println("Attempting SSH connection...") + comm, err = ssh.New(config) + if err != nil { + log.Printf("SSH handshake err: %s", err) + + // Only count this as an attempt if we were able to attempt + // to authenticate. Note this is very brittle since it depends + // on the string of the error... but I don't see any other way. + if strings.Contains(err.Error(), "authenticate") { + log.Printf("Detected authentication error. Increasing handshake attempts.") + handshakeAttempts += 1 + } + + if handshakeAttempts < 10 { + // Try to connect via SSH a handful of times + continue + } + + return nil, err + } + + break + */ + } + + return nil +}