fix: chroot builder

builder/osc/chroot/builder.go

@@ -1,4 +1,4 @@
-// The chroot package is able to create an Outscale OMI without requiring
+// Package chroot is able to create an Outscale OMI without requiring
 // the launch of a new instance for every build. It does this by attaching
 // and mounting the root volume of another OMI and chrooting into that
 // directory. It then creates an OMI from that attached drive.
@@ -11,7 +11,6 @@ import (
 	"net/http"
 	"runtime"
 
-	awschroot "github.com/hashicorp/packer/builder/amazon/chroot"
 	osccommon "github.com/hashicorp/packer/builder/osc/common"
 	"github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/helper/config"
@@ -238,7 +237,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
 	}
 
 	steps = append(steps,
-		&awschroot.StepFlock{},
+		&StepFlock{},
 		&StepPrepareDevice{},
 		&StepCreateVolume{
 			RootVolumeType: b.config.RootVolumeType,
@@ -247,21 +246,21 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
 			Ctx:            b.config.ctx,
 		},
 		&StepLinkVolume{},
-		&awschroot.StepEarlyUnflock{},
-		&awschroot.StepPreMountCommands{
+		&StepEarlyUnflock{},
+		&StepPreMountCommands{
 			Commands: b.config.PreMountCommands,
 		},
 		&StepMountDevice{
 			MountOptions:   b.config.MountOptions,
 			MountPartition: b.config.MountPartition,
 		},
-		&awschroot.StepPostMountCommands{
+		&StepPostMountCommands{
 			Commands: b.config.PostMountCommands,
 		},
-		&awschroot.StepMountExtra{},
-		&awschroot.StepCopyFiles{},
-		&awschroot.StepChrootProvision{},
-		&awschroot.StepEarlyCleanup{},
+		&StepMountExtra{},
+		&StepCopyFiles{},
+		&StepChrootProvision{},
+		&StepEarlyCleanup{},
 		&StepSnapshot{},
 		&osccommon.StepDeregisterOMI{
 			AccessConfig:        &b.config.AccessConfig,

builder/osc/chroot/cleanup.go

@@ -0,0 +1,10 @@
+package chroot
+
+import (
+	"github.com/hashicorp/packer/helper/multistep"
+)
+
+// Cleanup is an interface that some steps implement for early cleanup.
+type Cleanup interface {
+	CleanupFunc(multistep.StateBag) error
+}

builder/osc/chroot/communicator.go

@@ -2,6 +2,7 @@ package chroot
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"log"
@@ -23,7 +24,7 @@ type Communicator struct {
 	CmdWrapper CommandWrapper
 }
 
-func (c *Communicator) Start(cmd *packer.RemoteCmd) error {
+func (c *Communicator) Start(ctx context.Context, cmd *packer.RemoteCmd) error {
 	// need extra escapes for the command since we're wrapping it in quotes
 	cmd.Command = strconv.Quote(cmd.Command)
 	command, err := c.CmdWrapper(

builder/osc/chroot/communicator_test.go

@@ -0,0 +1,15 @@
+package chroot
+
+import (
+	"testing"
+
+	"github.com/hashicorp/packer/packer"
+)
+
+func TestCommunicator_ImplementsCommunicator(t *testing.T) {
+	var raw interface{}
+	raw = &Communicator{}
+	if _, ok := raw.(packer.Communicator); !ok {
+		t.Fatalf("Communicator should be a communicator")
+	}
+}

builder/osc/chroot/lockfile.go

@@ -0,0 +1,16 @@
+// +build windows
+
+package chroot
+
+import (
+	"errors"
+	"os"
+)
+
+func lockFile(*os.File) error {
+	return errors.New("not supported on Windows")
+}
+
+func unlockFile(f *os.File) error {
+	return nil
+}

builder/osc/chroot/lockfile_unix.go

@@ -0,0 +1,27 @@
+// +build !windows
+
+package chroot
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+// See: http://linux.die.net/include/sys/file.h
+const LOCK_EX = 2
+const LOCK_NB = 4
+const LOCK_UN = 8
+
+func lockFile(f *os.File) error {
+	err := unix.Flock(int(f.Fd()), LOCK_EX)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func unlockFile(f *os.File) error {
+	return unix.Flock(int(f.Fd()), LOCK_UN)
+}

builder/osc/chroot/run_local_commands.go

@@ -0,0 +1,41 @@
+package chroot
+
+import (
+	"context"
+	"fmt"
+
+	sl "github.com/hashicorp/packer/common/shell-local"
+	"github.com/hashicorp/packer/packer"
+	"github.com/hashicorp/packer/template/interpolate"
+)
+
+func RunLocalCommands(commands []string, wrappedCommand CommandWrapper, ictx interpolate.Context, ui packer.Ui) error {
+	ctx := context.TODO()
+	for _, rawCmd := range commands {
+		intCmd, err := interpolate.Render(rawCmd, &ictx)
+		if err != nil {
+			return fmt.Errorf("Error interpolating: %s", err)
+		}
+
+		command, err := wrappedCommand(intCmd)
+		if err != nil {
+			return fmt.Errorf("Error wrapping command: %s", err)
+		}
+
+		ui.Say(fmt.Sprintf("Executing command: %s", command))
+		comm := &sl.Communicator{
+			ExecuteCommand: []string{"sh", "-c", command},
+		}
+		cmd := &packer.RemoteCmd{Command: command}
+		if err := cmd.RunWithUi(ctx, comm, ui); err != nil {
+			return fmt.Errorf("Error executing command: %s", err)
+		}
+		if cmd.ExitStatus() != 0 {
+			return fmt.Errorf(
+				"Received non-zero exit code %d from command: %s",
+				cmd.ExitStatus(),
+				command)
+		}
+	}
+	return nil
+}

builder/osc/chroot/step_chroot_provision.go

@@ -0,0 +1,37 @@
+package chroot
+
+import (
+	"context"
+	"log"
+
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+// StepChrootProvision provisions the instance within a chroot.
+type StepChrootProvision struct {
+}
+
+func (s *StepChrootProvision) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	hook := state.Get("hook").(packer.Hook)
+	mountPath := state.Get("mount_path").(string)
+	ui := state.Get("ui").(packer.Ui)
+	wrappedCommand := state.Get("wrappedCommand").(CommandWrapper)
+
+	// Create our communicator
+	comm := &Communicator{
+		Chroot:     mountPath,
+		CmdWrapper: wrappedCommand,
+	}
+
+	// Provision
+	log.Println("Running the provision hook")
+	if err := hook.Run(ctx, packer.HookProvision, ui, comm, nil); err != nil {
+		state.Put("error", err)
+		return multistep.ActionHalt
+	}
+
+	return multistep.ActionContinue
+}
+
+func (s *StepChrootProvision) Cleanup(state multistep.StateBag) {}

builder/osc/chroot/step_copy_files.go

@@ -0,0 +1,91 @@
+package chroot
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"path/filepath"
+
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+// StepCopyFiles copies some files from the host into the chroot environment.
+//
+// Produces:
+//   copy_files_cleanup CleanupFunc - A function to clean up the copied files
+//   early.
+type StepCopyFiles struct {
+	files []string
+}
+
+func (s *StepCopyFiles) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	config := state.Get("config").(*Config)
+	mountPath := state.Get("mount_path").(string)
+	ui := state.Get("ui").(packer.Ui)
+	wrappedCommand := state.Get("wrappedCommand").(CommandWrapper)
+	stderr := new(bytes.Buffer)
+
+	s.files = make([]string, 0, len(config.CopyFiles))
+	if len(config.CopyFiles) > 0 {
+		ui.Say("Copying files from host to chroot...")
+		for _, path := range config.CopyFiles {
+			ui.Message(path)
+			chrootPath := filepath.Join(mountPath, path)
+			log.Printf("Copying '%s' to '%s'", path, chrootPath)
+
+			cmdText, err := wrappedCommand(fmt.Sprintf("cp --remove-destination %s %s", path, chrootPath))
+			if err != nil {
+				err := fmt.Errorf("Error building copy command: %s", err)
+				state.Put("error", err)
+				ui.Error(err.Error())
+				return multistep.ActionHalt
+			}
+
+			stderr.Reset()
+			cmd := ShellCommand(cmdText)
+			cmd.Stderr = stderr
+			if err := cmd.Run(); err != nil {
+				err := fmt.Errorf(
+					"Error copying file: %s\nnStderr: %s", err, stderr.String())
+				state.Put("error", err)
+				ui.Error(err.Error())
+				return multistep.ActionHalt
+			}
+
+			s.files = append(s.files, chrootPath)
+		}
+	}
+
+	state.Put("copy_files_cleanup", s)
+	return multistep.ActionContinue
+}
+
+func (s *StepCopyFiles) Cleanup(state multistep.StateBag) {
+	ui := state.Get("ui").(packer.Ui)
+	if err := s.CleanupFunc(state); err != nil {
+		ui.Error(err.Error())
+	}
+}
+
+func (s *StepCopyFiles) CleanupFunc(state multistep.StateBag) error {
+	wrappedCommand := state.Get("wrappedCommand").(CommandWrapper)
+	if s.files != nil {
+		for _, file := range s.files {
+			log.Printf("Removing: %s", file)
+			localCmdText, err := wrappedCommand(fmt.Sprintf("rm -f %s", file))
+			if err != nil {
+				return err
+			}
+
+			localCmd := ShellCommand(localCmdText)
+			if err := localCmd.Run(); err != nil {
+				return err
+			}
+		}
+	}
+
+	s.files = nil
+	return nil
+}

builder/osc/chroot/step_copy_files_test.go

@@ -0,0 +1,11 @@
+package chroot
+
+import "testing"
+
+func TestCopyFilesCleanupFunc_ImplementsCleanupFunc(t *testing.T) {
+	var raw interface{}
+	raw = new(StepCopyFiles)
+	if _, ok := raw.(Cleanup); !ok {
+		t.Fatalf("cleanup func should be a CleanupFunc")
+	}
+}

builder/osc/chroot/step_early_cleanup.go

@@ -0,0 +1,39 @@
+package chroot
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+// StepEarlyCleanup performs some of the cleanup steps early in order to
+// prepare for snapshotting and creating an AMI.
+type StepEarlyCleanup struct{}
+
+func (s *StepEarlyCleanup) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	ui := state.Get("ui").(packer.Ui)
+	cleanupKeys := []string{
+		"copy_files_cleanup",
+		"mount_extra_cleanup",
+		"mount_device_cleanup",
+		"attach_cleanup",
+	}
+
+	for _, key := range cleanupKeys {
+		c := state.Get(key).(Cleanup)
+		log.Printf("Running cleanup func: %s", key)
+		if err := c.CleanupFunc(state); err != nil {
+			err := fmt.Errorf("Error cleaning up: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+	}
+
+	return multistep.ActionContinue
+}
+
+func (s *StepEarlyCleanup) Cleanup(state multistep.StateBag) {}

builder/osc/chroot/step_early_unflock.go

@@ -0,0 +1,30 @@
+package chroot
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+// StepEarlyUnflock unlocks the flock.
+type StepEarlyUnflock struct{}
+
+func (s *StepEarlyUnflock) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	cleanup := state.Get("flock_cleanup").(Cleanup)
+	ui := state.Get("ui").(packer.Ui)
+
+	log.Println("Unlocking file lock...")
+	if err := cleanup.CleanupFunc(state); err != nil {
+		err := fmt.Errorf("Error unlocking file lock: %s", err)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	return multistep.ActionContinue
+}
+
+func (s *StepEarlyUnflock) Cleanup(state multistep.StateBag) {}

builder/osc/chroot/step_flock.go

@@ -0,0 +1,74 @@
+package chroot
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+// StepFlock provisions the instance within a chroot.
+//
+// Produces:
+//   flock_cleanup Cleanup - To perform early cleanup
+type StepFlock struct {
+	fh *os.File
+}
+
+func (s *StepFlock) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	ui := state.Get("ui").(packer.Ui)
+
+	lockfile := "/var/lock/packer-chroot/lock"
+	if err := os.MkdirAll(filepath.Dir(lockfile), 0755); err != nil {
+		err := fmt.Errorf("Error creating lock: %s", err)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	log.Printf("Obtaining lock: %s", lockfile)
+	f, err := os.Create(lockfile)
+	if err != nil {
+		err := fmt.Errorf("Error creating lock: %s", err)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	// LOCK!
+	if err := lockFile(f); err != nil {
+		err := fmt.Errorf("Error obtaining lock: %s", err)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	// Set the file handle, we can't close it because we need to hold
+	// the lock.
+	s.fh = f
+
+	state.Put("flock_cleanup", s)
+	return multistep.ActionContinue
+}
+
+func (s *StepFlock) Cleanup(state multistep.StateBag) {
+	s.CleanupFunc(state)
+}
+
+func (s *StepFlock) CleanupFunc(state multistep.StateBag) error {
+	if s.fh == nil {
+		return nil
+	}
+
+	log.Printf("Unlocking: %s", s.fh.Name())
+	if err := unlockFile(s.fh); err != nil {
+		return err
+	}
+
+	s.fh = nil
+	return nil
+}

builder/osc/chroot/step_mount_device.go

@@ -71,7 +71,8 @@ func (s *StepMountDevice) Run(_ context.Context, state multistep.StateBag) multi
 		return multistep.ActionHalt
 	}
 
-	log.Printf("Mount path: %s", mountPath)
+	log.Printf("[DEBUG] Device: %s", device)
+	log.Printf("[DEBUG] Mount path: %s", mountPath)
 
 	if err := os.MkdirAll(mountPath, 0755); err != nil {
 		err := fmt.Errorf("Error creating mount directory: %s", err)
@@ -92,9 +93,23 @@ func (s *StepMountDevice) Run(_ context.Context, state multistep.StateBag) multi
 		return multistep.ActionHalt
 	}
 
-	deviceMount := fmt.Sprintf("/dev/%s", strings.Replace(string(realDeviceName), "\n", "", -1))
+	log.Printf("[DEBUG] RealDeviceName: %s", realDeviceName)
+
+	realDeviceNameSplitted := strings.Split(string(realDeviceName), "\n")
+	log.Printf("[DEBUG] RealDeviceName Splitted %+v", realDeviceNameSplitted)
+	log.Printf("[DEBUG] RealDeviceName Splitted Length %d", len(realDeviceNameSplitted))
+	log.Printf("[DEBUG] RealDeviceName Splitted [0] %s", realDeviceNameSplitted[0])
+	log.Printf("[DEBUG] RealDeviceName Splitted [1] %s", realDeviceNameSplitted[1])
+
+	realDeviceNameStr := realDeviceNameSplitted[0]
+	if realDeviceNameStr == "" {
+		realDeviceNameStr = realDeviceNameSplitted[1]
+	}
+
+	deviceMount := fmt.Sprintf("/dev/%s", strings.Replace(realDeviceNameStr, "\n", "", -1))
 
 	log.Printf("[DEBUG] s.MountPartition  = %s", s.MountPartition)
+	log.Printf("[DEBUG ] DeviceMount: %s", deviceMount)
 
 	if virtualizationType == "hvm" && s.MountPartition != "0" {
 		deviceMount = fmt.Sprintf("%s%s", deviceMount, s.MountPartition)

builder/osc/chroot/step_mount_extra.go

@@ -0,0 +1,137 @@
+package chroot
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"syscall"
+
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+// StepMountExtra mounts the attached device.
+//
+// Produces:
+//   mount_extra_cleanup CleanupFunc - To perform early cleanup
+type StepMountExtra struct {
+	mounts []string
+}
+
+func (s *StepMountExtra) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	config := state.Get("config").(*Config)
+	mountPath := state.Get("mount_path").(string)
+	ui := state.Get("ui").(packer.Ui)
+	wrappedCommand := state.Get("wrappedCommand").(CommandWrapper)
+
+	s.mounts = make([]string, 0, len(config.ChrootMounts))
+
+	ui.Say("Mounting additional paths within the chroot...")
+	for _, mountInfo := range config.ChrootMounts {
+		innerPath := mountPath + mountInfo[2]
+
+		if err := os.MkdirAll(innerPath, 0755); err != nil {
+			err := fmt.Errorf("Error creating mount directory: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		flags := "-t " + mountInfo[0]
+		if mountInfo[0] == "bind" {
+			flags = "--bind"
+		}
+
+		ui.Message(fmt.Sprintf("Mounting: %s", mountInfo[2]))
+		stderr := new(bytes.Buffer)
+		mountCommand, err := wrappedCommand(fmt.Sprintf(
+			"mount %s %s %s",
+			flags,
+			mountInfo[1],
+			innerPath))
+		if err != nil {
+			err := fmt.Errorf("Error creating mount command: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		cmd := ShellCommand(mountCommand)
+		cmd.Stderr = stderr
+		if err := cmd.Run(); err != nil {
+			err := fmt.Errorf(
+				"Error mounting: %s\nStderr: %s", err, stderr.String())
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		s.mounts = append(s.mounts, innerPath)
+	}
+
+	state.Put("mount_extra_cleanup", s)
+	return multistep.ActionContinue
+}
+
+func (s *StepMountExtra) Cleanup(state multistep.StateBag) {
+	ui := state.Get("ui").(packer.Ui)
+
+	if err := s.CleanupFunc(state); err != nil {
+		ui.Error(err.Error())
+		return
+	}
+}
+
+func (s *StepMountExtra) CleanupFunc(state multistep.StateBag) error {
+	if s.mounts == nil {
+		return nil
+	}
+
+	wrappedCommand := state.Get("wrappedCommand").(CommandWrapper)
+	for len(s.mounts) > 0 {
+		var path string
+		lastIndex := len(s.mounts) - 1
+		path, s.mounts = s.mounts[lastIndex], s.mounts[:lastIndex]
+
+		grepCommand, err := wrappedCommand(fmt.Sprintf("grep %s /proc/mounts", path))
+		if err != nil {
+			return fmt.Errorf("Error creating grep command: %s", err)
+		}
+
+		// Before attempting to unmount,
+		// check to see if path is already unmounted
+		stderr := new(bytes.Buffer)
+		cmd := ShellCommand(grepCommand)
+		cmd.Stderr = stderr
+		if err := cmd.Run(); err != nil {
+			if exitError, ok := err.(*exec.ExitError); ok {
+				if status, ok := exitError.Sys().(syscall.WaitStatus); ok {
+					exitStatus := status.ExitStatus()
+					if exitStatus == 1 {
+						// path has already been unmounted
+						// just skip this path
+						continue
+					}
+				}
+			}
+		}
+
+		unmountCommand, err := wrappedCommand(fmt.Sprintf("umount %s", path))
+		if err != nil {
+			return fmt.Errorf("Error creating unmount command: %s", err)
+		}
+
+		stderr = new(bytes.Buffer)
+		cmd = ShellCommand(unmountCommand)
+		cmd.Stderr = stderr
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf(
+				"Error unmounting device: %s\nStderr: %s", err, stderr.String())
+		}
+	}
+
+	s.mounts = nil
+	return nil
+}

builder/osc/chroot/step_post_mount_commands.go

@@ -0,0 +1,47 @@
+package chroot
+
+import (
+	"context"
+
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+type postMountCommandsData struct {
+	Device    string
+	MountPath string
+}
+
+// StepPostMountCommands allows running arbitrary commands after mounting the
+// device, but prior to the bind mount and copy steps.
+type StepPostMountCommands struct {
+	Commands []string
+}
+
+func (s *StepPostMountCommands) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	config := state.Get("config").(*Config)
+	device := state.Get("device").(string)
+	mountPath := state.Get("mount_path").(string)
+	ui := state.Get("ui").(packer.Ui)
+	wrappedCommand := state.Get("wrappedCommand").(CommandWrapper)
+
+	if len(s.Commands) == 0 {
+		return multistep.ActionContinue
+	}
+
+	ictx := config.ctx
+	ictx.Data = &postMountCommandsData{
+		Device:    device,
+		MountPath: mountPath,
+	}
+
+	ui.Say("Running post-mount commands...")
+	if err := RunLocalCommands(s.Commands, wrappedCommand, ictx, ui); err != nil {
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+	return multistep.ActionContinue
+}
+
+func (s *StepPostMountCommands) Cleanup(state multistep.StateBag) {}

builder/osc/chroot/step_pre_mount_commands.go

@@ -0,0 +1,41 @@
+package chroot
+
+import (
+	"context"
+
+	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
+)
+
+type preMountCommandsData struct {
+	Device string
+}
+
+// StepPreMountCommands sets up the a new block device when building from scratch
+type StepPreMountCommands struct {
+	Commands []string
+}
+
+func (s *StepPreMountCommands) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
+	config := state.Get("config").(*Config)
+	device := state.Get("device").(string)
+	ui := state.Get("ui").(packer.Ui)
+	wrappedCommand := state.Get("wrappedCommand").(CommandWrapper)
+
+	if len(s.Commands) == 0 {
+		return multistep.ActionContinue
+	}
+
+	ictx := config.ctx
+	ictx.Data = &preMountCommandsData{Device: device}
+
+	ui.Say("Running device setup commands...")
+	if err := RunLocalCommands(s.Commands, wrappedCommand, ictx, ui); err != nil {
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+	return multistep.ActionContinue
+}
+
+func (s *StepPreMountCommands) Cleanup(state multistep.StateBag) {}