add from_scratch option to amazon-chroot builder

This provides an alternate mode for the amazon-chroot builder which uses
a blank volume to build the image.  It adds StepPreMountCommands to
permit partitioning and format commands to be executed before mounting
the new volume.

builder/amazon/chroot/builder.go

@@ -25,19 +25,23 @@ const BuilderId = "mitchellh.amazon.chroot"
 // Config is the configuration that is chained through the steps and
 // settable from the template.
 type Config struct {
-	common.PackerConfig    `mapstructure:",squash"`
-	awscommon.AccessConfig `mapstructure:",squash"`
-	awscommon.AMIConfig    `mapstructure:",squash"`
+	common.PackerConfig       `mapstructure:",squash"`
+	awscommon.AMIBlockDevices `mapstructure:",squash"`
+	awscommon.AMIConfig       `mapstructure:",squash"`
+	awscommon.AccessConfig    `mapstructure:",squash"`
 
-	ChrootMounts   [][]string `mapstructure:"chroot_mounts"`
-	CommandWrapper string     `mapstructure:"command_wrapper"`
-	CopyFiles      []string   `mapstructure:"copy_files"`
-	DevicePath     string     `mapstructure:"device_path"`
-	MountPath      string     `mapstructure:"mount_path"`
-	SourceAmi      string     `mapstructure:"source_ami"`
-	RootVolumeSize int64      `mapstructure:"root_volume_size"`
-	MountOptions   []string   `mapstructure:"mount_options"`
-	MountPartition int        `mapstructure:"mount_partition"`
+	ChrootMounts     [][]string `mapstructure:"chroot_mounts"`
+	CommandWrapper   string     `mapstructure:"command_wrapper"`
+	CopyFiles        []string   `mapstructure:"copy_files"`
+	DevicePath       string     `mapstructure:"device_path"`
+	FromScratch      bool       `mapstructure:"from_scratch"`
+	MountOptions     []string   `mapstructure:"mount_options"`
+	MountPartition   int        `mapstructure:"mount_partition"`
+	MountPath        string     `mapstructure:"mount_path"`
+	PreMountCommands []string   `mapstructure:"pre_mount_commands"`
+	RootDeviceName   string     `mapstructure:"root_device_name"`
+	RootVolumeSize   int64      `mapstructure:"root_volume_size"`
+	SourceAmi        string     `mapstructure:"source_ami"`
 
 	ctx interpolate.Context
 }
@@ -59,6 +63,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 		InterpolateFilter: &interpolate.RenderFilter{
 			Exclude: []string{
 				"command_wrapper",
+				"pre_mount_commands",
 				"mount_path",
 			},
 		},
@@ -86,7 +91,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 		}
 	}
 
-	if len(b.config.CopyFiles) == 0 {
+	if len(b.config.CopyFiles) == 0 && !b.config.FromScratch {
 		b.config.CopyFiles = []string{"/etc/resolv.conf"}
 	}
 
@@ -115,8 +120,32 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 		}
 	}
 
-	if b.config.SourceAmi == "" {
-		errs = packer.MultiErrorAppend(errs, errors.New("source_ami is required."))
+	if b.config.FromScratch {
+		if b.config.RootVolumeSize == 0 {
+			errs = packer.MultiErrorAppend(
+				errs, errors.New("root_volume_size is required with from_scratch."))
+		}
+		if len(b.config.PreMountCommands) == 0 {
+			errs = packer.MultiErrorAppend(
+				errs, errors.New("pre_mount_commands is required with from_scratch."))
+		}
+		if b.config.AMIVirtType == "" {
+			errs = packer.MultiErrorAppend(
+				errs, errors.New("ami_virtualization_type is required with from_scratch."))
+		}
+		if b.config.RootDeviceName == "" {
+			errs = packer.MultiErrorAppend(
+				errs, errors.New("root_device_name is required with from_scratch."))
+		}
+		if len(b.config.AMIMappings) == 0 {
+			errs = packer.MultiErrorAppend(
+				errs, errors.New("ami_block_device_mappings is required with from_scratch."))
+		}
+	} else {
+		if b.config.SourceAmi == "" {
+			errs = packer.MultiErrorAppend(
+				errs, errors.New("source_ami is required."))
+		}
 	}
 
 	if errs != nil && len(errs.Errors) > 0 {
@@ -161,11 +190,19 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			ForceDeregister: b.config.AMIForceDeregister,
 		},
 		&StepInstanceInfo{},
-		&awscommon.StepSourceAMIInfo{
-			SourceAmi:          b.config.SourceAmi,
-			EnhancedNetworking: b.config.AMIEnhancedNetworking,
-		},
-		&StepCheckRootDevice{},
+	}
+
+	if !b.config.FromScratch {
+		steps = append(steps,
+			&awscommon.StepSourceAMIInfo{
+				SourceAmi:          b.config.SourceAmi,
+				EnhancedNetworking: b.config.AMIEnhancedNetworking,
+			},
+			&StepCheckRootDevice{},
+		)
+	}
+
+	steps = append(steps,
 		&StepFlock{},
 		&StepPrepareDevice{},
 		&StepCreateVolume{
@@ -173,6 +210,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		},
 		&StepAttachVolume{},
 		&StepEarlyUnflock{},
+		&StepPreMountCommands{
+			Commands: b.config.PreMountCommands,
+		},
 		&StepMountDevice{
 			MountOptions:   b.config.MountOptions,
 			MountPartition: b.config.MountPartition,
@@ -203,7 +243,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		&awscommon.StepCreateTags{
 			Tags: b.config.AMITags,
 		},
-	}
+	)
 
 	// Run!
 	if b.config.PackerDebug {

builder/amazon/chroot/run_local_commands.go

@@ -0,0 +1,37 @@
+package chroot
+
+import (
+	"fmt"
+
+	"github.com/mitchellh/packer/packer"
+	"github.com/mitchellh/packer/post-processor/shell-local"
+	"github.com/mitchellh/packer/template/interpolate"
+)
+
+func RunLocalCommands(commands []string, wrappedCommand CommandWrapper, ctx interpolate.Context, ui packer.Ui) error {
+	for _, rawCmd := range commands {
+		intCmd, err := interpolate.Render(rawCmd, &ctx)
+		if err != nil {
+			return fmt.Errorf("Error interpolating: %s", err)
+		}
+
+		command, err := wrappedCommand(intCmd)
+		if err != nil {
+			return fmt.Errorf("Error wrapping command: %s", err)
+		}
+
+		ui.Say(fmt.Sprintf("Executing command: %s", command))
+		comm := &shell_local.Communicator{}
+		cmd := &packer.RemoteCmd{Command: command}
+		if err := cmd.StartWithUi(comm, ui); err != nil {
+			return fmt.Errorf("Error executing command: %s", err)
+		}
+		if cmd.ExitStatus != 0 {
+			return fmt.Errorf(
+				"Received non-zero exit code %d from command: %s",
+				cmd.ExitStatus,
+				command)
+		}
+	}
+	return nil
+}

builder/amazon/chroot/step_create_volume.go

@@ -22,40 +22,52 @@ type StepCreateVolume struct {
 }
 
 func (s *StepCreateVolume) Run(state multistep.StateBag) multistep.StepAction {
+	config := state.Get("config").(*Config)
 	ec2conn := state.Get("ec2").(*ec2.EC2)
-	image := state.Get("source_image").(*ec2.Image)
 	instance := state.Get("instance").(*ec2.Instance)
 	ui := state.Get("ui").(packer.Ui)
 
-	// Determine the root device snapshot
-	log.Printf("Searching for root device of the image (%s)", *image.RootDeviceName)
-	var rootDevice *ec2.BlockDeviceMapping
-	for _, device := range image.BlockDeviceMappings {
-		if *device.DeviceName == *image.RootDeviceName {
-			rootDevice = device
-			break
+	var createVolume *ec2.CreateVolumeInput
+	if config.FromScratch {
+		createVolume = &ec2.CreateVolumeInput{
+			AvailabilityZone: instance.Placement.AvailabilityZone,
+			Size:             aws.Int64(s.RootVolumeSize),
+			VolumeType:       aws.String(ec2.VolumeTypeGp2),
+		}
+	} else {
+		// Determine the root device snapshot
+		image := state.Get("source_image").(*ec2.Image)
+		log.Printf("Searching for root device of the image (%s)", *image.RootDeviceName)
+		var rootDevice *ec2.BlockDeviceMapping
+		for _, device := range image.BlockDeviceMappings {
+			if *device.DeviceName == *image.RootDeviceName {
+				rootDevice = device
+				break
+			}
+		}
+
+		if rootDevice == nil {
+			err := fmt.Errorf("Couldn't find root device!")
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		ui.Say("Creating the root volume...")
+		vs := *rootDevice.Ebs.VolumeSize
+		if s.RootVolumeSize > *rootDevice.Ebs.VolumeSize {
+			vs = s.RootVolumeSize
+		}
+
+		createVolume = &ec2.CreateVolumeInput{
+			AvailabilityZone: instance.Placement.AvailabilityZone,
+			Size:             aws.Int64(vs),
+			SnapshotId:       rootDevice.Ebs.SnapshotId,
+			VolumeType:       rootDevice.Ebs.VolumeType,
+			Iops:             rootDevice.Ebs.Iops,
 		}
 	}
 
-	if rootDevice == nil {
-		err := fmt.Errorf("Couldn't find root device!")
-		state.Put("error", err)
-		ui.Error(err.Error())
-		return multistep.ActionHalt
-	}
-
-	ui.Say("Creating the root volume...")
-	vs := *rootDevice.Ebs.VolumeSize
-	if s.RootVolumeSize > *rootDevice.Ebs.VolumeSize {
-		vs = s.RootVolumeSize
-	}
-	createVolume := &ec2.CreateVolumeInput{
-		AvailabilityZone: instance.Placement.AvailabilityZone,
-		Size:             aws.Int64(vs),
-		SnapshotId:       rootDevice.Ebs.SnapshotId,
-		VolumeType:       rootDevice.Ebs.VolumeType,
-		Iops:             rootDevice.Ebs.Iops,
-	}
 	log.Printf("Create args: %+v", createVolume)
 
 	createVolumeResp, err := ec2conn.CreateVolume(createVolume)

builder/amazon/chroot/step_mount_device.go

@@ -33,10 +33,18 @@ type StepMountDevice struct {
 func (s *StepMountDevice) Run(state multistep.StateBag) multistep.StepAction {
 	config := state.Get("config").(*Config)
 	ui := state.Get("ui").(packer.Ui)
-	image := state.Get("source_image").(*ec2.Image)
 	device := state.Get("device").(string)
 	wrappedCommand := state.Get("wrappedCommand").(CommandWrapper)
 
+	var virtualizationType string
+	if config.FromScratch {
+		virtualizationType = config.AMIVirtType
+	} else {
+		image := state.Get("source_image").(*ec2.Image)
+		virtualizationType = *image.VirtualizationType
+		log.Printf("Source image virtualization type is: %s", virtualizationType)
+	}
+
 	ctx := config.ctx
 	ctx.Data = &mountPathData{Device: filepath.Base(device)}
 	mountPath, err := interpolate.Render(config.MountPath, &ctx)
@@ -65,9 +73,8 @@ func (s *StepMountDevice) Run(state multistep.StateBag) multistep.StepAction {
 		return multistep.ActionHalt
 	}
 
-	log.Printf("Source image virtualization type is: %s", *image.VirtualizationType)
 	deviceMount := device
-	if *image.VirtualizationType == "hvm" {
+	if virtualizationType == "hvm" {
 		deviceMount = fmt.Sprintf("%s%d", device, s.MountPartition)
 	}
 	state.Put("deviceMount", deviceMount)

builder/amazon/chroot/step_pre_mount_commands.go

@@ -0,0 +1,39 @@
+package chroot
+
+import (
+	"github.com/mitchellh/multistep"
+	"github.com/mitchellh/packer/packer"
+)
+
+type preMountCommandsData struct {
+	Device string
+}
+
+// StepPreMountCommands sets up the a new block device when building from scratch
+type StepPreMountCommands struct {
+	Commands []string
+}
+
+func (s *StepPreMountCommands) Run(state multistep.StateBag) multistep.StepAction {
+	config := state.Get("config").(*Config)
+	device := state.Get("device").(string)
+	ui := state.Get("ui").(packer.Ui)
+	wrappedCommand := state.Get("wrappedCommand").(CommandWrapper)
+
+	if len(s.Commands) == 0 {
+		return multistep.ActionContinue
+	}
+
+	ctx := config.ctx
+	ctx.Data = &preMountCommandsData{Device: device}
+
+	ui.Say("Running device setup commands...")
+	if err := RunLocalCommands(s.Commands, wrappedCommand, ctx, ui); err != nil {
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+	return multistep.ActionContinue
+}
+
+func (s *StepPreMountCommands) Cleanup(state multistep.StateBag) {}

builder/amazon/chroot/step_register_ami.go

@@ -18,22 +18,36 @@ type StepRegisterAMI struct {
 func (s *StepRegisterAMI) Run(state multistep.StateBag) multistep.StepAction {
 	config := state.Get("config").(*Config)
 	ec2conn := state.Get("ec2").(*ec2.EC2)
-	image := state.Get("source_image").(*ec2.Image)
 	snapshotId := state.Get("snapshot_id").(string)
 	ui := state.Get("ui").(packer.Ui)
 
 	ui.Say("Registering the AMI...")
-	blockDevices := make([]*ec2.BlockDeviceMapping, len(image.BlockDeviceMappings))
-	for i, device := range image.BlockDeviceMappings {
+
+	var (
+		registerOpts   *ec2.RegisterImageInput
+		blockDevices   []*ec2.BlockDeviceMapping
+		image          *ec2.Image
+		rootDeviceName string
+	)
+
+	if config.FromScratch {
+		blockDevices = config.AMIBlockDevices.BuildAMIDevices()
+		rootDeviceName = config.RootDeviceName
+	} else {
+		image = state.Get("source_image").(*ec2.Image)
+		blockDevices = make([]*ec2.BlockDeviceMapping, len(image.BlockDeviceMappings))
+		rootDeviceName = *image.RootDeviceName
+	}
+	for i, device := range blockDevices {
 		newDevice := device
-		if *newDevice.DeviceName == *image.RootDeviceName {
+		if *newDevice.DeviceName == rootDeviceName {
 			if newDevice.Ebs != nil {
 				newDevice.Ebs.SnapshotId = aws.String(snapshotId)
 			} else {
 				newDevice.Ebs = &ec2.EbsBlockDevice{SnapshotId: aws.String(snapshotId)}
 			}
 
-			if s.RootVolumeSize > *newDevice.Ebs.VolumeSize {
+			if config.FromScratch || s.RootVolumeSize > *newDevice.Ebs.VolumeSize {
 				newDevice.Ebs.VolumeSize = aws.Int64(s.RootVolumeSize)
 			}
 		}
@@ -47,7 +61,17 @@ func (s *StepRegisterAMI) Run(state multistep.StateBag) multistep.StepAction {
 		blockDevices[i] = newDevice
 	}
 
-	registerOpts := buildRegisterOpts(config, image, blockDevices)
+	if config.FromScratch {
+		registerOpts = &ec2.RegisterImageInput{
+			Name:                &config.AMIName,
+			Architecture:        aws.String(ec2.ArchitectureValuesX8664),
+			RootDeviceName:      aws.String(rootDeviceName),
+			VirtualizationType:  aws.String(config.AMIVirtType),
+			BlockDeviceMappings: blockDevices,
+		}
+	} else {
+		registerOpts = buildRegisterOpts(config, image, blockDevices)
+	}
 
 	// Set SriovNetSupport to "simple". See http://goo.gl/icuXh5
 	if config.AMIEnhancedNetworking {
@@ -105,6 +129,5 @@ func buildRegisterOpts(config *Config, image *ec2.Image, blockDevices []*ec2.Blo
 		registerOpts.KernelId = image.KernelId
 		registerOpts.RamdiskId = image.RamdiskId
 	}
-
 	return registerOpts
 }