From ac44f4175d92e6052028eea1fb68bf1fc7fb319a Mon Sep 17 00:00:00 2001 From: Peter Sankauskas Date: Fri, 13 Jun 2014 23:01:35 -0700 Subject: [PATCH 1/7] Adding a script to compile the linux binaries --- scripts/linuxcompile.sh | 60 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 scripts/linuxcompile.sh diff --git a/scripts/linuxcompile.sh b/scripts/linuxcompile.sh new file mode 100755 index 000000000..a7f693d38 --- /dev/null +++ b/scripts/linuxcompile.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# To be able to cross compile: +# cd /usr/local/go/src +# sudo GOOS=linux GOARCH=amd64 ./make.bash --no-clean +# +# This script only builds the application from source. +set -e + +NO_COLOR="\x1b[0m" +OK_COLOR="\x1b[32;01m" +ERROR_COLOR="\x1b[31;01m" +WARN_COLOR="\x1b[33;01m" + +# http://stackoverflow.com/questions/4023830/bash-how-compare-two-strings-in-version-format +verify_go () { + if [[ $1 == $2 ]]; then + return 0 + fi + + local IFS=. + local i ver1=($1) ver2=($2) + + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done + + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + echo -e "${ERROR_COLOR}==> Required Go version $1 not installed. Found $2 instead" + exit 1 + fi + done +} + +GO_MINIMUM_VERSION=1.2 +GO_INSTALLED_VERSION=$(go version | cut -d ' ' -f 3) +GO_INSTALLED_VERSION=${GO_INSTALLED_VERSION#"go"} + +echo -e "${OK_COLOR}==> Verifying Go" +verify_go $GO_MINIMUM_VERSION $GO_INSTALLED_VERSION + +# Get the parent directory of where this script is. +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done +DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" + +# Change into that directory +cd $DIR + +# Compile the thing +export XC_ARCH=amd64 +export XC_OS=linux +./scripts/compile.sh + +echo "-- DONE --" +echo "Binaries are in ./pkg/linux_amd64/" From a59ee93bcac73f58eabe77aebbe3b21e80da967c Mon Sep 17 00:00:00 2001 From: Peter Sankauskas Date: Tue, 29 Jul 2014 16:07:49 -0700 Subject: [PATCH 2/7] To be able to build both PV and HVM images, it is not possible to use both /dev/sd[f-p] and [1-16] as the HVM images get attached at /dev/sdf but must be mounted a /dev/sdf1. This reduces the number of simultaneous packer runs possible significantly, but unless you are Netflix, who have Aminator anyway, this is probably never going to be an issue --- builder/amazon/chroot/device.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/builder/amazon/chroot/device.go b/builder/amazon/chroot/device.go index 77b7b444a..a0c0b9dc8 100644 --- a/builder/amazon/chroot/device.go +++ b/builder/amazon/chroot/device.go @@ -27,11 +27,12 @@ func AvailableDevice() (string, error) { continue } - for i := 1; i < 16; i++ { - device := fmt.Sprintf("/dev/%s%c%d", prefix, letter, i) - if _, err := os.Stat(device); err != nil { - return device, nil - } + // To be able to build both Paravirtual and HVM images, the unnumbered + // device and the first numbered one must be available. + // E.g. /dev/xvdf and /dev/xvdf1 + numbered_device := fmt.Sprintf("%s%d", device, 1) + if _, err := os.Stat(numbered_device); err != nil { + return device, nil } } From 6b751cac392d5856189ade81d544e1f11b4fa1a0 Mon Sep 17 00:00:00 2001 From: Peter Sankauskas Date: Tue, 29 Jul 2014 16:15:53 -0700 Subject: [PATCH 3/7] Formatting --- builder/amazon/chroot/device.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/chroot/device.go b/builder/amazon/chroot/device.go index a0c0b9dc8..02a68433c 100644 --- a/builder/amazon/chroot/device.go +++ b/builder/amazon/chroot/device.go @@ -28,7 +28,7 @@ func AvailableDevice() (string, error) { } // To be able to build both Paravirtual and HVM images, the unnumbered - // device and the first numbered one must be available. + // device and the first numbered one must be available. // E.g. /dev/xvdf and /dev/xvdf1 numbered_device := fmt.Sprintf("%s%d", device, 1) if _, err := os.Stat(numbered_device); err != nil { From 460e2da2485d1392b6a28103e37f98331d85e625 Mon Sep 17 00:00:00 2001 From: Peter Sankauskas Date: Tue, 29 Jul 2014 21:55:20 -0700 Subject: [PATCH 4/7] The mount command for a PV image that is attached to /dev/sdf is: mount /dev/xvdf /mnt/point while for an HVM image that is attached to /dev/sdf, its mount command is mount /dev/xvdf1 /mnt/point so this code enabled that --- builder/amazon/chroot/step_mount_device.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/builder/amazon/chroot/step_mount_device.go b/builder/amazon/chroot/step_mount_device.go index a6419774f..3c3d959c1 100644 --- a/builder/amazon/chroot/step_mount_device.go +++ b/builder/amazon/chroot/step_mount_device.go @@ -3,6 +3,7 @@ package chroot import ( "bytes" "fmt" + "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" "log" @@ -26,6 +27,7 @@ type StepMountDevice struct { func (s *StepMountDevice) Run(state multistep.StateBag) multistep.StepAction { config := state.Get("config").(*Config) ui := state.Get("ui").(packer.Ui) + image := state.Get("source_image").(*ec2.Image) device := state.Get("device").(string) wrappedCommand := state.Get("wrappedCommand").(CommandWrapper) @@ -57,10 +59,17 @@ func (s *StepMountDevice) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } + log.Printf("Source image virtualization type is: %s", image.VirtualizationType) + deviceMount := device + if image.VirtualizationType == "hvm" { + deviceMount = fmt.Sprintf("%s%d", device, 1) + } + state.Put("deviceMount", deviceMount) + ui.Say("Mounting the root device...") stderr := new(bytes.Buffer) mountCommand, err := wrappedCommand( - fmt.Sprintf("mount %s %s", device, mountPath)) + fmt.Sprintf("mount %s %s", deviceMount, mountPath)) if err != nil { err := fmt.Errorf("Error creating mount command: %s", err) state.Put("error", err) From a2c0b104f0504e26a4be149784320f7859585c99 Mon Sep 17 00:00:00 2001 From: Peter Sankauskas Date: Tue, 29 Jul 2014 21:56:37 -0700 Subject: [PATCH 5/7] Adding the conditional necessary to construct the right options for both PV and HVM images. Also adding a test to make sure it is doing the right thing --- builder/amazon/chroot/step_register_ami.go | 26 ++++-- .../amazon/chroot/step_register_ami_test.go | 83 +++++++++++++++++++ 2 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 builder/amazon/chroot/step_register_ami_test.go diff --git a/builder/amazon/chroot/step_register_ami.go b/builder/amazon/chroot/step_register_ami.go index 69ec6dbb6..62e6a3ff0 100644 --- a/builder/amazon/chroot/step_register_ami.go +++ b/builder/amazon/chroot/step_register_ami.go @@ -30,14 +30,7 @@ func (s *StepRegisterAMI) Run(state multistep.StateBag) multistep.StepAction { blockDevices[i] = newDevice } - registerOpts := &ec2.RegisterImage{ - Name: config.AMIName, - Architecture: image.Architecture, - KernelId: image.KernelId, - RamdiskId: image.RamdiskId, - RootDeviceName: image.RootDeviceName, - BlockDevices: blockDevices, - } + registerOpts := buildRegisterOpts(config, image, blockDevices) // Set SriovNetSupport to "simple". See http://goo.gl/icuXh5 if config.AMIEnhancedNetworking { @@ -77,3 +70,20 @@ func (s *StepRegisterAMI) Run(state multistep.StateBag) multistep.StepAction { } func (s *StepRegisterAMI) Cleanup(state multistep.StateBag) {} + +func buildRegisterOpts(config *Config, image *ec2.Image, blockDevices []ec2.BlockDeviceMapping) *ec2.RegisterImage { + registerOpts := &ec2.RegisterImage{ + Name: config.AMIName, + Architecture: image.Architecture, + RootDeviceName: image.RootDeviceName, + BlockDevices: blockDevices, + VirtType: config.AMIVirtType, + } + + if config.AMIVirtType != "hvm" { + registerOpts.KernelId = image.KernelId + registerOpts.RamdiskId = image.RamdiskId + } + + return registerOpts +} diff --git a/builder/amazon/chroot/step_register_ami_test.go b/builder/amazon/chroot/step_register_ami_test.go new file mode 100644 index 000000000..7d75687bc --- /dev/null +++ b/builder/amazon/chroot/step_register_ami_test.go @@ -0,0 +1,83 @@ +package chroot + +import ( + "fmt" + "testing" + + "github.com/kr/pretty" + "github.com/mitchellh/goamz/ec2" + // awscommon "github.com/mitchellh/packer/builder/amazon/common" +) + +func testImage() ec2.Image { + return ec2.Image{ + Id: "ami-abcd1234", + Name: "ami_test_name", + Architecture: "x86_64", + KernelId: "aki-abcd1234", + } +} + +func TestStepRegisterAmi_buildRegisterOpts_pv(t *testing.T) { + config := Config{} + config.AMIName = "test_ami_name" + config.AMIDescription = "test_ami_description" + config.AMIVirtType = "paravirtual" + + image := testImage() + + blockDevices := []ec2.BlockDeviceMapping{} + + opts := buildRegisterOpts(&config, &image, blockDevices) + + fmt.Println("******** PAS ********") + fmt.Printf("%# v", pretty.Formatter(opts)) + + expected := config.AMIVirtType + if opts.VirtType != expected { + t.Fatalf("Unexpected VirtType value: expected %s got %s\n", expected, opts.VirtType) + } + + expected = config.AMIName + if opts.Name != expected { + t.Fatalf("Unexpected Name value: expected %s got %s\n", expected, opts.Name) + } + + expected = image.KernelId + if opts.KernelId != expected { + t.Fatalf("Unexpected KernelId value: expected %s got %s\n", expected, opts.KernelId) + } + +} + +func TestStepRegisterAmi_buildRegisterOpts_hvm(t *testing.T) { + config := Config{} + config.AMIName = "test_ami_name" + config.AMIDescription = "test_ami_description" + config.AMIVirtType = "hvm" + + image := testImage() + + blockDevices := []ec2.BlockDeviceMapping{} + + opts := buildRegisterOpts(&config, &image, blockDevices) + + fmt.Println("******** PAS ********") + fmt.Printf("%# v", pretty.Formatter(opts)) + + expected := config.AMIVirtType + if opts.VirtType != expected { + t.Fatalf("Unexpected VirtType value: expected %s got %s\n", expected, opts.VirtType) + } + + expected = config.AMIName + if opts.Name != expected { + t.Fatalf("Unexpected Name value: expected %s got %s\n", expected, opts.Name) + } + + expected = "" + if opts.KernelId != expected { + t.Fatalf("Unexpected KernelId value: expected %s got %s\n", expected, opts.KernelId) + } + +} From e878495f6792ffed61fe8410ccffda1ab06af386 Mon Sep 17 00:00:00 2001 From: Peter Sankauskas Date: Tue, 29 Jul 2014 22:10:49 -0700 Subject: [PATCH 6/7] Removing my helper script --- scripts/linuxcompile.sh | 60 ----------------------------------------- 1 file changed, 60 deletions(-) delete mode 100755 scripts/linuxcompile.sh diff --git a/scripts/linuxcompile.sh b/scripts/linuxcompile.sh deleted file mode 100755 index a7f693d38..000000000 --- a/scripts/linuxcompile.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# To be able to cross compile: -# cd /usr/local/go/src -# sudo GOOS=linux GOARCH=amd64 ./make.bash --no-clean -# -# This script only builds the application from source. -set -e - -NO_COLOR="\x1b[0m" -OK_COLOR="\x1b[32;01m" -ERROR_COLOR="\x1b[31;01m" -WARN_COLOR="\x1b[33;01m" - -# http://stackoverflow.com/questions/4023830/bash-how-compare-two-strings-in-version-format -verify_go () { - if [[ $1 == $2 ]]; then - return 0 - fi - - local IFS=. - local i ver1=($1) ver2=($2) - - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do - ver1[i]=0 - done - - for ((i=0; i<${#ver1[@]}; i++)); do - if [[ -z ${ver2[i]} ]]; then - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})); then - echo -e "${ERROR_COLOR}==> Required Go version $1 not installed. Found $2 instead" - exit 1 - fi - done -} - -GO_MINIMUM_VERSION=1.2 -GO_INSTALLED_VERSION=$(go version | cut -d ' ' -f 3) -GO_INSTALLED_VERSION=${GO_INSTALLED_VERSION#"go"} - -echo -e "${OK_COLOR}==> Verifying Go" -verify_go $GO_MINIMUM_VERSION $GO_INSTALLED_VERSION - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that directory -cd $DIR - -# Compile the thing -export XC_ARCH=amd64 -export XC_OS=linux -./scripts/compile.sh - -echo "-- DONE --" -echo "Binaries are in ./pkg/linux_amd64/" From ab9f0bc3c8b46f943048a672377fa40688164a6e Mon Sep 17 00:00:00 2001 From: Peter Sankauskas Date: Tue, 29 Jul 2014 22:18:43 -0700 Subject: [PATCH 7/7] Removing my debugging output --- builder/amazon/chroot/step_register_ami_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/builder/amazon/chroot/step_register_ami_test.go b/builder/amazon/chroot/step_register_ami_test.go index 7d75687bc..f85106a8d 100644 --- a/builder/amazon/chroot/step_register_ami_test.go +++ b/builder/amazon/chroot/step_register_ami_test.go @@ -1,12 +1,8 @@ package chroot import ( - "fmt" "testing" - - "github.com/kr/pretty" "github.com/mitchellh/goamz/ec2" - // awscommon "github.com/mitchellh/packer/builder/amazon/common" ) func testImage() ec2.Image { @@ -30,9 +26,6 @@ func TestStepRegisterAmi_buildRegisterOpts_pv(t *testing.T) { opts := buildRegisterOpts(&config, &image, blockDevices) - fmt.Println("******** PAS ********") - fmt.Printf("%# v", pretty.Formatter(opts)) - expected := config.AMIVirtType if opts.VirtType != expected { t.Fatalf("Unexpected VirtType value: expected %s got %s\n", expected, opts.VirtType) @@ -62,9 +55,6 @@ func TestStepRegisterAmi_buildRegisterOpts_hvm(t *testing.T) { opts := buildRegisterOpts(&config, &image, blockDevices) - fmt.Println("******** PAS ********") - fmt.Printf("%# v", pretty.Formatter(opts)) - expected := config.AMIVirtType if opts.VirtType != expected { t.Fatalf("Unexpected VirtType value: expected %s got %s\n", expected, opts.VirtType)