Write generic Builder-Provisioner acceptance test logic (#8963)

Makefile

@@ -1,5 +1,7 @@
 TEST?=$(shell go list ./...)
 VET?=$(shell go list ./...)
+ACC_TEST_BUILDERS?=all
+ACC_TEST_PROVISIONERS?=all
 # Get the current full sha from git
 GITSHA:=$(shell git rev-parse HEAD)
 # Get the current local branch name from git (if we can, this may be blank)
@@ -133,6 +135,10 @@ generate-check: generate ## Check go code generation is on par
 test: mode-check vet ## Run unit tests
 	@go test $(TEST) $(TESTARGS) -timeout=3m
 
+# acctest runs provisioners acceptance tests
+provisioners-acctest: install-build-deps generate
+	ACC_TEST_BUILDERS=$(ACC_TEST_BUILDERS) ACC_TEST_PROVISIONERS=$(ACC_TEST_PROVISIONERS) go test ./provisioner/... -timeout=1h
+
 # testacc runs acceptance tests
 testacc: install-build-deps generate ## Run acceptance tests
 	@echo "WARN: Acceptance tests will take a long time to run and may cost money. Ctrl-C if you want to cancel."

builder/amazon/ebs/acceptance/builder_acceptance.go

@@ -0,0 +1,47 @@
+package amazon_acc
+
+// This is the code necessary for running the provisioner acceptance tests.
+// It provides the builder config and cleans up created resource.
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/packer/command"
+	"github.com/hashicorp/packer/packer"
+
+	testshelper "github.com/hashicorp/packer/helper/tests"
+)
+
+type AmazonEBSAccTest struct{}
+
+func (s *AmazonEBSAccTest) GetConfigs() (map[string]string, error) {
+	filePath := filepath.Join("../../builder/amazon/ebs/acceptance/test-fixtures/", "amazon-ebs.txt")
+	config, err := os.Open(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("Expected to find %s", filePath)
+	}
+	defer config.Close()
+
+	file, err := ioutil.ReadAll(config)
+	if err != nil {
+		return nil, fmt.Errorf("Uneble to read %s", filePath)
+	}
+	return map[string]string{"linux": string(file)}, nil
+}
+
+func (s *AmazonEBSAccTest) CleanUp() error {
+	helper := testshelper.AWSHelper{
+		Region:  "us-east-1",
+		AMIName: "packer-acc-test",
+	}
+	return helper.CleanUpAmi()
+}
+
+func (s *AmazonEBSAccTest) GetBuilderStore() packer.MapOfBuilder {
+	return packer.MapOfBuilder{
+		"amazon-ebs": func() (packer.Builder, error) { return command.Builders["amazon-ebs"], nil },
+	}
+}

builder/amazon/ebs/acceptance/test-fixtures/amazon-ebs.txt

@@ -0,0 +1,12 @@
+{
+  "type": "amazon-ebs",
+  "ami_name": "packer-acc-test",
+  "instance_type": "m1.small",
+  "region": "us-east-1",
+  "ssh_username": "ubuntu",
+  "source_ami": "ami-0568456c",
+  "force_deregister" : true,
+  "tags": {
+    "packer-test": "true"
+  }
+}

builder/virtualbox/iso/acceptance/builder_acceptance.go

@@ -0,0 +1,45 @@
+package virtualbox_acc
+
+// This is the code necessary for running the provisioner acceptance tests.
+// It provides the builder config and cleans up created resource.
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/packer/command"
+	"github.com/hashicorp/packer/packer"
+
+	testshelper "github.com/hashicorp/packer/helper/tests"
+)
+
+type VirtualBoxISOAccTest struct{}
+
+func (v *VirtualBoxISOAccTest) GetConfigs() (map[string]string, error) {
+	filePath := filepath.Join("../../builder/virtualbox/iso/acceptance/test-fixtures/", "virtualbox-iso.txt")
+	config, err := os.Open(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("Expected to find %s", filePath)
+	}
+	defer config.Close()
+
+	file, err := ioutil.ReadAll(config)
+	if err != nil {
+		return nil, fmt.Errorf("Uneble to read %s", filePath)
+	}
+	return map[string]string{"linux": string(file)}, nil
+}
+
+func (v *VirtualBoxISOAccTest) CleanUp() error {
+	testshelper.CleanupFiles("virtualbox-iso-packer-acc-test")
+	testshelper.CleanupFiles("packer_cache")
+	return nil
+}
+
+func (v *VirtualBoxISOAccTest) GetBuilderStore() packer.MapOfBuilder {
+	return packer.MapOfBuilder{
+		"virtualbox-iso": func() (packer.Builder, error) { return command.Builders["virtualbox-iso"], nil },
+	}
+}

builder/virtualbox/iso/acceptance/test-fixtures/http/preseed.cfg

@@ -0,0 +1,42 @@
+# Preseeding only locale sets language, country and locale.
+d-i debian-installer/locale string en_US
+
+# Keyboard selection.
+d-i console-setup/ask_detect boolean false
+d-i keyboard-configuration/xkb-keymap select us
+
+choose-mirror-bin mirror/http/proxy string
+d-i base-installer/kernel/override-image string linux-server
+d-i clock-setup/utc boolean true
+d-i clock-setup/utc-auto boolean true
+d-i finish-install/reboot_in_progress note
+d-i grub-installer/only_debian boolean true
+d-i grub-installer/with_other_os boolean true
+d-i mirror/country string manual
+d-i mirror/http/directory string /ubuntu/
+d-i mirror/http/hostname string archive.ubuntu.com
+d-i mirror/http/proxy string
+d-i partman-auto-lvm/guided_size string max
+d-i partman-auto/choose_recipe select atomic
+d-i partman-auto/method string lvm
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm_nooverwrite boolean true
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i partman/confirm_write_new_label boolean true
+d-i passwd/user-fullname string vagrant
+d-i passwd/user-uid string 1000
+d-i passwd/user-password password vagrant
+d-i passwd/user-password-again password vagrant
+d-i passwd/username string vagrant
+d-i pkgsel/include string openssh-server cryptsetup build-essential libssl-dev libreadline-dev zlib1g-dev linux-source dkms nfs-kernel-server nfs-common linux-headers-$(uname -r) perl
+d-i pkgsel/install-language-support boolean false
+d-i pkgsel/update-policy select none
+d-i pkgsel/upgrade select full-upgrade
+d-i time/zone string UTC
+d-i user-setup/allow-password-weak boolean true
+d-i user-setup/encrypt-home boolean false
+tasksel tasksel/first multiselect standard, server

builder/virtualbox/iso/acceptance/test-fixtures/virtualbox-iso.txt

@@ -0,0 +1,44 @@
+{
+  "type": "virtualbox-iso",
+  "iso_checksum": "946a6077af6f5f95a51f82fdc44051c7aa19f9cfc5f737954845a6050543d7c2",
+  "iso_checksum_type": "sha256",
+  "iso_url": "http://old-releases.ubuntu.com/releases/14.04.1/ubuntu-14.04.1-server-amd64.iso",
+  "disk_size": "40960",
+  "guest_os_type": "Ubuntu_64",
+  "headless" : "true",
+
+  "ssh_password": "vagrant",
+  "ssh_username": "vagrant",
+  "ssh_wait_timeout": "10000s",
+
+  "http_directory": "../../builder/virtualbox/iso/acceptance/test-fixtures/http",
+  "boot_wait": "10s",
+  "boot_command": [
+    "<esc><wait>",
+    "<esc><wait>",
+    "<enter><wait>",
+    "/install/vmlinuz<wait>",
+    " auto<wait>",
+    " console-setup/ask_detect=false<wait>",
+    " console-setup/layoutcode=us<wait>",
+    " console-setup/modelcode=pc105<wait>",
+    " debconf/frontend=noninteractive<wait>",
+    " debian-installer=en_US.UTF-8<wait>",
+    " fb=false<wait>",
+    " initrd=/install/initrd.gz<wait>",
+    " kbd-chooser/method=us<wait>",
+    " keyboard-configuration/layout=USA<wait>",
+    " keyboard-configuration/variant=USA<wait>",
+    " locale=en_US.UTF-8<wait>",
+    " netcfg/get_domain=vm<wait>",
+    " netcfg/get_hostname=vagrant<wait>",
+    " noapic<wait>",
+    " preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg<wait>",
+    " -- <wait>",
+    "<enter><wait>"
+  ],
+
+  "output_directory": "virtualbox-iso-packer-acc-test",
+  "shutdown_command": "echo 'vagrant' | sudo -S shutdown -P now",
+  "post_shutdown_delay": "60s"
+}

helper/tests/acc/provisioners.go

@@ -0,0 +1,130 @@
+package acc
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/hashicorp/packer/helper/tests"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	amazonEBS "github.com/hashicorp/packer/builder/amazon/ebs/acceptance"
+	virtualboxISO "github.com/hashicorp/packer/builder/virtualbox/iso/acceptance"
+	"github.com/hashicorp/packer/command"
+	"github.com/hashicorp/packer/packer"
+)
+
+func TestProvisionersAgainstBuilders(provisionerAcc ProvisionerAcceptance, t *testing.T) {
+	provisioner := provisionerAcc.GetName()
+	builders := checkBuilders(t)
+
+	// build template file and run a build for each builder with the provisioner
+	for _, builder := range builders {
+		builderAcc := BuildersAccTest[builder]
+		builderConfigs, err := builderAcc.GetConfigs()
+		if err != nil {
+			t.Fatalf("bad: failed to read builder config: %s", err.Error())
+		}
+
+		for vmOS, builderConfig := range builderConfigs {
+			if !provisionerAcc.IsCompatible(builder, vmOS) {
+				continue
+			}
+
+			testName := fmt.Sprintf("testing %s builder against %s provisioner", builder, provisioner)
+			t.Run(testName, func(t *testing.T) {
+				provisionerConfig, err := provisionerAcc.GetConfig()
+				if err != nil {
+					t.Fatalf("bad: failed to read provisioner config: %s", err.Error())
+				}
+
+				// Write json template
+				out := bytes.NewBuffer(nil)
+				fmt.Fprintf(out, `{"builders": [%s],"provisioners": [%s]}`, builderConfig, provisionerConfig)
+				fileName := fmt.Sprintf("%s_%s.json", builder, provisioner)
+				filePath := filepath.Join("./", fileName)
+				writeJsonTemplate(out, filePath, t)
+
+				// set pre-config with necessary builder and provisioner
+				c := buildCommand(t, builderAcc, provisionerAcc)
+				args := []string{
+					filePath,
+				}
+
+				err = provisionerAcc.RunTest(c, args)
+				// Cleanup created resources
+				testshelper.CleanupFiles(fileName)
+				cleanErr := builderAcc.CleanUp()
+				if cleanErr != nil {
+					log.Printf("bad: failed to clean up resources: %s", cleanErr.Error())
+				}
+				if err != nil {
+					t.Fatalf("bad: failed to to run build: %s", err.Error())
+				}
+			})
+		}
+	}
+}
+
+func checkBuilders(t *testing.T) []string {
+	b := os.Getenv("ACC_TEST_BUILDERS")
+	// validate if we want to run provisioners acc tests
+	if b == "" {
+		t.Skip("Provisioners Acceptance tests skipped unless env 'ACC_TEST_BUILDERS' is set")
+	}
+
+	// Get builders type to test provisioners against
+	var builders []string
+	for k := range BuildersAccTest {
+		// This will validate that only defined builders are executed against
+		if b != "all" && !strings.Contains(b, k) {
+			continue
+		}
+		builders = append(builders, k)
+	}
+	return builders
+}
+
+func writeJsonTemplate(out *bytes.Buffer, filePath string, t *testing.T) {
+	outputFile, err := os.Create(filePath)
+	if err != nil {
+		t.Fatalf("bad: failed to create template file: %s", err.Error())
+	}
+	_, err = outputFile.Write(out.Bytes())
+	if err != nil {
+		t.Fatalf("bad: failed to write template file: %s", err.Error())
+	}
+	outputFile.Sync()
+}
+
+func buildCommand(t *testing.T, builder BuilderAcceptance, provisioner ProvisionerAcceptance) *command.BuildCommand {
+	c := &command.BuildCommand{
+		Meta: testshelper.TestMetaFile(t),
+	}
+	c.CoreConfig.Components.BuilderStore = builder.GetBuilderStore()
+	c.CoreConfig.Components.ProvisionerStore = provisioner.GetProvisionerStore()
+
+	return c
+}
+
+type ProvisionerAcceptance interface {
+	GetName() string
+	GetConfig() (string, error)
+	GetProvisionerStore() packer.MapOfProvisioner
+	IsCompatible(builder string, vmOS string) bool
+	RunTest(c *command.BuildCommand, args []string) error
+}
+
+type BuilderAcceptance interface {
+	GetConfigs() (map[string]string, error)
+	GetBuilderStore() packer.MapOfBuilder
+	CleanUp() error
+}
+
+// List of all builders available for acceptance test
+var BuildersAccTest = map[string]BuilderAcceptance{
+	"virtualbox-iso": new(virtualboxISO.VirtualBoxISOAccTest),
+	"amazon-ebs":     new(amazonEBS.AmazonEBSAccTest),
+}

helper/tests/aws.go

@@ -1,8 +1,7 @@
 package testshelper
 
 import (
-	"testing"
-
+	"fmt"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	awscommon "github.com/hashicorp/packer/builder/amazon/common"
@@ -13,11 +12,11 @@ type AWSHelper struct {
 	AMIName string
 }
 
-func (a *AWSHelper) CleanUpAmi(t *testing.T) {
+func (a *AWSHelper) CleanUpAmi() error {
 	accessConfig := &awscommon.AccessConfig{}
 	session, err := accessConfig.Session()
 	if err != nil {
-		t.Errorf("AWSAMICleanUp: Unable to create aws session %s", err.Error())
+		return fmt.Errorf("AWSAMICleanUp: Unable to create aws session %s", err.Error())
 	}
 
 	regionconn := ec2.New(session.Copy(&aws.Config{
@@ -31,13 +30,14 @@ func (a *AWSHelper) CleanUpAmi(t *testing.T) {
 			Values: aws.StringSlice([]string{a.AMIName}),
 		}}})
 	if err != nil {
-		t.Errorf("AWSAMICleanUp: Unable to find Image %s: %s", a.AMIName, err.Error())
+		return fmt.Errorf("AWSAMICleanUp: Unable to find Image %s: %s", a.AMIName, err.Error())
 	}
 
 	_, err = regionconn.DeregisterImage(&ec2.DeregisterImageInput{
 		ImageId: resp.Images[0].ImageId,
 	})
 	if err != nil {
-		t.Errorf("AWSAMICleanUp: Unable to Deregister Image %s", err.Error())
+		return fmt.Errorf("AWSAMICleanUp: Unable to Deregister Image %s", err.Error())
 	}
+	return nil
 }

helper/tests/core.go

@@ -2,7 +2,6 @@ package testshelper
 
 import (
 	"bytes"
-	"fmt"
 	"os"
 	"testing"
 
@@ -56,32 +55,3 @@ func CleanupFiles(moreFiles ...string) {
 		os.RemoveAll(file)
 	}
 }
-
-func FatalCommand(t *testing.T, m command.Meta) {
-	ui := m.Ui.(*packer.BasicUi)
-	out := ui.Writer.(*bytes.Buffer)
-	err := ui.ErrorWriter.(*bytes.Buffer)
-	t.Fatalf(
-		"Bad exit code.\n\nStdout:\n\n%s\n\nStderr:\n\n%s",
-		out.String(),
-		err.String())
-}
-
-const TestEnvVar = "PACKER_ACC"
-
-func AccTestPreValidate(t *testing.T) {
-	// We only run acceptance tests if an env var is set because they're
-	// slow and generally require some outside configuration.
-	if os.Getenv(TestEnvVar) == "" {
-		t.Skip(fmt.Sprintf(
-			"Acceptance tests skipped unless env '%s' set",
-			TestEnvVar))
-		return
-	}
-
-	// We require verbose mode so that the user knows what is going on.
-	if !testing.Verbose() {
-		t.Fatal("Acceptance tests must be run with the -v flag on tests")
-		return
-	}
-}

provisioner/shell/provisioner_acc_test.go

@@ -1,41 +1,81 @@
 package shell_test
 
 import (
+	"bytes"
+	"fmt"
+	"github.com/hashicorp/packer/helper/tests/acc"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
+	"github.com/hashicorp/packer/packer"
+
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/packer/command"
 	testshelper "github.com/hashicorp/packer/helper/tests"
 )
 
-func TestBuildShellProvisionerWithBuildVariablesSharing(t *testing.T) {
-	testshelper.AccTestPreValidate(t)
+func TestShellProvisioner(t *testing.T) {
+	p := os.Getenv("ACC_TEST_PROVISIONERS")
+	if p != "all" && !strings.Contains(p, "shell") {
+		t.Skip()
+	}
+	acc.TestProvisionersAgainstBuilders(new(ShellProvisionerAccTest), t)
+}
 
-	UUID, _ := uuid.GenerateUUID()
-	os.Setenv("PACKER_RUN_UUID", UUID)
-	c := &command.BuildCommand{
-		Meta: testshelper.TestMetaFile(t),
+type ShellProvisionerAccTest struct{}
+
+func (s *ShellProvisionerAccTest) GetName() string {
+	return "shell"
+}
+
+func (s *ShellProvisionerAccTest) GetConfig() (string, error) {
+	filePath := filepath.Join("./test-fixtures", "shell-provisioner.txt")
+	config, err := os.Open(filePath)
+	if err != nil {
+		return "", fmt.Errorf("Expected to find %s", filePath)
+	}
+	defer config.Close()
+
+	file, err := ioutil.ReadAll(config)
+	return string(file), nil
+}
+
+func (s *ShellProvisionerAccTest) GetProvisionerStore() packer.MapOfProvisioner {
+	return packer.MapOfProvisioner{
+		"shell": func() (packer.Provisioner, error) { return command.Provisioners["shell"], nil },
+		"file":  func() (packer.Provisioner, error) { return command.Provisioners["file"], nil },
+	}
+}
+
+func (s *ShellProvisionerAccTest) IsCompatible(builder string, vmOS string) bool {
+	return vmOS == "linux"
+}
+
+func (s *ShellProvisionerAccTest) RunTest(c *command.BuildCommand, args []string) error {
+	UUID := os.Getenv("PACKER_RUN_UUID")
+	if UUID == "" {
+		UUID, _ = uuid.GenerateUUID()
+		os.Setenv("PACKER_RUN_UUID", UUID)
 	}
 
 	file := "provisioner.shell." + UUID + ".txt"
 	defer testshelper.CleanupFiles(file)
 
-	args := []string{
-		filepath.Join("./test-fixtures", "shell-provisioner.json"),
-	}
 	if code := c.Run(args); code != 0 {
-		testshelper.FatalCommand(t, c.Meta)
+		ui := c.Meta.Ui.(*packer.BasicUi)
+		out := ui.Writer.(*bytes.Buffer)
+		err := ui.ErrorWriter.(*bytes.Buffer)
+		return fmt.Errorf(
+			"Bad exit code.\n\nStdout:\n\n%s\n\nStderr:\n\n%s",
+			out.String(),
+			err.String())
 	}
 
 	if !testshelper.FileExists(file) {
-		t.Errorf("Expected to find %s", file)
-	} else {
-		helper := testshelper.AWSHelper{
-			Region:  "us-east-1",
-			AMIName: "packer-test-shell-interpolate",
-		}
-		helper.CleanUpAmi(t)
+		return fmt.Errorf("Expected to find %s", file)
 	}
+	return nil
 }

provisioner/shell/test-fixtures/shell-provisioner.json

@@ -1,30 +0,0 @@
-{
-  "builders": [
-    {
-      "type": "amazon-ebs",
-      "ami_name": "packer-test-shell-interpolate",
-      "instance_type": "m1.small",
-      "region": "us-east-1",
-      "ssh_username": "ubuntu",
-      "source_ami": "ami-0568456c",
-      "force_deregister" : true,
-      "tags": {
-        "packer-test": "true"
-      }
-    }
-  ],
-  "provisioners": [
-    {
-      "type": "shell",
-      "inline": [
-        "echo {{ build `ID`}} > provisioner.{{ build `PackerRunUUID`}}.txt"
-      ]
-    },
-    {
-      "type": "file",
-      "source": "provisioner.{{ build `PackerRunUUID`}}.txt",
-      "destination": "provisioner.shell.{{ build `PackerRunUUID`}}.txt",
-      "direction": "download"
-    }
-  ]
-}

provisioner/shell/test-fixtures/shell-provisioner.txt

@@ -0,0 +1,12 @@
+{
+  "type": "shell",
+  "inline": [
+    "echo {{ build `ID`}} > provisioner.{{ build `PackerRunUUID`}}.txt"
+  ]
+},
+{
+    "type": "file",
+    "source": "provisioner.{{ build `PackerRunUUID`}}.txt",
+    "destination": "provisioner.shell.{{ build `PackerRunUUID`}}.txt",
+    "direction": "download"
+}