Merge pull request #2135 from mitchellh/f-acceptance

Builder acceptance test framework

.travis.yml

@@ -3,8 +3,6 @@ sudo: false
 language: go
 
 go:
-  - 1.2
-  - 1.3
   - 1.4
   - tip
 

Makefile

@@ -10,10 +10,23 @@ bin:
 dev:
 	@TF_DEV=1 sh -c "$(CURDIR)/scripts/build.sh"
 
+# generate runs `go generate` to build the dynamically generated
+# source files.
+generate:
+	go generate ./...
+
 test:
 	go test $(TEST) $(TESTARGS) -timeout=10s
 	@$(MAKE) vet
 
+# testacc runs acceptance tests
+testacc: generate
+	@if [ "$(TEST)" = "./..." ]; then \
+		echo "ERROR: Set TEST to a specific package"; \
+		exit 1; \
+	fi
+	PACKER_ACC=1 go test $(TEST) -v $(TESTARGS) -timeout 45m
+
 testrace:
 	go test -race $(TEST) $(TESTARGS)
 
@@ -30,4 +43,4 @@ vet:
 		echo "and fix them if necessary before submitting the code for reviewal."; \
 	fi
 
-.PHONY: bin default test updatedeps vet
+.PHONY: bin default generate test testacc updatedeps vet

README.md

@@ -121,3 +121,35 @@ package by specifying the `TEST` variable. For example below, only
 
     $ make test TEST=./packer
     ...
+
+### Acceptance Tests
+
+Packer has comprehensive [acceptance tests](https://en.wikipedia.org/wiki/Acceptance_testing)
+covering the builders of Packer.
+
+If you're working on a feature of a builder or a new builder and want
+verify it is functioning (and also hasn't broken anything else), we recommend
+running the acceptance tests.
+
+**Warning:** The acceptance tests create/destroy/modify *real resources*, which
+may incur real costs in some cases. In the presence of a bug, it is technically
+possible that broken backends could leave dangling data behind. Therefore,
+please run the acceptance tests at your own risk. At the very least,
+we recommend running them in their own private account for whatever builder
+you're testing.
+
+To run the acceptance tests, invoke `make testacc`:
+
+```sh
+$ make testacc TEST=./builder/amazon/ebs
+...
+```
+
+The `TEST` variable is required, and you should specify the folder where the
+backend is. The `TESTARGS` variable is recommended to filter down to a specific
+resource to test, since testing all of them at once can sometimes take a very
+long time.
+
+Acceptance tests typically require other environment variables to be set for
+things such as access keys. The test itself should error early and tell
+you what to set, so it is not documented here.

builder/amazon/ebs/builder_acc_test.go

@@ -0,0 +1,39 @@
+package ebs
+
+import (
+	"os"
+	"testing"
+
+	builderT "github.com/mitchellh/packer/helper/builder/testing"
+)
+
+func TestBuilderAcc_basic(t *testing.T) {
+	builderT.Test(t, builderT.TestCase{
+		PreCheck: func() { testAccPreCheck(t) },
+		Builder:  &Builder{},
+		Template: testBuilderAccBasic,
+	})
+}
+
+func testAccPreCheck(t *testing.T) {
+	if v := os.Getenv("AWS_ACCESS_KEY_ID"); v == "" {
+		t.Fatal("AWS_ACCESS_KEY_ID must be set for acceptance tests")
+	}
+
+	if v := os.Getenv("AWS_SECRET_ACCESS_KEY"); v == "" {
+		t.Fatal("AWS_SECRET_ACCESS_KEY must be set for acceptance tests")
+	}
+}
+
+const testBuilderAccBasic = `
+{
+	"builders": [{
+		"type": "test",
+		"region": "us-east-1",
+		"instance_type": "m3.medium",
+		"source_ami": "ami-76b2a71e",
+		"ssh_username": "ubuntu",
+		"ami_name": "packer-test {{timestamp}}"
+	}]
+}
+`

helper/builder/testing/testing.go

@@ -0,0 +1,186 @@
+package testing
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/mitchellh/packer/packer"
+	"github.com/mitchellh/packer/template"
+)
+
+// TestEnvVar must be set to a non-empty value for acceptance tests to run.
+const TestEnvVar = "PACKER_ACC"
+
+// TestCase is a single set of tests to run for a backend. A TestCase
+// should generally map 1:1 to each test method for your acceptance
+// tests.
+type TestCase struct {
+	// Precheck, if non-nil, will be called once before the test case
+	// runs at all. This can be used for some validation prior to the
+	// test running.
+	PreCheck func()
+
+	// Builder is the Builder that will be tested. It will be available
+	// as the "test" builder in the template.
+	Builder packer.Builder
+
+	// Template is the template contents to use.
+	Template string
+
+	// Check is called after this step is executed in order to test that
+	// the step executed successfully. If this is not set, then the next
+	// step will be called
+	Check TestCheckFunc
+
+	// Teardown will be called before the test case is over regardless
+	// of if the test succeeded or failed. This should return an error
+	// in the case that the test can't guarantee all resources were
+	// properly cleaned up.
+	Teardown TestTeardownFunc
+}
+
+// TestCheckFunc is the callback used for Check in TestStep.
+type TestCheckFunc func([]packer.Artifact) error
+
+// TestTeardownFunc is the callback used for Teardown in TestCase.
+type TestTeardownFunc func() error
+
+// TestT is the interface used to handle the test lifecycle of a test.
+//
+// Users should just use a *testing.T object, which implements this.
+type TestT interface {
+	Error(args ...interface{})
+	Fatal(args ...interface{})
+	Skip(args ...interface{})
+}
+
+// Test performs an acceptance test on a backend with the given test case.
+//
+// Tests are not run unless an environmental variable "TF_ACC" is
+// set to some non-empty value. This is to avoid test cases surprising
+// a user by creating real resources.
+//
+// Tests will fail unless the verbose flag (`go test -v`, or explicitly
+// the "-test.v" flag) is set. Because some acceptance tests take quite
+// long, we require the verbose flag so users are able to see progress
+// output.
+func Test(t TestT, c TestCase) {
+	// We only run acceptance tests if an env var is set because they're
+	// slow and generally require some outside configuration.
+	if os.Getenv(TestEnvVar) == "" {
+		t.Skip(fmt.Sprintf(
+			"Acceptance tests skipped unless env '%s' set",
+			TestEnvVar))
+		return
+	}
+
+	// We require verbose mode so that the user knows what is going on.
+	if !testTesting && !testing.Verbose() {
+		t.Fatal("Acceptance tests must be run with the -v flag on tests")
+		return
+	}
+
+	// Run the PreCheck if we have it
+	if c.PreCheck != nil {
+		c.PreCheck()
+	}
+
+	// Parse the template
+	log.Printf("[DEBUG] Parsing template...")
+	tpl, err := template.Parse(strings.NewReader(c.Template))
+	if err != nil {
+		t.Fatal(fmt.Sprintf("Failed to parse template: %s", err))
+		return
+	}
+
+	// Build the core
+	log.Printf("[DEBUG] Initializing core...")
+	core, err := packer.NewCore(&packer.CoreConfig{
+		Components: packer.ComponentFinder{
+			Builder: func(n string) (packer.Builder, error) {
+				if n == "test" {
+					return c.Builder, nil
+				}
+
+				return nil, nil
+			},
+		},
+		Template: tpl,
+	})
+	if err != nil {
+		t.Fatal(fmt.Sprintf("Failed to init core: %s", err))
+		return
+	}
+
+	// Get the build
+	log.Printf("[DEBUG] Retrieving 'test' build")
+	build, err := core.Build("test")
+	if err != nil {
+		t.Fatal(fmt.Sprintf("Failed to get 'test' build: %s", err))
+		return
+	}
+
+	// Prepare it
+	log.Printf("[DEBUG] Preparing 'test' build")
+	warnings, err := build.Prepare()
+	if err != nil {
+		t.Fatal(fmt.Sprintf("Prepare error: %s", err))
+		return
+	}
+	if len(warnings) > 0 {
+		t.Fatal(fmt.Sprintf(
+			"Prepare warnings:\n\n%s",
+			strings.Join(warnings, "\n")))
+		return
+	}
+
+	// Run it! We use a temporary directory for caching and discard
+	// any UI output. We discard since it shows up in logs anyways.
+	log.Printf("[DEBUG] Running 'test' build")
+	cache := &packer.FileCache{CacheDir: os.TempDir()}
+	ui := &packer.BasicUi{
+		Reader:      os.Stdin,
+		Writer:      ioutil.Discard,
+		ErrorWriter: ioutil.Discard,
+	}
+	artifacts, err := build.Run(ui, cache)
+	if err != nil {
+		t.Fatal(fmt.Sprintf("Run error:\n\n%s", err))
+		goto TEARDOWN
+	}
+
+	// Check function
+	if c.Check != nil {
+		log.Printf("[DEBUG] Running check function")
+		if err := c.Check(artifacts); err != nil {
+			t.Fatal(fmt.Sprintf("Check error:\n\n%s", err))
+			goto TEARDOWN
+		}
+	}
+
+TEARDOWN:
+	// Delete all artifacts
+	for _, a := range artifacts {
+		if err := a.Destroy(); err != nil {
+			t.Error(fmt.Sprintf(
+				"!!! ERROR REMOVING ARTIFACT '%s': %s !!!",
+				a.String(), err))
+		}
+	}
+
+	// Teardown
+	if c.Teardown != nil {
+		log.Printf("[DEBUG] Running teardown function")
+		if err := c.Teardown(); err != nil {
+			t.Fatal(fmt.Sprintf("Teardown failure:\n\n%s", err))
+			return
+		}
+	}
+}
+
+// This is for unit tests of this package.
+var testTesting = false

helper/builder/testing/testing_test.go

@@ -0,0 +1,88 @@
+package testing
+
+import (
+	"os"
+	"testing"
+)
+
+func init() {
+	testTesting = true
+
+	if err := os.Setenv(TestEnvVar, "1"); err != nil {
+		panic(err)
+	}
+}
+
+func TestTest_noEnv(t *testing.T) {
+	// Unset the variable
+	if err := os.Setenv(TestEnvVar, ""); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.Setenv(TestEnvVar, "1")
+
+	mt := new(mockT)
+	Test(mt, TestCase{})
+
+	if !mt.SkipCalled {
+		t.Fatal("skip not called")
+	}
+}
+
+func TestTest_preCheck(t *testing.T) {
+	called := false
+
+	mt := new(mockT)
+	Test(mt, TestCase{
+		PreCheck: func() { called = true },
+	})
+
+	if !called {
+		t.Fatal("precheck should be called")
+	}
+}
+
+// mockT implements TestT for testing
+type mockT struct {
+	ErrorCalled bool
+	ErrorArgs   []interface{}
+	FatalCalled bool
+	FatalArgs   []interface{}
+	SkipCalled  bool
+	SkipArgs    []interface{}
+
+	f bool
+}
+
+func (t *mockT) Error(args ...interface{}) {
+	t.ErrorCalled = true
+	t.ErrorArgs = args
+	t.f = true
+}
+
+func (t *mockT) Fatal(args ...interface{}) {
+	t.FatalCalled = true
+	t.FatalArgs = args
+	t.f = true
+}
+
+func (t *mockT) Skip(args ...interface{}) {
+	t.SkipCalled = true
+	t.SkipArgs = args
+	t.f = true
+}
+
+func (t *mockT) failed() bool {
+	return t.f
+}
+
+func (t *mockT) failMessage() string {
+	if t.FatalCalled {
+		return t.FatalArgs[0].(string)
+	} else if t.ErrorCalled {
+		return t.ErrorArgs[0].(string)
+	} else if t.SkipCalled {
+		return t.SkipArgs[0].(string)
+	}
+
+	return "unknown"
+}