Merge pull request #5086 from localghost/ansible_local_playbook_files

Add playbook_files to execute multiple ansible playbooks with ansible-local.

provisioner/ansible-local/communicator_mock.go

@@ -0,0 +1,38 @@
+package ansiblelocal
+
+import (
+	"github.com/hashicorp/packer/packer"
+	"io"
+	"os"
+)
+
+type communicatorMock struct {
+	startCommand      []string
+	uploadDestination []string
+}
+
+func (c *communicatorMock) Start(cmd *packer.RemoteCmd) error {
+	c.startCommand = append(c.startCommand, cmd.Command)
+	cmd.SetExited(0)
+	return nil
+}
+
+func (c *communicatorMock) Upload(dst string, _ io.Reader, _ *os.FileInfo) error {
+	c.uploadDestination = append(c.uploadDestination, dst)
+	return nil
+}
+
+func (c *communicatorMock) UploadDir(dst, src string, exclude []string) error {
+	return nil
+}
+
+func (c *communicatorMock) Download(src string, dst io.Writer) error {
+	return nil
+}
+
+func (c *communicatorMock) DownloadDir(src, dst string, exclude []string) error {
+	return nil
+}
+
+func (c *communicatorMock) verify() {
+}

provisioner/ansible-local/provisioner.go

@@ -38,6 +38,9 @@ type Config struct {
 	// The main playbook file to execute.
 	PlaybookFile string `mapstructure:"playbook_file"`
 
+	// The playbook files to execute.
+	PlaybookFiles []string `mapstructure:"playbook_files"`
+
 	// An array of local paths of playbook files to upload.
 	PlaybookPaths []string `mapstructure:"playbook_paths"`
 
@@ -66,6 +69,8 @@ type Config struct {
 
 type Provisioner struct {
 	config Config
+
+	playbookFiles []string
 }
 
 func (p *Provisioner) Prepare(raws ...interface{}) error {
@@ -80,6 +85,9 @@ func (p *Provisioner) Prepare(raws ...interface{}) error {
 		return err
 	}
 
+	// Reset the state.
+	p.playbookFiles = make([]string, 0, len(p.config.PlaybookFiles))
+
 	// Defaults
 	if p.config.Command == "" {
 		p.config.Command = "ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook"
@@ -94,9 +102,32 @@ func (p *Provisioner) Prepare(raws ...interface{}) error {
 
 	// Validation
 	var errs *packer.MultiError
-	err = validateFileConfig(p.config.PlaybookFile, "playbook_file", true)
-	if err != nil {
-		errs = packer.MultiErrorAppend(errs, err)
+
+	// Check that either playbook_file or playbook_files is specified
+	if len(p.config.PlaybookFiles) != 0 && p.config.PlaybookFile != "" {
+		errs = packer.MultiErrorAppend(errs, fmt.Errorf("Either playbook_file or playbook_files can be specified, not both"))
+	}
+	if len(p.config.PlaybookFiles) == 0 && p.config.PlaybookFile == "" {
+		errs = packer.MultiErrorAppend(errs, fmt.Errorf("Either playbook_file or playbook_files must be specified"))
+	}
+	if p.config.PlaybookFile != "" {
+		err = validateFileConfig(p.config.PlaybookFile, "playbook_file", true)
+		if err != nil {
+			errs = packer.MultiErrorAppend(errs, err)
+		}
+	}
+
+	for _, playbookFile := range p.config.PlaybookFiles {
+		if err := validateFileConfig(playbookFile, "playbook_files", true); err != nil {
+			errs = packer.MultiErrorAppend(errs, err)
+		} else {
+			playbookFile, err := filepath.Abs(playbookFile)
+			if err != nil {
+				errs = packer.MultiErrorAppend(errs, err)
+			} else {
+				p.playbookFiles = append(p.playbookFiles, playbookFile)
+			}
+		}
 	}
 
 	// Check that the inventory file exists, if configured
@@ -169,11 +200,15 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
 		}
 	}
 
-	ui.Message("Uploading main Playbook file...")
-	src := p.config.PlaybookFile
-	dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src)))
-	if err := p.uploadFile(ui, comm, dst, src); err != nil {
-		return fmt.Errorf("Error uploading main playbook: %s", err)
+	if p.config.PlaybookFile != "" {
+		ui.Message("Uploading main Playbook file...")
+		src := p.config.PlaybookFile
+		dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src)))
+		if err := p.uploadFile(ui, comm, dst, src); err != nil {
+			return fmt.Errorf("Error uploading main playbook: %s", err)
+		}
+	} else if err := p.provisionPlaybookFiles(ui, comm); err != nil {
+		return err
 	}
 
 	if len(p.config.InventoryFile) == 0 {
@@ -204,16 +239,16 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
 
 	if len(p.config.GalaxyFile) > 0 {
 		ui.Message("Uploading galaxy file...")
-		src = p.config.GalaxyFile
-		dst = filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src)))
+		src := p.config.GalaxyFile
+		dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src)))
 		if err := p.uploadFile(ui, comm, dst, src); err != nil {
 			return fmt.Errorf("Error uploading galaxy file: %s", err)
 		}
 	}
 
 	ui.Message("Uploading inventory file...")
-	src = p.config.InventoryFile
-	dst = filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src)))
+	src := p.config.InventoryFile
+	dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src)))
 	if err := p.uploadFile(ui, comm, dst, src); err != nil {
 		return fmt.Errorf("Error uploading inventory file: %s", err)
 	}
@@ -279,6 +314,44 @@ func (p *Provisioner) Cancel() {
 	os.Exit(0)
 }
 
+func (p *Provisioner) provisionPlaybookFiles(ui packer.Ui, comm packer.Communicator) error {
+	var playbookDir string
+	if p.config.PlaybookDir != "" {
+		var err error
+		playbookDir, err = filepath.Abs(p.config.PlaybookDir)
+		if err != nil {
+			return err
+		}
+	}
+	for index, playbookFile := range p.playbookFiles {
+		if playbookDir != "" && strings.HasPrefix(playbookFile, playbookDir) {
+			p.playbookFiles[index] = strings.TrimPrefix(playbookFile, playbookDir)
+			continue
+		}
+		if err := p.provisionPlaybookFile(ui, comm, playbookFile); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *Provisioner) provisionPlaybookFile(ui packer.Ui, comm packer.Communicator, playbookFile string) error {
+	ui.Message(fmt.Sprintf("Uploading playbook file: %s", playbookFile))
+
+	remoteDir := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Dir(playbookFile)))
+	remotePlaybookFile := filepath.ToSlash(filepath.Join(p.config.StagingDir, playbookFile))
+
+	if err := p.createDir(ui, comm, remoteDir); err != nil {
+		return fmt.Errorf("Error uploading playbook file: %s [%s]", playbookFile, err)
+	}
+
+	if err := p.uploadFile(ui, comm, remotePlaybookFile, playbookFile); err != nil {
+		return fmt.Errorf("Error uploading playbook: %s [%s]", playbookFile, err)
+	}
+
+	return nil
+}
+
 func (p *Provisioner) executeGalaxy(ui packer.Ui, comm packer.Communicator) error {
 	rolesDir := filepath.ToSlash(filepath.Join(p.config.StagingDir, "roles"))
 	galaxyFile := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.GalaxyFile)))
@@ -301,7 +374,6 @@ func (p *Provisioner) executeGalaxy(ui packer.Ui, comm packer.Communicator) erro
 }
 
 func (p *Provisioner) executeAnsible(ui packer.Ui, comm packer.Communicator) error {
-	playbook := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.PlaybookFile)))
 	inventory := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.InventoryFile)))
 
 	extraArgs := fmt.Sprintf(" --extra-vars \"packer_build_name=%s packer_builder_type=%s packer_http_addr=%s\" ",
@@ -317,8 +389,28 @@ func (p *Provisioner) executeAnsible(ui packer.Ui, comm packer.Communicator) err
 		}
 	}
 
+	if p.config.PlaybookFile != "" {
+		playbookFile := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.PlaybookFile)))
+		if err := p.executeAnsiblePlaybook(ui, comm, playbookFile, extraArgs, inventory); err != nil {
+			return err
+		}
+	}
+
+	for _, playbookFile := range p.playbookFiles {
+		playbookFile = filepath.ToSlash(filepath.Join(p.config.StagingDir, playbookFile))
+		if err := p.executeAnsiblePlaybook(ui, comm, playbookFile, extraArgs, inventory); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *Provisioner) executeAnsiblePlaybook(
+	ui packer.Ui, comm packer.Communicator, playbookFile, extraArgs, inventory string,
+) error {
 	command := fmt.Sprintf("cd %s && %s %s%s -c local -i %s",
-		p.config.StagingDir, p.config.Command, playbook, extraArgs, inventory)
+		p.config.StagingDir, p.config.Command, playbookFile, extraArgs, inventory,
+	)
 	ui.Message(fmt.Sprintf("Executing Ansible: %s", command))
 	cmd := &packer.RemoteCmd{
 		Command: command,

provisioner/ansible-local/provisioner_test.go

@@ -7,14 +7,14 @@ import (
 	"strings"
 	"testing"
 
+	"fmt"
+	"github.com/hashicorp/packer/builder/docker"
 	"github.com/hashicorp/packer/packer"
+	"github.com/hashicorp/packer/provisioner/file"
+	"github.com/hashicorp/packer/template"
+	"os/exec"
 )
 
-func testConfig() map[string]interface{} {
-	m := make(map[string]interface{})
-	return m
-}
-
 func TestProvisioner_Impl(t *testing.T) {
 	var raw interface{}
 	raw = &Provisioner{}
@@ -73,6 +73,107 @@ func TestProvisionerPrepare_PlaybookFile(t *testing.T) {
 	}
 }
 
+func TestProvisionerPrepare_PlaybookFiles(t *testing.T) {
+	var p Provisioner
+	config := testConfig()
+
+	err := p.Prepare(config)
+	if err == nil {
+		t.Fatal("should have error")
+	}
+
+	config["playbook_file"] = ""
+	config["playbook_files"] = []string{}
+	err = p.Prepare(config)
+	if err == nil {
+		t.Fatal("should have error")
+	}
+
+	playbook_file, err := ioutil.TempFile("", "playbook")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.Remove(playbook_file.Name())
+
+	config["playbook_file"] = playbook_file.Name()
+	config["playbook_files"] = []string{"some_other_file"}
+	err = p.Prepare(config)
+	if err == nil {
+		t.Fatal("should have error")
+	}
+
+	p = Provisioner{}
+	config["playbook_file"] = playbook_file.Name()
+	config["playbook_files"] = []string{}
+	err = p.Prepare(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	config["playbook_file"] = ""
+	config["playbook_files"] = []string{playbook_file.Name()}
+	err = p.Prepare(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestProvisionerProvision_PlaybookFiles(t *testing.T) {
+	var p Provisioner
+	config := testConfig()
+
+	playbooks := createTempFiles("", 3)
+	defer removeFiles(playbooks...)
+
+	config["playbook_files"] = playbooks
+	err := p.Prepare(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	comm := &communicatorMock{}
+	if err := p.Provision(&uiStub{}, comm); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	assertPlaybooksUploaded(comm, playbooks)
+	assertPlaybooksExecuted(comm, playbooks)
+}
+
+func TestProvisionerProvision_PlaybookFilesWithPlaybookDir(t *testing.T) {
+	var p Provisioner
+	config := testConfig()
+
+	playbook_dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("Failed to create playbook_dir: %s", err)
+	}
+	defer os.RemoveAll(playbook_dir)
+	playbooks := createTempFiles(playbook_dir, 3)
+
+	playbookNames := make([]string, 0, len(playbooks))
+	playbooksInPlaybookDir := make([]string, 0, len(playbooks))
+	for _, playbook := range playbooks {
+		playbooksInPlaybookDir = append(playbooksInPlaybookDir, strings.TrimPrefix(playbook, playbook_dir))
+		playbookNames = append(playbookNames, filepath.Base(playbook))
+	}
+
+	config["playbook_files"] = playbooks
+	config["playbook_dir"] = playbook_dir
+	err = p.Prepare(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	comm := &communicatorMock{}
+	if err := p.Provision(&uiStub{}, comm); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	assertPlaybooksNotUploaded(comm, playbookNames)
+	assertPlaybooksExecuted(comm, playbooksInPlaybookDir)
+}
+
 func TestProvisionerPrepare_InventoryFile(t *testing.T) {
 	var p Provisioner
 	config := testConfig()
@@ -211,3 +312,216 @@ func TestProvisionerPrepare_CleanStagingDir(t *testing.T) {
 		t.Fatalf("expected clean_staging_directory to be set")
 	}
 }
+
+func TestProvisionerProvisionDocker_PlaybookFiles(t *testing.T) {
+	testProvisionerProvisionDockerWithPlaybookFiles(t, playbookFilesDockerTemplate)
+}
+
+func TestProvisionerProvisionDocker_PlaybookFilesWithPlaybookDir(t *testing.T) {
+	testProvisionerProvisionDockerWithPlaybookFiles(t, playbookFilesWithPlaybookDirDockerTemplate)
+}
+
+func testProvisionerProvisionDockerWithPlaybookFiles(t *testing.T, templateString string) {
+	if os.Getenv("PACKER_ACC") == "" {
+		t.Skip("This test is only run with PACKER_ACC=1")
+	}
+
+	ui := packer.TestUi(t)
+	cache := &packer.FileCache{CacheDir: os.TempDir()}
+
+	tpl, err := template.Parse(strings.NewReader(templateString))
+	if err != nil {
+		t.Fatalf("Unable to parse config: %s", err)
+	}
+
+	// Check if docker executable can be found.
+	_, err = exec.LookPath("docker")
+	if err != nil {
+		t.Error("docker command not found; please make sure docker is installed")
+	}
+
+	// Setup the builder
+	builder := &docker.Builder{}
+	warnings, err := builder.Prepare(tpl.Builders["docker"].Config)
+	if err != nil {
+		t.Fatalf("Error preparing configuration %s", err)
+	}
+	if len(warnings) > 0 {
+		t.Fatal("Encountered configuration warnings; aborting")
+	}
+
+	ansible := &Provisioner{}
+	err = ansible.Prepare(tpl.Provisioners[0].Config)
+	if err != nil {
+		t.Fatalf("Error preparing ansible-local provisioner: %s", err)
+	}
+
+	download := &file.Provisioner{}
+	err = download.Prepare(tpl.Provisioners[1].Config)
+	if err != nil {
+		t.Fatalf("Error preparing download: %s", err)
+	}
+
+	// Add hooks so the provisioners run during the build
+	hooks := map[string][]packer.Hook{}
+	hooks[packer.HookProvision] = []packer.Hook{
+		&packer.ProvisionHook{
+			Provisioners: []*packer.HookedProvisioner{
+				{ansible, nil, ""},
+				{download, nil, ""},
+			},
+		},
+	}
+	hook := &packer.DispatchHook{Mapping: hooks}
+
+	artifact, err := builder.Run(ui, hook, cache)
+	if err != nil {
+		t.Fatalf("Error running build %s", err)
+	}
+	defer os.Remove("hello_world")
+	defer artifact.Destroy()
+
+	actualContent, err := ioutil.ReadFile("hello_world")
+	if err != nil {
+		t.Fatalf("Expected file not found: %s", err)
+	}
+
+	expectedContent := "Hello world!"
+	if string(actualContent) != expectedContent {
+		t.Fatalf(`Unexpected file content: expected="%s", actual="%s"`, expectedContent, actualContent)
+	}
+}
+
+func assertPlaybooksExecuted(comm *communicatorMock, playbooks []string) {
+	cmdIndex := 0
+	for _, playbook := range playbooks {
+		playbook = filepath.ToSlash(playbook)
+		for ; cmdIndex < len(comm.startCommand); cmdIndex++ {
+			cmd := comm.startCommand[cmdIndex]
+			if strings.Contains(cmd, "ansible-playbook") && strings.Contains(cmd, playbook) {
+				break
+			}
+		}
+		if cmdIndex == len(comm.startCommand) {
+			panic(fmt.Sprintf("Playbook %s was not executed", playbook))
+		}
+	}
+}
+
+func assertPlaybooksUploaded(comm *communicatorMock, playbooks []string) {
+	uploadIndex := 0
+	for _, playbook := range playbooks {
+		playbook = filepath.ToSlash(playbook)
+		for ; uploadIndex < len(comm.uploadDestination); uploadIndex++ {
+			dest := comm.uploadDestination[uploadIndex]
+			if strings.HasSuffix(dest, playbook) {
+				break
+			}
+		}
+		if uploadIndex == len(comm.uploadDestination) {
+			panic(fmt.Sprintf("Playbook %s was not uploaded", playbook))
+		}
+	}
+}
+
+func assertPlaybooksNotUploaded(comm *communicatorMock, playbooks []string) {
+	for _, playbook := range playbooks {
+		playbook = filepath.ToSlash(playbook)
+		for _, destination := range comm.uploadDestination {
+			if strings.HasSuffix(destination, playbook) {
+				panic(fmt.Sprintf("Playbook %s was uploaded", playbook))
+			}
+		}
+	}
+}
+
+func testConfig() map[string]interface{} {
+	m := make(map[string]interface{})
+	return m
+}
+
+func createTempFile(dir string) string {
+	file, err := ioutil.TempFile(dir, "")
+	if err != nil {
+		panic(fmt.Sprintf("err: %s", err))
+	}
+	return file.Name()
+}
+
+func createTempFiles(dir string, numFiles int) []string {
+	files := make([]string, 0, numFiles)
+	defer func() {
+		// Cleanup the files if not all were created.
+		if len(files) < numFiles {
+			for _, file := range files {
+				os.Remove(file)
+			}
+		}
+	}()
+
+	for i := 0; i < numFiles; i++ {
+		files = append(files, createTempFile(dir))
+	}
+	return files
+}
+
+func removeFiles(files ...string) {
+	for _, file := range files {
+		os.Remove(file)
+	}
+}
+
+const playbookFilesDockerTemplate = `
+{
+	"builders": [
+		{
+			"type": "docker",
+			"image": "williamyeh/ansible:centos7",
+			"discard": true
+		}
+	],
+	"provisioners": [
+		{
+			"type": "ansible-local",
+			"playbook_files": [
+				"test-fixtures/hello.yml",
+				"test-fixtures/world.yml"
+			]
+		},
+		{
+			"type": "file",
+			"source": "/tmp/hello_world",
+			"destination": "hello_world",
+			"direction": "download"
+		}
+	]
+}
+`
+
+const playbookFilesWithPlaybookDirDockerTemplate = `
+{
+	"builders": [
+		{
+			"type": "docker",
+			"image": "williamyeh/ansible:centos7",
+			"discard": true
+		}
+	],
+	"provisioners": [
+		{
+			"type": "ansible-local",
+			"playbook_files": [
+				"test-fixtures/hello.yml",
+				"test-fixtures/world.yml"
+			],
+			"playbook_dir": "test-fixtures"
+		},
+		{
+			"type": "file",
+			"source": "/tmp/hello_world",
+			"destination": "hello_world",
+			"direction": "download"
+		}
+	]
+}
+`

provisioner/ansible-local/test-fixtures/hello.yml

@@ -0,0 +1,5 @@
+---
+- hosts: all
+  tasks:
+    - name: write Hello
+      shell: echo -n "Hello" >> /tmp/hello_world

provisioner/ansible-local/test-fixtures/world.yml

@@ -0,0 +1,5 @@
+---
+- hosts: all
+  tasks:
+    - name: write world!
+      shell: echo -n " world!" >> /tmp/hello_world

provisioner/ansible-local/ui_stub.go

@@ -0,0 +1,15 @@
+package ansiblelocal
+
+type uiStub struct{}
+
+func (su *uiStub) Ask(string) (string, error) {
+	return "", nil
+}
+
+func (su *uiStub) Error(string) {}
+
+func (su *uiStub) Machine(string, ...string) {}
+
+func (su *uiStub) Message(string) {}
+
+func (su *uiStub) Say(msg string) {}

website/source/docs/provisioners/ansible-local.html.md

@@ -47,7 +47,12 @@ Required:
 
 -   `playbook_file` (string) - The playbook file to be executed by ansible. This
     file must exist on your local system and will be uploaded to the
-    remote machine.
+    remote machine. This option is exclusive with `playbook_files`.
+    
+-   `playbook_files` (array of strings) - The playbook files to be executed by ansible.
+    These files must exist on your local system. If the files don't exist in the `playbook_dir`
+    or you don't set `playbook_dir` they will be uploaded to the remote machine. This option
+    is exclusive with `playbook_file`.
 
 Optional: