From fa2f277c67f545b745c027211066c7eda4d805bf Mon Sep 17 00:00:00 2001 From: Jerry Clinesmith Date: Tue, 10 Sep 2013 22:00:29 -0500 Subject: [PATCH 01/46] #348: chef-solo provisioner: add support for data_bags and roles --- provisioner/chef-solo/provisioner.go | 64 +++++++++++++++++++++-- provisioner/chef-solo/provisioner_test.go | 22 ++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/provisioner/chef-solo/provisioner.go b/provisioner/chef-solo/provisioner.go index fe96ad467..7aa4cfc59 100644 --- a/provisioner/chef-solo/provisioner.go +++ b/provisioner/chef-solo/provisioner.go @@ -18,6 +18,8 @@ type Config struct { common.PackerConfig `mapstructure:",squash"` CookbookPaths []string `mapstructure:"cookbook_paths"` + RolesPath string `mapstructure:"roles_path"` + DataBagsPath string `mapstructure:"data_bags_path"` ExecuteCommand string `mapstructure:"execute_command"` InstallCommand string `mapstructure:"install_command"` RemoteCookbookPaths []string `mapstructure:"remote_cookbook_paths"` @@ -35,7 +37,9 @@ type Provisioner struct { } type ConfigTemplate struct { - CookbookPaths string + CookbookPaths string + RolesPath string + DataBagsPath string } type ExecuteTemplate struct { @@ -129,6 +133,24 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { errs, fmt.Errorf("Bad cookbook path '%s': %s", path, err)) } } + + if p.config.RolesPath != "" { + pFileInfo, err := os.Stat(p.config.RolesPath) + + if err != nil || !pFileInfo.IsDir() { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Bad roles path '%s': %s", p.config.RolesPath, err)) + } + } + + if p.config.DataBagsPath != "" { + pFileInfo, err := os.Stat(p.config.DataBagsPath) + + if err != nil || !pFileInfo.IsDir() { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Bad data bags path '%s': %s", p.config.DataBagsPath, err)) + } + } // Process the user variables within the JSON and set the JSON. // Do this early so that we can validate and show errors. @@ -165,8 +187,24 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { cookbookPaths = append(cookbookPaths, targetPath) } + + rolesPath := "" + if p.config.RolesPath != "" { + rolesPath := fmt.Sprintf("%s/roles", p.config.StagingDir) + if err := p.uploadDirectory(ui, comm, rolesPath, p.config.RolesPath); err != nil { + return fmt.Errorf("Error uploading roles: %s", err) + } + } - configPath, err := p.createConfig(ui, comm, cookbookPaths) + dataBagsPath := "" + if p.config.DataBagsPath != "" { + dataBagsPath := fmt.Sprintf("%s/data_bags", p.config.StagingDir) + if err := p.uploadDirectory(ui, comm, dataBagsPath, p.config.DataBagsPath); err != nil { + return fmt.Errorf("Error uploading data bags: %s", err) + } + } + + configPath, err := p.createConfig(ui, comm, cookbookPaths, rolesPath, dataBagsPath) if err != nil { return fmt.Errorf("Error creating Chef config file: %s", err) } @@ -203,7 +241,7 @@ func (p *Provisioner) uploadDirectory(ui packer.Ui, comm packer.Communicator, ds return comm.UploadDir(dst, src, nil) } -func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, localCookbooks []string) (string, error) { +func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, localCookbooks []string, rolesPath string, dataBagsPath string) (string, error) { ui.Message("Creating configuration file 'solo.rb'") cookbook_paths := make([]string, len(p.config.RemoteCookbookPaths)+len(localCookbooks)) @@ -215,9 +253,21 @@ func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, local i = len(p.config.RemoteCookbookPaths) + i cookbook_paths[i] = fmt.Sprintf(`"%s"`, path) } + + roles_path := "" + if rolesPath != "" { + roles_path = fmt.Sprintf(`"%s"`, rolesPath) + } + + data_bags_path := "" + if dataBagsPath != "" { + data_bags_path = fmt.Sprintf(`"%s"`, dataBagsPath) + } configString, err := p.config.tpl.Process(DefaultConfigTemplate, &ConfigTemplate{ CookbookPaths: strings.Join(cookbook_paths, ","), + RolesPath: roles_path, + DataBagsPath: data_bags_path, }) if err != nil { return "", err @@ -368,5 +418,11 @@ func (p *Provisioner) processJsonUserVars() (map[string]interface{}, error) { } var DefaultConfigTemplate = ` -cookbook_path [{{.CookbookPaths}}] +cookbook_path [{{.CookbookPaths}}] +{{if .RolesPath != ""}} +role_path {{.RolesPath}} +{{end}} +{{if .DataBagsPath != ""}} +data_bag_path {{.DataBagsPath}} +{{end}} ` diff --git a/provisioner/chef-solo/provisioner_test.go b/provisioner/chef-solo/provisioner_test.go index 332718d01..73d882e95 100644 --- a/provisioner/chef-solo/provisioner_test.go +++ b/provisioner/chef-solo/provisioner_test.go @@ -32,11 +32,25 @@ func TestProvisionerPrepare_cookbookPaths(t *testing.T) { t.Fatalf("err: %s", err) } + rolesPath, err := ioutil.TempDir("", "roles") + if err != nil { + t.Fatalf("err: %s", err) + } + + dataBagsPath, err := ioutil.TempDir("", "data_bags") + if err != nil { + t.Fatalf("err: %s", err) + } + defer os.Remove(path1) defer os.Remove(path2) + defer os.Remove(rolesPath) + defer os.Remove(dataBagsPath) config := testConfig() config["cookbook_paths"] = []string{path1, path2} + config["roles_path"] = rolesPath + config["data_bags_path"] = dataBagsPath err = p.Prepare(config) if err != nil { @@ -50,6 +64,14 @@ func TestProvisionerPrepare_cookbookPaths(t *testing.T) { if p.config.CookbookPaths[0] != path1 || p.config.CookbookPaths[1] != path2 { t.Fatalf("unexpected: %#v", p.config.CookbookPaths) } + + if p.config.RolesPath != rolesPath { + t.Fatalf("unexpected: %#v", p.config.RolesPath) + } + + if p.config.DataBagsPath != dataBagsPath { + t.Fatalf("unexpected: %#v", p.config.DataBagsPath) + } } func TestProvisionerPrepare_json(t *testing.T) { From bcf6d537dcd486c37e9f9509b70b7055f6b8edf8 Mon Sep 17 00:00:00 2001 From: Jerry Clinesmith Date: Tue, 10 Sep 2013 22:00:52 -0500 Subject: [PATCH 02/46] #348: chef-solo provisioner: update docs --- website/source/docs/provisioners/chef-solo.html.markdown | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/website/source/docs/provisioners/chef-solo.html.markdown b/website/source/docs/provisioners/chef-solo.html.markdown index df6101b90..d5b6f4b01 100644 --- a/website/source/docs/provisioners/chef-solo.html.markdown +++ b/website/source/docs/provisioners/chef-solo.html.markdown @@ -37,6 +37,14 @@ configuration is actually required, but at least `run_list` is recommended. to the remote machine in the directory specified by the `staging_directory`. By default, this is empty. +* `roles_path` (string) - The path to the "roles" directory on your local filesystem. + These will be uploaded to the remote machine in the directory specified by the + `staging_directory`. By default, this is empty. + +* `data_bags_path` (string) - The path to the "data_bags" directory on your local filesystem. + These will be uploaded to the remote machine in the directory specified by the + `staging_directory`. By default, this is empty. + * `execute_command` (string) - The command used to execute Chef. This has various [configuration template variables](/docs/templates/configuration-templates.html) available. See below for more information. From dcb1bb7624f15b24984cde558feeb0683a685318 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Tue, 17 Sep 2013 04:35:07 -0700 Subject: [PATCH 03/46] updatedeps target. --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 776d27dca..934a12852 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ NO_COLOR=\033[0m OK_COLOR=\033[32;01m ERROR_COLOR=\033[31;01m WARN_COLOR=\033[33;01m +DEPS = $(go list -f '{{range .TestImports}}{{.}} {{end}}' ./...) all: deps @mkdir -p bin/ @@ -11,7 +12,11 @@ all: deps deps: @echo "$(OK_COLOR)==> Installing dependencies$(NO_COLOR)" @go get -d -v ./... - @go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs -n1 go get -d + @echo $(DEPS) | xargs -n1 go get -d + +updatedeps: + @echo "$(OK_COLOR)==> Updating all dependencies$(NO_COLOR)" + @echo $(DEPS) | xargs -n1 go get -d -u clean: @rm -rf bin/ local/ pkg/ src/ website/.sass-cache website/build From bd0eac81041086915fddb736b4d3b70bc168dc5b Mon Sep 17 00:00:00 2001 From: Ali Abbas Date: Tue, 17 Sep 2013 21:12:50 +0200 Subject: [PATCH 04/46] Enforce go build version dependency --- go_version.go | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 go_version.go diff --git a/go_version.go b/go_version.go new file mode 100644 index 000000000..4a557c621 --- /dev/null +++ b/go_version.go @@ -0,0 +1,3 @@ +// +build !go1.1 + +"packer requires go version 1.1 or greated to build" From c7e16811630d76bdd9f5e7ebaac84362bce6523c Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Wed, 18 Sep 2013 13:42:15 -0700 Subject: [PATCH 05/46] exclude everything in /tmp, but keep /tmp itself There is IMO a bug with image builder that it removes /tmp, in the current setup. This patch makes the image bundle ignore everyting in /tmp, but keeps /tmp on the box. --- builder/amazon/instance/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 071e0e24e..44691ecee 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -83,7 +83,7 @@ func (b *Builder) Prepare(raws ...interface{}) error { "-u {{.AccountId}} " + "-c {{.CertPath}} " + "-r {{.Architecture}} " + - "-e {{.PrivatePath}} " + + "-e {{.PrivatePath}}/* " + "-d {{.Destination}} " + "-p {{.Prefix}} " + "--batch" From b3559b90fd4ab5ff05c214dec3685a3fd978590f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 13:50:01 -0700 Subject: [PATCH 06/46] Update CHANGELOG --- CHANGELOG.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2ee3d54..dec40d02c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,15 @@ FEATURES: IMPROVEMENTS: -* builder/amazon/*: Interrupts work while waiting for AMI to be ready. +* builder/amazon/all: Interrupts work while waiting for AMI to be ready. BUG FIXES: -* builder/amazon/*: While waiting for AMI, will detect "failed" state. -* builder/amazon/*: Waiting for state will detect if the resource (AMI, +* builder/amazon/all: While waiting for AMI, will detect "failed" state. +* builder/amazon/all: Waiting for state will detect if the resource (AMI, instance, etc.) disappears from under it. +* builder/amazon/instance: Exclude only contents of /tmp, not /tmp + itself. [GH-437] * builder/virtualbox: F1-F12 and delete scancodes now work. [GH-425] * provisioner/puppet-masterless: Fix failure case when both facter vars are used and prevent_sudo. [GH-415] From cf10fff7fa23b93e37ed3881ac4da9abea66744b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 13:59:23 -0700 Subject: [PATCH 07/46] builder/amazon/common: save access/secret key from env [GH-434] --- CHANGELOG.md | 2 ++ builder/amazon/common/access_config.go | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dec40d02c..ee5aad398 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ BUG FIXES: instance, etc.) disappears from under it. * builder/amazon/instance: Exclude only contents of /tmp, not /tmp itself. [GH-437] +* builder/amazon/instance: Make AccessKey/SecretKey available to bundle + command even when they come from the environment. [GH-434] * builder/virtualbox: F1-F12 and delete scancodes now work. [GH-425] * provisioner/puppet-masterless: Fix failure case when both facter vars are used and prevent_sudo. [GH-415] diff --git a/builder/amazon/common/access_config.go b/builder/amazon/common/access_config.go index 10ce2eba2..1690869d0 100644 --- a/builder/amazon/common/access_config.go +++ b/builder/amazon/common/access_config.go @@ -18,7 +18,14 @@ type AccessConfig struct { // Auth returns a valid aws.Auth object for access to AWS services, or // an error if the authentication couldn't be resolved. func (c *AccessConfig) Auth() (aws.Auth, error) { - return aws.GetAuth(c.AccessKey, c.SecretKey) + auth, err := aws.GetAuth(c.AccessKey, c.SecretKey) + if err == nil { + // Store the accesskey and secret that we got... + c.AccessKey = auth.AccessKey + c.SecretKey = auth.SecretKey + } + + return auth, err } // Region returns the aws.Region object for access to AWS services, requesting From 76bb457e0fb811731073b55baacb4503e5f99b2a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 14:13:04 -0700 Subject: [PATCH 08/46] Revert "Enforce go build version dependency" This reverts commit 0a1d696457ffe859ef8581e249230f39c1723958. Unfortunately, this was causing `go fmt ./...` to fail. --- go_version.go | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 go_version.go diff --git a/go_version.go b/go_version.go deleted file mode 100644 index 4a557c621..000000000 --- a/go_version.go +++ /dev/null @@ -1,3 +0,0 @@ -// +build !go1.1 - -"packer requires go version 1.1 or greated to build" From b2d5b15155cb489fce874c635da8daf3e1935d25 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 14:14:18 -0700 Subject: [PATCH 09/46] fmt --- provisioner/chef-solo/provisioner.go | 24 +++++++++++------------ provisioner/chef-solo/provisioner_test.go | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/provisioner/chef-solo/provisioner.go b/provisioner/chef-solo/provisioner.go index d06c3785f..1262ab6d2 100644 --- a/provisioner/chef-solo/provisioner.go +++ b/provisioner/chef-solo/provisioner.go @@ -20,8 +20,8 @@ type Config struct { ConfigTemplate string `mapstructure:"config_template"` CookbookPaths []string `mapstructure:"cookbook_paths"` - RolesPath string `mapstructure:"roles_path"` - DataBagsPath string `mapstructure:"data_bags_path"` + RolesPath string `mapstructure:"roles_path"` + DataBagsPath string `mapstructure:"data_bags_path"` ExecuteCommand string `mapstructure:"execute_command"` InstallCommand string `mapstructure:"install_command"` RemoteCookbookPaths []string `mapstructure:"remote_cookbook_paths"` @@ -39,9 +39,9 @@ type Provisioner struct { } type ConfigTemplate struct { - CookbookPaths string - RolesPath string - DataBagsPath string + CookbookPaths string + RolesPath string + DataBagsPath string } type ExecuteTemplate struct { @@ -147,10 +147,10 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { errs, fmt.Errorf("Bad cookbook path '%s': %s", path, err)) } } - + if p.config.RolesPath != "" { pFileInfo, err := os.Stat(p.config.RolesPath) - + if err != nil || !pFileInfo.IsDir() { errs = packer.MultiErrorAppend( errs, fmt.Errorf("Bad roles path '%s': %s", p.config.RolesPath, err)) @@ -159,7 +159,7 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { if p.config.DataBagsPath != "" { pFileInfo, err := os.Stat(p.config.DataBagsPath) - + if err != nil || !pFileInfo.IsDir() { errs = packer.MultiErrorAppend( errs, fmt.Errorf("Bad data bags path '%s': %s", p.config.DataBagsPath, err)) @@ -201,7 +201,7 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { cookbookPaths = append(cookbookPaths, targetPath) } - + rolesPath := "" if p.config.RolesPath != "" { rolesPath := fmt.Sprintf("%s/roles", p.config.StagingDir) @@ -267,7 +267,7 @@ func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, local i = len(p.config.RemoteCookbookPaths) + i cookbook_paths[i] = fmt.Sprintf(`"%s"`, path) } - + roles_path := "" if rolesPath != "" { roles_path = fmt.Sprintf(`"%s"`, rolesPath) @@ -297,8 +297,8 @@ func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, local configString, err := p.config.tpl.Process(tpl, &ConfigTemplate{ CookbookPaths: strings.Join(cookbook_paths, ","), - RolesPath: roles_path, - DataBagsPath: data_bags_path, + RolesPath: roles_path, + DataBagsPath: data_bags_path, }) if err != nil { return "", err diff --git a/provisioner/chef-solo/provisioner_test.go b/provisioner/chef-solo/provisioner_test.go index 10a342dad..9a84a2a96 100644 --- a/provisioner/chef-solo/provisioner_test.go +++ b/provisioner/chef-solo/provisioner_test.go @@ -107,7 +107,7 @@ func TestProvisionerPrepare_cookbookPaths(t *testing.T) { if p.config.CookbookPaths[0] != path1 || p.config.CookbookPaths[1] != path2 { t.Fatalf("unexpected: %#v", p.config.CookbookPaths) } - + if p.config.RolesPath != rolesPath { t.Fatalf("unexpected: %#v", p.config.RolesPath) } From c5b46fb4172be616aaa41efdea96823a2f9bdda7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 14:14:40 -0700 Subject: [PATCH 10/46] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee5aad398..7ad2b1fae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ FEATURES: * provisioner/chef-solo: Ability to specify a custom Chef configuration template. +* provisioner/chef-solo: Roles and data bags support. [GH-348] IMPROVEMENTS: From 7034f69f2c08f776fa0e556ebf6736c6675b2cef Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 14:17:07 -0700 Subject: [PATCH 11/46] provisioner/chef-solo: move quoting to template /cc @jerryclinesmith - I want the quoting to go into the template. I realize the cookbooks path doesn't do this and you were following that. That one is just weird because it is an array and Go templates kind of suck. --- provisioner/chef-solo/provisioner.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/provisioner/chef-solo/provisioner.go b/provisioner/chef-solo/provisioner.go index 1262ab6d2..dd69d1410 100644 --- a/provisioner/chef-solo/provisioner.go +++ b/provisioner/chef-solo/provisioner.go @@ -40,8 +40,8 @@ type Provisioner struct { type ConfigTemplate struct { CookbookPaths string - RolesPath string DataBagsPath string + RolesPath string } type ExecuteTemplate struct { @@ -268,16 +268,6 @@ func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, local cookbook_paths[i] = fmt.Sprintf(`"%s"`, path) } - roles_path := "" - if rolesPath != "" { - roles_path = fmt.Sprintf(`"%s"`, rolesPath) - } - - data_bags_path := "" - if dataBagsPath != "" { - data_bags_path = fmt.Sprintf(`"%s"`, dataBagsPath) - } - // Read the template tpl := DefaultConfigTemplate if p.config.ConfigTemplate != "" { @@ -297,8 +287,8 @@ func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, local configString, err := p.config.tpl.Process(tpl, &ConfigTemplate{ CookbookPaths: strings.Join(cookbook_paths, ","), - RolesPath: roles_path, - DataBagsPath: data_bags_path, + RolesPath: rolesPath, + DataBagsPath: dataBagsPath, }) if err != nil { return "", err @@ -451,9 +441,9 @@ func (p *Provisioner) processJsonUserVars() (map[string]interface{}, error) { var DefaultConfigTemplate = ` cookbook_path [{{.CookbookPaths}}] {{if .RolesPath != ""}} -role_path {{.RolesPath}} +role_path "{{.RolesPath}}" {{end}} {{if .DataBagsPath != ""}} -data_bag_path {{.DataBagsPath}} +data_bag_path "{{.DataBagsPath}}" {{end}} ` From ba9d85c744ff3b439631c4c91aded64b05f80271 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 14:17:54 -0700 Subject: [PATCH 12/46] provisioner/chef-solo: template process roles/data bags path /cc @jerryclinesmith --- provisioner/chef-solo/provisioner.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/provisioner/chef-solo/provisioner.go b/provisioner/chef-solo/provisioner.go index dd69d1410..0f4fe10ab 100644 --- a/provisioner/chef-solo/provisioner.go +++ b/provisioner/chef-solo/provisioner.go @@ -87,6 +87,8 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { templates := map[string]*string{ "config_template": &p.config.ConfigTemplate, + "data_bags_path": &p.config.DataBagsPath, + "roles_path": &p.config.RolesPath, "staging_dir": &p.config.StagingDir, } From 1b1b535ee52c9b78887cd7c9926ab55d914f9dcc Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 14:19:50 -0700 Subject: [PATCH 13/46] provisioner/chef-solo: one test per test /cc @jerryclinesmith --- provisioner/chef-solo/provisioner_test.go | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/provisioner/chef-solo/provisioner_test.go b/provisioner/chef-solo/provisioner_test.go index 9a84a2a96..c18df246a 100644 --- a/provisioner/chef-solo/provisioner_test.go +++ b/provisioner/chef-solo/provisioner_test.go @@ -117,6 +117,50 @@ func TestProvisionerPrepare_cookbookPaths(t *testing.T) { } } +func TestProvisionerPrepare_dataBagsPath(t *testing.T) { + var p Provisioner + + dataBagsPath, err := ioutil.TempDir("", "data_bags") + if err != nil { + t.Fatalf("err: %s", err) + } + defer os.Remove(dataBagsPath) + + config := testConfig() + config["data_bags_path"] = dataBagsPath + + err = p.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + if p.config.DataBagsPath != dataBagsPath { + t.Fatalf("unexpected: %#v", p.config.DataBagsPath) + } +} + +func TestProvisionerPrepare_rolesPath(t *testing.T) { + var p Provisioner + + rolesPath, err := ioutil.TempDir("", "roles") + if err != nil { + t.Fatalf("err: %s", err) + } + defer os.Remove(rolesPath) + + config := testConfig() + config["roles_path"] = rolesPath + + err = p.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + if p.config.RolesPath != rolesPath { + t.Fatalf("unexpected: %#v", p.config.RolesPath) + } +} + func TestProvisionerPrepare_json(t *testing.T) { config := testConfig() config["json"] = map[string]interface{}{ From 2e9dbb72b01005b6c94c0cb3a3e48f04335d3e9a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 14:43:42 -0700 Subject: [PATCH 14/46] post-processor/vagrant: fix slice copy to actually work for override [GH-426] --- CHANGELOG.md | 1 + post-processor/vagrant/post-processor.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ad2b1fae..ddf709bba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ BUG FIXES: * builder/amazon/instance: Make AccessKey/SecretKey available to bundle command even when they come from the environment. [GH-434] * builder/virtualbox: F1-F12 and delete scancodes now work. [GH-425] +* post-processor/vagrant: Override configurations properly work. [GH-426] * provisioner/puppet-masterless: Fix failure case when both facter vars are used and prevent_sudo. [GH-415] diff --git a/post-processor/vagrant/post-processor.go b/post-processor/vagrant/post-processor.go index 13e39a763..ecfae80de 100644 --- a/post-processor/vagrant/post-processor.go +++ b/post-processor/vagrant/post-processor.go @@ -77,7 +77,7 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } // Create the proper list of configurations - ppConfigs := make([]interface{}, 0, len(p.rawConfigs)+1) + ppConfigs := make([]interface{}, len(p.rawConfigs), len(p.rawConfigs)+1) copy(ppConfigs, p.rawConfigs) ppConfigs = append(ppConfigs, raw) From 6fc89e95841c6318afc712ce8461a10671c98b53 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 14:51:51 -0700 Subject: [PATCH 15/46] post-processor/vagrant: set output path always in a new raw --- post-processor/vagrant/post-processor.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/post-processor/vagrant/post-processor.go b/post-processor/vagrant/post-processor.go index ecfae80de..e6f629068 100644 --- a/post-processor/vagrant/post-processor.go +++ b/post-processor/vagrant/post-processor.go @@ -45,10 +45,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { tpl.UserVars = p.config.PackerUserVars // Defaults - ppExtraConfig := make(map[string]interface{}) if p.config.OutputPath == "" { p.config.OutputPath = "packer_{{ .BuildName }}_{{.Provider}}.box" - ppExtraConfig["output"] = p.config.OutputPath } // Accumulate any errors @@ -58,10 +56,16 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { errs, fmt.Errorf("Error parsing output template: %s", err)) } + // Store extra configuration we'll send to each post-processor type + ppExtraConfig := make(map[string]interface{}) + ppExtraConfig["output"] = p.config.OutputPath + // Store the extra configuration for post-processors p.rawConfigs = append(p.rawConfigs, ppExtraConfig) - // TODO(mitchellh): Properly handle multiple raw configs + // TODO(mitchellh): Properly handle multiple raw configs. This isn't + // very pressing at the moment because at the time of this comment + // only the first member of raws can contain the actual type-overrides. var mapConfig map[string]interface{} if err := mapstructure.Decode(raws[0], &mapConfig); err != nil { errs = packer.MultiErrorAppend(errs, From aa694072d75b9b46d0b2f906a249c1e9b16e6f7c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 15:01:06 -0700 Subject: [PATCH 16/46] post-processor/vagrant: simplify logic, only send overrides to PP [GH-413] /cc @jasonberanek --- post-processor/vagrant/post-processor.go | 57 +++++++++++++----------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/post-processor/vagrant/post-processor.go b/post-processor/vagrant/post-processor.go index e6f629068..952f574ad 100644 --- a/post-processor/vagrant/post-processor.go +++ b/post-processor/vagrant/post-processor.go @@ -24,15 +24,12 @@ type Config struct { } type PostProcessor struct { - config Config - premade map[string]packer.PostProcessor - rawConfigs []interface{} + config Config + premade map[string]packer.PostProcessor + extraConfig map[string]interface{} } func (p *PostProcessor) Configure(raws ...interface{}) error { - // Store the raw configs for usage later - p.rawConfigs = raws - _, err := common.DecodeConfig(&p.config, raws...) if err != nil { return err @@ -57,11 +54,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } // Store extra configuration we'll send to each post-processor type - ppExtraConfig := make(map[string]interface{}) - ppExtraConfig["output"] = p.config.OutputPath - - // Store the extra configuration for post-processors - p.rawConfigs = append(p.rawConfigs, ppExtraConfig) + p.extraConfig = make(map[string]interface{}) + p.extraConfig["output"] = p.config.OutputPath // TODO(mitchellh): Properly handle multiple raw configs. This isn't // very pressing at the moment because at the time of this comment @@ -75,18 +69,14 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { p.premade = make(map[string]packer.PostProcessor) for k, raw := range mapConfig { - pp := keyToPostProcessor(k) - if pp == nil { + pp, err := p.subPostProcessor(k, raw, p.extraConfig) + if err != nil { + errs = packer.MultiErrorAppend(errs, err) continue } - // Create the proper list of configurations - ppConfigs := make([]interface{}, len(p.rawConfigs), len(p.rawConfigs)+1) - copy(ppConfigs, p.rawConfigs) - ppConfigs = append(ppConfigs, raw) - - if err := pp.Configure(ppConfigs...); err != nil { - errs = packer.MultiErrorAppend(errs, err) + if pp == nil { + continue } p.premade[k] = pp @@ -110,13 +100,15 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac pp, ok := p.premade[ppName] if !ok { log.Printf("Premade post-processor for '%s' not found. Creating.", ppName) - pp = keyToPostProcessor(ppName) - if pp == nil { - return nil, false, fmt.Errorf("Vagrant box post-processor not found: %s", ppName) + + var err error + pp, err = p.subPostProcessor(ppName, nil, p.extraConfig) + if err != nil { + return nil, false, err } - if err := pp.Configure(p.rawConfigs...); err != nil { - return nil, false, err + if pp == nil { + return nil, false, fmt.Errorf("Vagrant box post-processor not found: %s", ppName) } } @@ -124,6 +116,21 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac return pp.PostProcess(ui, artifact) } +func (p *PostProcessor) subPostProcessor(key string, specific interface{}, extra map[string]interface{}) (packer.PostProcessor, error) { + pp := keyToPostProcessor(key) + if pp == nil { + return nil, nil + } + + if err := pp.Configure(extra, specific); err != nil { + return nil, err + } + + return pp, nil +} + +// keyToPostProcessor maps a configuration key to the actual post-processor +// it will be configuring. This returns a new instance of that post-processor. func keyToPostProcessor(key string) packer.PostProcessor { switch key { case "aws": From 877dfb81fe7124a28401252164365dfcab969390 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 16:18:39 -0700 Subject: [PATCH 17/46] common: Allow user variables to be used for ints/bools/etc. [GH-418] --- CHANGELOG.md | 2 ++ common/config.go | 5 +++-- website/source/docs/templates/user-variables.html.markdown | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddf709bba..1c167e1d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ FEATURES: IMPROVEMENTS: +* core: User variables can now be used for integer, boolean, etc. + values. [GH-418] * builder/amazon/all: Interrupts work while waiting for AMI to be ready. BUG FIXES: diff --git a/common/config.go b/common/config.go index 2a2c8b21f..b579562d5 100644 --- a/common/config.go +++ b/common/config.go @@ -41,8 +41,9 @@ func CheckUnusedConfig(md *mapstructure.Metadata) *packer.MultiError { func DecodeConfig(target interface{}, raws ...interface{}) (*mapstructure.Metadata, error) { var md mapstructure.Metadata decoderConfig := &mapstructure.DecoderConfig{ - Metadata: &md, - Result: target, + Metadata: &md, + Result: target, + WeaklyTypedInput: true, } decoder, err := mapstructure.NewDecoder(decoderConfig) diff --git a/website/source/docs/templates/user-variables.html.markdown b/website/source/docs/templates/user-variables.html.markdown index 8f19aadea..edce239ad 100644 --- a/website/source/docs/templates/user-variables.html.markdown +++ b/website/source/docs/templates/user-variables.html.markdown @@ -54,7 +54,7 @@ validation will fail. Using the variables is extremely easy. Variables are used by calling the user function in the form of {{user `variable`}}. -This function can be used in _any string_ within the template, in +This function can be used in _any value_ within the template, in builders, provisioners, _anything_. The user variable is available globally within the template. From f85c9e435479ae16972ecac02a647c07ab523834 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 17:15:48 -0700 Subject: [PATCH 18/46] packer/rpc: set keep-alive on all RPC connections [GH-416] --- CHANGELOG.md | 2 ++ packer/rpc/build.go | 4 ++-- packer/rpc/builder.go | 7 +++---- packer/rpc/command.go | 2 +- packer/rpc/communicator.go | 12 ++++++------ packer/rpc/dial.go | 33 +++++++++++++++++++++++++++++++++ packer/rpc/environment.go | 12 ++++++------ packer/rpc/hook.go | 2 +- packer/rpc/post_processor.go | 4 ++-- packer/rpc/provisioner.go | 2 +- 10 files changed, 57 insertions(+), 23 deletions(-) create mode 100644 packer/rpc/dial.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c167e1d4..8a85c2ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ IMPROVEMENTS: BUG FIXES: +* core: Set TCP KeepAlives on internally created RPC connections so that + they don't die. [GH-416] * builder/amazon/all: While waiting for AMI, will detect "failed" state. * builder/amazon/all: Waiting for state will detect if the resource (AMI, instance, etc.) disappears from under it. diff --git a/packer/rpc/build.go b/packer/rpc/build.go index 6d81ce870..8f21b4ce9 100644 --- a/packer/rpc/build.go +++ b/packer/rpc/build.go @@ -52,7 +52,7 @@ func (b *build) Run(ui packer.Ui, cache packer.Cache) ([]packer.Artifact, error) artifacts := make([]packer.Artifact, len(result)) for i, addr := range result { - client, err := rpc.Dial("tcp", addr) + client, err := rpcDial(addr) if err != nil { return nil, err } @@ -92,7 +92,7 @@ func (b *BuildServer) Prepare(v map[string]string, reply *error) error { } func (b *BuildServer) Run(args *BuildRunArgs, reply *[]string) error { - client, err := rpc.Dial("tcp", args.UiRPCAddress) + client, err := rpcDial(args.UiRPCAddress) if err != nil { return err } diff --git a/packer/rpc/builder.go b/packer/rpc/builder.go index 8c56e4959..5e2c20d77 100644 --- a/packer/rpc/builder.go +++ b/packer/rpc/builder.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/mitchellh/packer/packer" "log" - "net" "net/rpc" ) @@ -95,7 +94,7 @@ func (b *builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe return nil, nil } - client, err := rpc.Dial("tcp", response.RPCAddress) + client, err := rpcDial(response.RPCAddress) if err != nil { return nil, err } @@ -119,12 +118,12 @@ func (b *BuilderServer) Prepare(args *BuilderPrepareArgs, reply *error) error { } func (b *BuilderServer) Run(args *BuilderRunArgs, reply *interface{}) error { - client, err := rpc.Dial("tcp", args.RPCAddress) + client, err := rpcDial(args.RPCAddress) if err != nil { return err } - responseC, err := net.Dial("tcp", args.ResponseAddress) + responseC, err := tcpDial(args.ResponseAddress) if err != nil { return err } diff --git a/packer/rpc/command.go b/packer/rpc/command.go index 18cd5667e..3e2b48b2f 100644 --- a/packer/rpc/command.go +++ b/packer/rpc/command.go @@ -66,7 +66,7 @@ func (c *CommandServer) Help(args *interface{}, reply *string) error { } func (c *CommandServer) Run(args *CommandRunArgs, reply *int) error { - client, err := rpc.Dial("tcp", args.RPCAddress) + client, err := rpcDial(args.RPCAddress) if err != nil { return err } diff --git a/packer/rpc/communicator.go b/packer/rpc/communicator.go index 21b507f0b..77e321153 100644 --- a/packer/rpc/communicator.go +++ b/packer/rpc/communicator.go @@ -177,7 +177,7 @@ func (c *CommunicatorServer) Start(args *CommunicatorStartArgs, reply *interface toClose := make([]net.Conn, 0) if args.StdinAddress != "" { - stdinC, err := net.Dial("tcp", args.StdinAddress) + stdinC, err := tcpDial(args.StdinAddress) if err != nil { return err } @@ -187,7 +187,7 @@ func (c *CommunicatorServer) Start(args *CommunicatorStartArgs, reply *interface } if args.StdoutAddress != "" { - stdoutC, err := net.Dial("tcp", args.StdoutAddress) + stdoutC, err := tcpDial(args.StdoutAddress) if err != nil { return err } @@ -197,7 +197,7 @@ func (c *CommunicatorServer) Start(args *CommunicatorStartArgs, reply *interface } if args.StderrAddress != "" { - stderrC, err := net.Dial("tcp", args.StderrAddress) + stderrC, err := tcpDial(args.StderrAddress) if err != nil { return err } @@ -208,7 +208,7 @@ func (c *CommunicatorServer) Start(args *CommunicatorStartArgs, reply *interface // Connect to the response address so we can write our result to it // when ready. - responseC, err := net.Dial("tcp", args.ResponseAddress) + responseC, err := tcpDial(args.ResponseAddress) if err != nil { return err } @@ -234,7 +234,7 @@ func (c *CommunicatorServer) Start(args *CommunicatorStartArgs, reply *interface } func (c *CommunicatorServer) Upload(args *CommunicatorUploadArgs, reply *interface{}) (err error) { - readerC, err := net.Dial("tcp", args.ReaderAddress) + readerC, err := tcpDial(args.ReaderAddress) if err != nil { return } @@ -250,7 +250,7 @@ func (c *CommunicatorServer) UploadDir(args *CommunicatorUploadDirArgs, reply *e } func (c *CommunicatorServer) Download(args *CommunicatorDownloadArgs, reply *interface{}) (err error) { - writerC, err := net.Dial("tcp", args.WriterAddress) + writerC, err := tcpDial(args.WriterAddress) if err != nil { return } diff --git a/packer/rpc/dial.go b/packer/rpc/dial.go new file mode 100644 index 000000000..10e2cad14 --- /dev/null +++ b/packer/rpc/dial.go @@ -0,0 +1,33 @@ +package rpc + +import ( + "net" + "net/rpc" +) + +// rpcDial makes a TCP connection to a remote RPC server and returns +// the client. This will set the connection up properly so that keep-alives +// are set and so on and should be used to make all RPC connections within +// this package. +func rpcDial(address string) (*rpc.Client, error) { + tcpConn, err := tcpDial(address) + if err != nil { + return nil, err + } + + // Create an RPC client around our connection + return rpc.NewClient(tcpConn), nil +} + +// tcpDial connects via TCP to the designated address. +func tcpDial(address string) (*net.TCPConn, error) { + conn, err := net.Dial("tcp", address) + if err != nil { + return nil, err + } + + // Set a keep-alive so that the connection stays alive even when idle + tcpConn := conn.(*net.TCPConn) + tcpConn.SetKeepAlive(true) + return tcpConn, nil +} diff --git a/packer/rpc/environment.go b/packer/rpc/environment.go index 8ebf709c0..36db72c56 100644 --- a/packer/rpc/environment.go +++ b/packer/rpc/environment.go @@ -28,7 +28,7 @@ func (e *Environment) Builder(name string) (b packer.Builder, err error) { return } - client, err := rpc.Dial("tcp", reply) + client, err := rpcDial(reply) if err != nil { return } @@ -43,7 +43,7 @@ func (e *Environment) Cache() packer.Cache { panic(err) } - client, err := rpc.Dial("tcp", reply) + client, err := rpcDial(reply) if err != nil { panic(err) } @@ -64,7 +64,7 @@ func (e *Environment) Hook(name string) (h packer.Hook, err error) { return } - client, err := rpc.Dial("tcp", reply) + client, err := rpcDial(reply) if err != nil { return } @@ -80,7 +80,7 @@ func (e *Environment) PostProcessor(name string) (p packer.PostProcessor, err er return } - client, err := rpc.Dial("tcp", reply) + client, err := rpcDial(reply) if err != nil { return } @@ -96,7 +96,7 @@ func (e *Environment) Provisioner(name string) (p packer.Provisioner, err error) return } - client, err := rpc.Dial("tcp", reply) + client, err := rpcDial(reply) if err != nil { return } @@ -109,7 +109,7 @@ func (e *Environment) Ui() packer.Ui { var reply string e.client.Call("Environment.Ui", new(interface{}), &reply) - client, err := rpc.Dial("tcp", reply) + client, err := rpcDial(reply) if err != nil { panic(err) } diff --git a/packer/rpc/hook.go b/packer/rpc/hook.go index 687d991a7..223b96df2 100644 --- a/packer/rpc/hook.go +++ b/packer/rpc/hook.go @@ -46,7 +46,7 @@ func (h *hook) Cancel() { } func (h *HookServer) Run(args *HookRunArgs, reply *interface{}) error { - client, err := rpc.Dial("tcp", args.RPCAddress) + client, err := rpcDial(args.RPCAddress) if err != nil { return err } diff --git a/packer/rpc/post_processor.go b/packer/rpc/post_processor.go index fb43cb7d9..0a5eaefd3 100644 --- a/packer/rpc/post_processor.go +++ b/packer/rpc/post_processor.go @@ -57,7 +57,7 @@ func (p *postProcessor) PostProcess(ui packer.Ui, a packer.Artifact) (packer.Art return nil, false, nil } - client, err := rpc.Dial("tcp", response.RPCAddress) + client, err := rpcDial(response.RPCAddress) if err != nil { return nil, false, err } @@ -75,7 +75,7 @@ func (p *PostProcessorServer) Configure(args *PostProcessorConfigureArgs, reply } func (p *PostProcessorServer) PostProcess(address string, reply *PostProcessorProcessResponse) error { - client, err := rpc.Dial("tcp", address) + client, err := rpcDial(address) if err != nil { return err } diff --git a/packer/rpc/provisioner.go b/packer/rpc/provisioner.go index 7d3ed1617..4cd329d6b 100644 --- a/packer/rpc/provisioner.go +++ b/packer/rpc/provisioner.go @@ -65,7 +65,7 @@ func (p *ProvisionerServer) Prepare(args *ProvisionerPrepareArgs, reply *error) } func (p *ProvisionerServer) Provision(args *ProvisionerProvisionArgs, reply *interface{}) error { - client, err := rpc.Dial("tcp", args.RPCAddress) + client, err := rpcDial(args.RPCAddress) if err != nil { return err } From 865f19750324f0909ad94a21ec4c2fe3b05b3e2a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 18 Sep 2013 18:15:46 -0700 Subject: [PATCH 19/46] builder/virtualbox: support attaching guest additions [GH-405] --- CHANGELOG.md | 2 + builder/virtualbox/builder.go | 2 + .../virtualbox/step_attach_guest_additions.go | 81 +++++++++++++++++++ .../virtualbox/step_upload_guest_additions.go | 7 ++ .../docs/builders/virtualbox.html.markdown | 4 + 5 files changed, 96 insertions(+) create mode 100644 builder/virtualbox/step_attach_guest_additions.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a85c2ce1..c6a02f949 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ FEATURES: +* builders/virtualbox: Guest additions can be attached rather than uploaded, + easier to handle for Windows guests. [GH-405] * provisioner/chef-solo: Ability to specify a custom Chef configuration template. * provisioner/chef-solo: Roles and data bags support. [GH-348] diff --git a/builder/virtualbox/builder.go b/builder/virtualbox/builder.go index 18b16867a..452a7de49 100644 --- a/builder/virtualbox/builder.go +++ b/builder/virtualbox/builder.go @@ -28,6 +28,7 @@ type config struct { DiskSize uint `mapstructure:"disk_size"` FloppyFiles []string `mapstructure:"floppy_files"` Format string `mapstructure:"format"` + GuestAdditionsAttach bool `mapstructure:"guest_additions_attach"` GuestAdditionsPath string `mapstructure:"guest_additions_path"` GuestAdditionsURL string `mapstructure:"guest_additions_url"` GuestAdditionsSHA256 string `mapstructure:"guest_additions_sha256"` @@ -361,6 +362,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe new(stepCreateVM), new(stepCreateDisk), new(stepAttachISO), + new(stepAttachGuestAdditions), new(stepAttachFloppy), new(stepForwardSSH), new(stepVBoxManage), diff --git a/builder/virtualbox/step_attach_guest_additions.go b/builder/virtualbox/step_attach_guest_additions.go new file mode 100644 index 000000000..9bbeb949a --- /dev/null +++ b/builder/virtualbox/step_attach_guest_additions.go @@ -0,0 +1,81 @@ +package virtualbox + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" +) + +// This step attaches the VirtualBox guest additions as a inserted CD onto +// the virtual machine. +// +// Uses: +// config *config +// driver Driver +// guest_additions_path string +// ui packer.Ui +// vmName string +// +// Produces: +type stepAttachGuestAdditions struct { + attachedPath string +} + +func (s *stepAttachGuestAdditions) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(*config) + driver := state.Get("driver").(Driver) + guestAdditionsPath := state.Get("guest_additions_path").(string) + ui := state.Get("ui").(packer.Ui) + vmName := state.Get("vmName").(string) + + // If we're not attaching the guest additions then just return + if !config.GuestAdditionsAttach { + log.Println("Not attaching guest additions since we're uploading.") + return multistep.ActionContinue + } + + // Attach the guest additions to the computer + log.Println("Attaching guest additions ISO onto IDE controller...") + command := []string{ + "storageattach", vmName, + "--storagectl", "IDE Controller", + "--port", "1", + "--device", "0", + "--type", "dvddrive", + "--medium", guestAdditionsPath, + } + if err := driver.VBoxManage(command...); err != nil { + err := fmt.Errorf("Error attaching guest additions: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Track the path so that we can unregister it from VirtualBox later + s.attachedPath = guestAdditionsPath + + return multistep.ActionContinue +} + +func (s *stepAttachGuestAdditions) Cleanup(state multistep.StateBag) { + if s.attachedPath == "" { + return + } + + driver := state.Get("driver").(Driver) + ui := state.Get("ui").(packer.Ui) + vmName := state.Get("vmName").(string) + + command := []string{ + "storageattach", vmName, + "--storagectl", "IDE Controller", + "--port", "1", + "--device", "0", + "--medium", "none", + } + + if err := driver.VBoxManage(command...); err != nil { + ui.Error(fmt.Sprintf("Error unregistering guest additions: %s", err)) + } +} diff --git a/builder/virtualbox/step_upload_guest_additions.go b/builder/virtualbox/step_upload_guest_additions.go index 86b83b3c6..ff2252c86 100644 --- a/builder/virtualbox/step_upload_guest_additions.go +++ b/builder/virtualbox/step_upload_guest_additions.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" + "log" "os" ) @@ -21,6 +22,12 @@ func (s *stepUploadGuestAdditions) Run(state multistep.StateBag) multistep.StepA guestAdditionsPath := state.Get("guest_additions_path").(string) ui := state.Get("ui").(packer.Ui) + // If we're attaching then don't do this, since we attached. + if config.GuestAdditionsAttach { + log.Println("Not uploading guest additions since we're attaching.") + return multistep.ActionContinue + } + version, err := driver.Version() if err != nil { state.Put("error", fmt.Errorf("Error reading version for guest additions upload: %s", err)) diff --git a/website/source/docs/builders/virtualbox.html.markdown b/website/source/docs/builders/virtualbox.html.markdown index 75e6b1943..b10cd8cb0 100644 --- a/website/source/docs/builders/virtualbox.html.markdown +++ b/website/source/docs/builders/virtualbox.html.markdown @@ -85,6 +85,10 @@ Optional: * `format` (string) - Either "ovf" or "ova", this specifies the output format of the exported virtual machine. This defaults to "ovf". +* `guest_additions_attach` (bool) - If this is true (defaults to "false"), + the guest additions ISO will be attached to the virtual machine as a CD + rather than uploaded as a raw ISO. + * `guest_additions_path` (string) - The path on the guest virtual machine where the VirtualBox guest additions ISO will be uploaded. By default this is "VBoxGuestAdditions.iso" which should upload into the login directory From 229d790ebae55c2eed2152ca60d50399c56970d8 Mon Sep 17 00:00:00 2001 From: "Jason A. Beranek" Date: Wed, 18 Sep 2013 22:56:00 -0500 Subject: [PATCH 20/46] post-processor/vagrant: pass PackerConfig to sub post-processors --- post-processor/vagrant/post-processor.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/post-processor/vagrant/post-processor.go b/post-processor/vagrant/post-processor.go index 952f574ad..91969e558 100644 --- a/post-processor/vagrant/post-processor.go +++ b/post-processor/vagrant/post-processor.go @@ -56,6 +56,11 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { // Store extra configuration we'll send to each post-processor type p.extraConfig = make(map[string]interface{}) p.extraConfig["output"] = p.config.OutputPath + p.extraConfig["packer_build_name"] = p.config.PackerBuildName + p.extraConfig["packer_builder_type"] = p.config.PackerBuilderType + p.extraConfig["packer_debug"] = p.config.PackerDebug + p.extraConfig["packer_force"] = p.config.PackerForce + p.extraConfig["packer_user_variables"] = p.config.PackerUserVars // TODO(mitchellh): Properly handle multiple raw configs. This isn't // very pressing at the moment because at the time of this comment From 677f2989b9d12af3654d8e9afbef8ea7ca617331 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 19 Sep 2013 11:54:24 -0700 Subject: [PATCH 21/46] provisioner/chef-solo: template doesn't support comp [GH-442] --- provisioner/chef-solo/provisioner.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/provisioner/chef-solo/provisioner.go b/provisioner/chef-solo/provisioner.go index 0f4fe10ab..ddfc605fb 100644 --- a/provisioner/chef-solo/provisioner.go +++ b/provisioner/chef-solo/provisioner.go @@ -42,6 +42,12 @@ type ConfigTemplate struct { CookbookPaths string DataBagsPath string RolesPath string + + // Templates don't support boolean statements until Go 1.2. In the + // mean time, we do this. + // TODO(mitchellh): Remove when Go 1.2 is released + HasDataBagsPath bool + HasRolesPath bool } type ExecuteTemplate struct { @@ -288,9 +294,11 @@ func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, local } configString, err := p.config.tpl.Process(tpl, &ConfigTemplate{ - CookbookPaths: strings.Join(cookbook_paths, ","), - RolesPath: rolesPath, - DataBagsPath: dataBagsPath, + CookbookPaths: strings.Join(cookbook_paths, ","), + RolesPath: rolesPath, + DataBagsPath: dataBagsPath, + HasRolesPath: rolesPath != "", + HasDataBagsPath: dataBagsPath != "", }) if err != nil { return "", err @@ -442,10 +450,10 @@ func (p *Provisioner) processJsonUserVars() (map[string]interface{}, error) { var DefaultConfigTemplate = ` cookbook_path [{{.CookbookPaths}}] -{{if .RolesPath != ""}} +{{if .HasRolesPath}} role_path "{{.RolesPath}}" {{end}} -{{if .DataBagsPath != ""}} +{{if .HasDataBagsPath}} data_bag_path "{{.DataBagsPath}}" {{end}} ` From 71358222f05918644c1606732a371709654b5a44 Mon Sep 17 00:00:00 2001 From: "Jason A. Beranek" Date: Thu, 19 Sep 2013 23:30:22 -0500 Subject: [PATCH 22/46] communicator/ssh, builder/digitalocean: fix new SSH API from upstream --- builder/digitalocean/step_create_ssh_key.go | 3 ++- communicator/ssh/keychain.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/builder/digitalocean/step_create_ssh_key.go b/builder/digitalocean/step_create_ssh_key.go index 6a9f0426c..04699ec66 100644 --- a/builder/digitalocean/step_create_ssh_key.go +++ b/builder/digitalocean/step_create_ssh_key.go @@ -38,7 +38,8 @@ func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { state.Put("privateKey", string(pem.EncodeToMemory(&priv_blk))) // Marshal the public key into SSH compatible format - pub := ssh.NewRSAPublicKey(&priv.PublicKey) + // TODO properly handle the public key error + pub, _ := ssh.NewPublicKey(&priv.PublicKey) pub_sshformat := string(ssh.MarshalAuthorizedKey(pub)) // The name of the public key on DO diff --git a/communicator/ssh/keychain.go b/communicator/ssh/keychain.go index d7934f8a8..f9965c0da 100644 --- a/communicator/ssh/keychain.go +++ b/communicator/ssh/keychain.go @@ -60,9 +60,9 @@ func (k *SimpleKeychain) Key(i int) (ssh.PublicKey, error) { } switch key := k.keys[i].(type) { case *rsa.PrivateKey: - return ssh.NewRSAPublicKey(&key.PublicKey), nil + return ssh.NewPublicKey(&key.PublicKey) case *dsa.PrivateKey: - return ssh.NewDSAPublicKey(&key.PublicKey), nil + return ssh.NewPublicKey(&key.PublicKey) } panic("unknown key type") } From 9c519eda4197c0e9fd2dec038bac790f499d6412 Mon Sep 17 00:00:00 2001 From: Tim Mower Date: Fri, 20 Sep 2013 11:36:37 +0100 Subject: [PATCH 23/46] Fix syntax error in puppet template --- .../source/docs/provisioners/puppet-masterless.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/provisioners/puppet-masterless.html.markdown b/website/source/docs/provisioners/puppet-masterless.html.markdown index 11dbc6a85..792718570 100644 --- a/website/source/docs/provisioners/puppet-masterless.html.markdown +++ b/website/source/docs/provisioners/puppet-masterless.html.markdown @@ -79,7 +79,7 @@ By default, Packer uses the following command (broken across multiple lines for readability) to execute Puppet: ``` -{{.FacterVars}}{{if .Sudo} sudo -E {{end}}puppet apply \ +{{.FacterVars}}{{if .Sudo}} sudo -E {{end}}puppet apply \ --verbose \ --modulepath='{{.ModulePath}}' \ {{if .HasHieraConfigPath}}--hiera_config='{{.HieraConfigPath}}' {{end}} \ From abbac36796eebbad94f36358003bae82b86e4357 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 20 Sep 2013 10:21:59 -0700 Subject: [PATCH 24/46] provisioner/shell: convert windows line endings to Unix [GH-277] --- CHANGELOG.md | 3 + provisioner/shell/provisioner.go | 10 +++ provisioner/shell/unix_reader.go | 88 +++++++++++++++++++ provisioner/shell/unix_reader_test.go | 33 +++++++ .../docs/provisioners/shell.html.markdown | 4 + 5 files changed, 138 insertions(+) create mode 100644 provisioner/shell/unix_reader.go create mode 100644 provisioner/shell/unix_reader_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c6a02f949..0c5cb3d60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ IMPROVEMENTS: * core: User variables can now be used for integer, boolean, etc. values. [GH-418] * builder/amazon/all: Interrupts work while waiting for AMI to be ready. +* provisioner/shell: Script line-endings are automatically converted to + Unix-style line-endings. Can be disabled by setting "binary" to "true". + [GH-277] BUG FIXES: diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index e8a5f61a5..3cb75187d 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/packer" + "io" "io/ioutil" "log" "os" @@ -20,6 +21,10 @@ const DefaultRemotePath = "/tmp/script.sh" type config struct { common.PackerConfig `mapstructure:",squash"` + // If true, the script contains binary and line endings will not be + // converted from Windows to Unix-style. + Binary bool + // An inline script to execute. Multiple strings are all executed // in the context of a single shell. Inline []string @@ -259,6 +264,11 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { return err } + var r io.Reader = f + if !p.config.Binary { + r = &UnixReader{Reader: r} + } + if err := comm.Upload(p.config.RemotePath, f); err != nil { return fmt.Errorf("Error uploading script: %s", err) } diff --git a/provisioner/shell/unix_reader.go b/provisioner/shell/unix_reader.go new file mode 100644 index 000000000..5745dd291 --- /dev/null +++ b/provisioner/shell/unix_reader.go @@ -0,0 +1,88 @@ +package shell + +import ( + "bufio" + "bytes" + "io" + "sync" +) + +// UnixReader is a Reader implementation that automatically converts +// Windows line endings to Unix line endings. +type UnixReader struct { + Reader io.Reader + + buf []byte + once sync.Once + scanner *bufio.Scanner +} + +func (r *UnixReader) Read(p []byte) (n int, err error) { + // Create the buffered reader once + r.once.Do(func() { + r.scanner = bufio.NewScanner(r.Reader) + r.scanner.Split(scanUnixLine) + }) + + // If we have no data in our buffer, scan to the next token + if len(r.buf) == 0 { + if !r.scanner.Scan() { + err = r.scanner.Err() + if err == nil { + err = io.EOF + } + + return 0, err + } + + r.buf = r.scanner.Bytes() + } + + // Write out as much data as we can to the buffer, storing the rest + // for the next read. + n = len(p) + if n > len(r.buf) { + n = len(r.buf) + } + copy(p, r.buf) + r.buf = r.buf[n:] + + return +} + +// scanUnixLine is a bufio.Scanner SplitFunc. It tokenizes on lines, but +// only returns unix-style lines. So even if the line is "one\r\n", the +// token returned will be "one\n". +func scanUnixLine(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + + if i := bytes.IndexByte(data, '\n'); i >= 0 { + // We have a new-line terminated line. Return the line with the newline + return i + 1, dropCR(data[0 : i+1]), nil + } + + if atEOF { + // We have a final, non-terminated line + return len(data), dropCR(data), nil + } + + if data[len(data)-1] != '\r' { + // We have a normal line, just let it tokenize + return len(data), data, nil + } + + // We need more data + return 0, nil, nil +} + +func dropCR(data []byte) []byte { + if len(data) > 0 && data[len(data)-2] == '\r' { + // Trim off the last byte and replace it with a '\n' + data = data[0 : len(data)-1] + data[len(data)-1] = '\n' + } + + return data +} diff --git a/provisioner/shell/unix_reader_test.go b/provisioner/shell/unix_reader_test.go new file mode 100644 index 000000000..8dedf6300 --- /dev/null +++ b/provisioner/shell/unix_reader_test.go @@ -0,0 +1,33 @@ +package shell + +import ( + "bytes" + "io" + "testing" +) + +func TestUnixReader_impl(t *testing.T) { + var raw interface{} + raw = new(UnixReader) + if _, ok := raw.(io.Reader); !ok { + t.Fatal("should be reader") + } +} + +func TestUnixReader(t *testing.T) { + input := "one\r\ntwo\nthree\r\n" + expected := "one\ntwo\nthree\n" + + r := &UnixReader{ + Reader: bytes.NewReader([]byte(input)), + } + + result := new(bytes.Buffer) + if _, err := io.Copy(result, r); err != nil { + t.Fatalf("err: %s", err) + } + + if result.String() != expected { + t.Fatalf("bad: %#v", result.String()) + } +} diff --git a/website/source/docs/provisioners/shell.html.markdown b/website/source/docs/provisioners/shell.html.markdown index d3a895b3a..4c5a19992 100644 --- a/website/source/docs/provisioners/shell.html.markdown +++ b/website/source/docs/provisioners/shell.html.markdown @@ -47,6 +47,10 @@ Exactly _one_ of the following is required: Optional parameters: +* `binary` (boolean) - If true, specifies that the script(s) are binary + files, and Packer should therefore not convert Windows line endings to + Unix line endings (if there are any). By default this is false. + * `environment_vars` (array of strings) - An array of key/value pairs to inject prior to the execute_command. The format should be `key=value`. Packer injects some environmental variables by default From 118f4fdccea36f61738fb0f288d729d6064d3b35 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 20 Sep 2013 10:49:35 -0700 Subject: [PATCH 25/46] packer: `only` metaparameter for provisioners [GH-438] --- packer/template.go | 32 +++++++++++-- packer/template_test.go | 100 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 3 deletions(-) diff --git a/packer/template.go b/packer/template.go index 7c3dd0529..12d47e5a8 100644 --- a/packer/template.go +++ b/packer/template.go @@ -59,6 +59,7 @@ type RawPostProcessorConfig struct { type RawProvisionerConfig struct { Type string Override map[string]interface{} + Only []string RawConfig interface{} } @@ -237,9 +238,8 @@ func ParseTemplate(data []byte) (t *Template, err error) { continue } - // The provisioners not only don't need or want the override settings - // (as they are processed as part of the preparation below), but will - // actively reject them as invalid configuration. + // Delete the keys that we used + delete(v, "only") delete(v, "override") // Verify that the override keys exist... @@ -250,6 +250,17 @@ func ParseTemplate(data []byte) (t *Template, err error) { } } + // Verify that the only settings are good + if len(raw.Only) > 0 { + for _, n := range raw.Only { + if _, ok := t.Builders[n]; !ok { + errors = append(errors, + fmt.Errorf("provisioner %d: 'only' specified builder '%s' not found", + i+1, n)) + } + } + } + raw.RawConfig = v } @@ -425,6 +436,21 @@ func (t *Template) Build(name string, components *ComponentFinder) (b Build, err // Prepare the provisioners provisioners := make([]coreBuildProvisioner, 0, len(t.Provisioners)) for _, rawProvisioner := range t.Provisioners { + if len(rawProvisioner.Only) > 0 { + onlyFound := false + for _, n := range rawProvisioner.Only { + if n == name { + onlyFound = true + break + } + } + + if !onlyFound { + // Skip this provisioner + continue + } + } + var provisioner Provisioner provisioner, err = components.Provisioner(rawProvisioner.Type) if err != nil { diff --git a/packer/template_test.go b/packer/template_test.go index 883933367..c8fb6b4ae 100644 --- a/packer/template_test.go +++ b/packer/template_test.go @@ -9,6 +9,26 @@ import ( "testing" ) +func testTemplateComponentFinder() *ComponentFinder { + builder := testBuilder() + provisioner := &MockProvisioner{} + + builderMap := map[string]Builder{ + "test-builder": builder, + } + + provisionerMap := map[string]Provisioner{ + "test-prov": provisioner, + } + + builderFactory := func(n string) (Builder, error) { return builderMap[n], nil } + provFactory := func(n string) (Provisioner, error) { return provisionerMap[n], nil } + return &ComponentFinder{ + Builder: builderFactory, + Provisioner: provFactory, + } +} + func TestParseTemplateFile_basic(t *testing.T) { data := ` { @@ -663,6 +683,86 @@ func TestTemplate_Build(t *testing.T) { } } +func TestTemplateBuild_onlyProvInvalid(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "provisioners": [ + { + "type": "test-prov", + "only": "test5" + } + ] + } + ` + + _, err := ParseTemplate([]byte(data)) + if err == nil { + t.Fatal("should have error") + } +} + +func TestTemplateBuild_onlyProv(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "provisioners": [ + { + "type": "test-prov", + "only": ["test2"] + } + ] + } + ` + + template, err := ParseTemplate([]byte(data)) + if err != nil { + t.Fatalf("err: %s", err) + } + + // Verify test1 has no provisioners + build, err := template.Build("test1", testTemplateComponentFinder()) + if err != nil { + t.Fatalf("err: %s", err) + } + + cbuild := build.(*coreBuild) + if len(cbuild.provisioners) > 0 { + t.Fatal("should have no provisioners") + } + + // Verify test2 has no provisioners + build, err = template.Build("test2", testTemplateComponentFinder()) + if err != nil { + t.Fatalf("err: %s", err) + } + + cbuild = build.(*coreBuild) + if len(cbuild.provisioners) != 1 { + t.Fatalf("invalid: %d", len(cbuild.provisioners)) + } +} + func TestTemplate_Build_ProvisionerOverride(t *testing.T) { assert := asserts.NewTestingAsserts(t, true) From 45cd21a076bc887db10e52c519112e1fd457bfc2 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 20 Sep 2013 11:13:43 -0700 Subject: [PATCH 26/46] packer: `only` metaparameter for post-processors [GH-438] --- packer/template.go | 129 +++++++++++++++++++++++++++++----------- packer/template_test.go | 91 +++++++++++++++++++++++++++- 2 files changed, 184 insertions(+), 36 deletions(-) diff --git a/packer/template.go b/packer/template.go index 12d47e5a8..7e27bf93d 100644 --- a/packer/template.go +++ b/packer/template.go @@ -48,6 +48,8 @@ type RawBuilderConfig struct { // configuration. It contains the type of the post processor as well as the // raw configuration that is handed to the post-processor for it to process. type RawPostProcessorConfig struct { + TemplateOnlyExcept `mapstructure:",squash"` + Type string KeepInputArtifact bool `mapstructure:"keep_input_artifact"` RawConfig map[string]interface{} @@ -57,9 +59,10 @@ type RawPostProcessorConfig struct { // It contains the type of the provisioner as well as the raw configuration // that is handed to the provisioner for it to process. type RawProvisionerConfig struct { + TemplateOnlyExcept `mapstructure:",squash"` + Type string Override map[string]interface{} - Only []string RawConfig interface{} } @@ -190,32 +193,50 @@ func ParseTemplate(data []byte) (t *Template, err error) { continue } - t.PostProcessors[i] = make([]RawPostProcessorConfig, len(rawPP)) - configs := t.PostProcessors[i] + configs := make([]RawPostProcessorConfig, 0, len(rawPP)) for j, pp := range rawPP { - config := &configs[j] - if err := mapstructure.Decode(pp, config); err != nil { + var config RawPostProcessorConfig + if err := mapstructure.Decode(pp, &config); err != nil { if merr, ok := err.(*mapstructure.Error); ok { for _, err := range merr.Errors { - errors = append(errors, fmt.Errorf("Post-processor #%d.%d: %s", i+1, j+1, err)) + errors = append(errors, + fmt.Errorf("Post-processor #%d.%d: %s", i+1, j+1, err)) } } else { - errors = append(errors, fmt.Errorf("Post-processor %d.%d: %s", i+1, j+1, err)) + errors = append(errors, + fmt.Errorf("Post-processor %d.%d: %s", i+1, j+1, err)) } continue } if config.Type == "" { - errors = append(errors, fmt.Errorf("Post-processor %d.%d: missing 'type'", i+1, j+1)) + errors = append(errors, + fmt.Errorf("Post-processor %d.%d: missing 'type'", i+1, j+1)) continue } // Remove the input keep_input_artifact option + config.TemplateOnlyExcept.Prune(pp) delete(pp, "keep_input_artifact") + // Verify that the only settings are good + if errs := config.TemplateOnlyExcept.Validate(t.Builders); len(errs) > 0 { + for _, err := range errs { + errors = append(errors, + fmt.Errorf("Post-processor %d.%d: %s", i+1, j+1, err)) + } + + continue + } + config.RawConfig = pp + + // Add it to the list of configs + configs = append(configs, config) } + + t.PostProcessors[i] = configs } // Gather all the provisioners @@ -239,7 +260,7 @@ func ParseTemplate(data []byte) (t *Template, err error) { } // Delete the keys that we used - delete(v, "only") + raw.TemplateOnlyExcept.Prune(v) delete(v, "override") // Verify that the override keys exist... @@ -251,14 +272,8 @@ func ParseTemplate(data []byte) (t *Template, err error) { } // Verify that the only settings are good - if len(raw.Only) > 0 { - for _, n := range raw.Only { - if _, ok := t.Builders[n]; !ok { - errors = append(errors, - fmt.Errorf("provisioner %d: 'only' specified builder '%s' not found", - i+1, n)) - } - } + if errs := raw.TemplateOnlyExcept.Validate(t.Builders); len(errs) > 0 { + errors = append(errors, errs...) } raw.RawConfig = v @@ -411,8 +426,12 @@ func (t *Template) Build(name string, components *ComponentFinder) (b Build, err // Prepare the post-processors postProcessors := make([][]coreBuildPostProcessor, 0, len(t.PostProcessors)) for _, rawPPs := range t.PostProcessors { - current := make([]coreBuildPostProcessor, len(rawPPs)) - for i, rawPP := range rawPPs { + current := make([]coreBuildPostProcessor, 0, len(rawPPs)) + for _, rawPP := range rawPPs { + if rawPP.TemplateOnlyExcept.Skip(name) { + continue + } + pp, err := components.PostProcessor(rawPP.Type) if err != nil { return nil, err @@ -422,12 +441,18 @@ func (t *Template) Build(name string, components *ComponentFinder) (b Build, err return nil, fmt.Errorf("PostProcessor type not found: %s", rawPP.Type) } - current[i] = coreBuildPostProcessor{ + current = append(current, coreBuildPostProcessor{ processor: pp, processorType: rawPP.Type, config: rawPP.RawConfig, keepInputArtifact: rawPP.KeepInputArtifact, - } + }) + } + + // If we have no post-processors in this chain, just continue. + // This can happen if the post-processors skip certain builds. + if len(current) == 0 { + continue } postProcessors = append(postProcessors, current) @@ -436,19 +461,8 @@ func (t *Template) Build(name string, components *ComponentFinder) (b Build, err // Prepare the provisioners provisioners := make([]coreBuildProvisioner, 0, len(t.Provisioners)) for _, rawProvisioner := range t.Provisioners { - if len(rawProvisioner.Only) > 0 { - onlyFound := false - for _, n := range rawProvisioner.Only { - if n == name { - onlyFound = true - break - } - } - - if !onlyFound { - // Skip this provisioner - continue - } + if rawProvisioner.TemplateOnlyExcept.Skip(name) { + continue } var provisioner Provisioner @@ -497,3 +511,50 @@ func (t *Template) Build(name string, components *ComponentFinder) (b Build, err return } + +// TemplateOnlyExcept contains the logic required for "only" and "except" +// meta-parameters. +type TemplateOnlyExcept struct { + Only []string + Except []string +} + +// Prune will prune out the used values from the raw map. +func (t *TemplateOnlyExcept) Prune(raw map[string]interface{}) { + delete(raw, "except") + delete(raw, "only") +} + +// Skip tests if we should skip putting this item onto a build. +func (t *TemplateOnlyExcept) Skip(name string) bool { + if len(t.Only) > 0 { + onlyFound := false + for _, n := range t.Only { + if n == name { + onlyFound = true + break + } + } + + if !onlyFound { + // Skip this provisioner + return true + } + } + + return false +} + +// Validates the only/except parameters. +func (t *TemplateOnlyExcept) Validate(b map[string]RawBuilderConfig) (e []error) { + if len(t.Only) > 0 { + for _, n := range t.Only { + if _, ok := b[n]; !ok { + e = append(e, + fmt.Errorf("'only' specified builder '%s' not found", n)) + } + } + } + + return +} diff --git a/packer/template_test.go b/packer/template_test.go index c8fb6b4ae..c5c0cf146 100644 --- a/packer/template_test.go +++ b/packer/template_test.go @@ -11,21 +11,28 @@ import ( func testTemplateComponentFinder() *ComponentFinder { builder := testBuilder() + pp := new(TestPostProcessor) provisioner := &MockProvisioner{} builderMap := map[string]Builder{ "test-builder": builder, } + ppMap := map[string]PostProcessor{ + "test-pp": pp, + } + provisionerMap := map[string]Provisioner{ "test-prov": provisioner, } builderFactory := func(n string) (Builder, error) { return builderMap[n], nil } + ppFactory := func(n string) (PostProcessor, error) { return ppMap[n], nil } provFactory := func(n string) (Provisioner, error) { return provisionerMap[n], nil } return &ComponentFinder{ - Builder: builderFactory, - Provisioner: provFactory, + Builder: builderFactory, + PostProcessor: ppFactory, + Provisioner: provFactory, } } @@ -683,6 +690,86 @@ func TestTemplate_Build(t *testing.T) { } } +func TestTemplateBuild_onlyPPInvalid(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "post-processors": [ + { + "type": "test-pp", + "only": "test5" + } + ] + } + ` + + _, err := ParseTemplate([]byte(data)) + if err == nil { + t.Fatal("should have error") + } +} + +func TestTemplateBuild_onlyPP(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "post-processors": [ + { + "type": "test-pp", + "only": ["test2"] + } + ] + } + ` + + template, err := ParseTemplate([]byte(data)) + if err != nil { + t.Fatalf("err: %s", err) + } + + // Verify test1 has no post-processors + build, err := template.Build("test1", testTemplateComponentFinder()) + if err != nil { + t.Fatalf("err: %s", err) + } + + cbuild := build.(*coreBuild) + if len(cbuild.postProcessors) > 0 { + t.Fatal("should have no postProcessors") + } + + // Verify test2 has no post-processors + build, err = template.Build("test2", testTemplateComponentFinder()) + if err != nil { + t.Fatalf("err: %s", err) + } + + cbuild = build.(*coreBuild) + if len(cbuild.postProcessors) != 1 { + t.Fatalf("invalid: %d", len(cbuild.postProcessors)) + } +} + func TestTemplateBuild_onlyProvInvalid(t *testing.T) { data := ` { From a31a4207dfc190adfcda207755ee6a61ca02a543 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 20 Sep 2013 11:16:33 -0700 Subject: [PATCH 27/46] packer: `except` meta-parameter for both prov and PP [GH-438] --- packer/template.go | 7 ++ packer/template_test.go | 160 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) diff --git a/packer/template.go b/packer/template.go index 7e27bf93d..32513d33f 100644 --- a/packer/template.go +++ b/packer/template.go @@ -542,6 +542,13 @@ func (t *TemplateOnlyExcept) Skip(name string) bool { } } + // If the name is in the except list, then skip that + for _, n := range t.Except { + if n == name { + return true + } + } + return false } diff --git a/packer/template_test.go b/packer/template_test.go index c5c0cf146..3b14db021 100644 --- a/packer/template_test.go +++ b/packer/template_test.go @@ -690,6 +690,166 @@ func TestTemplate_Build(t *testing.T) { } } +func TestTemplateBuild_exeptPPInvalid(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "post-processors": [ + { + "type": "test-pp", + "except": "test5" + } + ] + } + ` + + _, err := ParseTemplate([]byte(data)) + if err == nil { + t.Fatal("should have error") + } +} + +func TestTemplateBuild_exceptPP(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "post-processors": [ + { + "type": "test-pp", + "except": ["test1"] + } + ] + } + ` + + template, err := ParseTemplate([]byte(data)) + if err != nil { + t.Fatalf("err: %s", err) + } + + // Verify test1 has no post-processors + build, err := template.Build("test1", testTemplateComponentFinder()) + if err != nil { + t.Fatalf("err: %s", err) + } + + cbuild := build.(*coreBuild) + if len(cbuild.postProcessors) > 0 { + t.Fatal("should have no postProcessors") + } + + // Verify test2 has no post-processors + build, err = template.Build("test2", testTemplateComponentFinder()) + if err != nil { + t.Fatalf("err: %s", err) + } + + cbuild = build.(*coreBuild) + if len(cbuild.postProcessors) != 1 { + t.Fatalf("invalid: %d", len(cbuild.postProcessors)) + } +} + +func TestTemplateBuild_exceptProvInvalid(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "provisioners": [ + { + "type": "test-prov", + "except": "test5" + } + ] + } + ` + + _, err := ParseTemplate([]byte(data)) + if err == nil { + t.Fatal("should have error") + } +} + +func TestTemplateBuild_exceptProv(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "provisioners": [ + { + "type": "test-prov", + "except": ["test1"] + } + ] + } + ` + + template, err := ParseTemplate([]byte(data)) + if err != nil { + t.Fatalf("err: %s", err) + } + + // Verify test1 has no provisioners + build, err := template.Build("test1", testTemplateComponentFinder()) + if err != nil { + t.Fatalf("err: %s", err) + } + + cbuild := build.(*coreBuild) + if len(cbuild.provisioners) > 0 { + t.Fatal("should have no provisioners") + } + + // Verify test2 has no provisioners + build, err = template.Build("test2", testTemplateComponentFinder()) + if err != nil { + t.Fatalf("err: %s", err) + } + + cbuild = build.(*coreBuild) + if len(cbuild.provisioners) != 1 { + t.Fatalf("invalid: %d", len(cbuild.provisioners)) + } +} + func TestTemplateBuild_onlyPPInvalid(t *testing.T) { data := ` { From 12ad2cf92ea90955021afea9fe74c5fa3aaed262 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 20 Sep 2013 11:18:00 -0700 Subject: [PATCH 28/46] packer: verify `except` has valid builders [GH-438] --- packer/template.go | 7 +++++++ packer/template_test.go | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packer/template.go b/packer/template.go index 32513d33f..cada49fd3 100644 --- a/packer/template.go +++ b/packer/template.go @@ -563,5 +563,12 @@ func (t *TemplateOnlyExcept) Validate(b map[string]RawBuilderConfig) (e []error) } } + for _, n := range t.Except { + if _, ok := b[n]; !ok { + e = append(e, + fmt.Errorf("'except' specified builder '%s' not found", n)) + } + } + return } diff --git a/packer/template_test.go b/packer/template_test.go index 3b14db021..62b1f54fa 100644 --- a/packer/template_test.go +++ b/packer/template_test.go @@ -707,7 +707,7 @@ func TestTemplateBuild_exeptPPInvalid(t *testing.T) { "post-processors": [ { "type": "test-pp", - "except": "test5" + "except": ["test5"] } ] } @@ -787,7 +787,7 @@ func TestTemplateBuild_exceptProvInvalid(t *testing.T) { "provisioners": [ { "type": "test-prov", - "except": "test5" + "except": ["test5"] } ] } @@ -867,7 +867,7 @@ func TestTemplateBuild_onlyPPInvalid(t *testing.T) { "post-processors": [ { "type": "test-pp", - "only": "test5" + "only": ["test5"] } ] } @@ -947,7 +947,7 @@ func TestTemplateBuild_onlyProvInvalid(t *testing.T) { "provisioners": [ { "type": "test-prov", - "only": "test5" + "only": ["test5"] } ] } From 5371f66599cda4600992dac3c4da969fb143b53d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 20 Sep 2013 11:20:05 -0700 Subject: [PATCH 29/46] packer: verify only one of 'only' or 'except' specified [GH-438] --- packer/template.go | 5 ++++ packer/template_test.go | 62 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/packer/template.go b/packer/template.go index cada49fd3..b398b54cc 100644 --- a/packer/template.go +++ b/packer/template.go @@ -554,6 +554,11 @@ func (t *TemplateOnlyExcept) Skip(name string) bool { // Validates the only/except parameters. func (t *TemplateOnlyExcept) Validate(b map[string]RawBuilderConfig) (e []error) { + if len(t.Only) > 0 && len(t.Except) > 0 { + e = append(e, + fmt.Errorf("Only one of 'only' or 'except' may be specified.")) + } + if len(t.Only) > 0 { for _, n := range t.Only { if _, ok := b[n]; !ok { diff --git a/packer/template_test.go b/packer/template_test.go index 62b1f54fa..aa531ad26 100644 --- a/packer/template_test.go +++ b/packer/template_test.go @@ -690,7 +690,67 @@ func TestTemplate_Build(t *testing.T) { } } -func TestTemplateBuild_exeptPPInvalid(t *testing.T) { +func TestTemplateBuild_exceptOnlyPP(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "post-processors": [ + { + "type": "test-pp", + "except": ["test1"], + "only": ["test1"] + } + ] + } + ` + + _, err := ParseTemplate([]byte(data)) + if err == nil { + t.Fatal("should have error") + } +} + +func TestTemplateBuild_exceptOnlyProv(t *testing.T) { + data := ` + { + "builders": [ + { + "name": "test1", + "type": "test-builder" + }, + { + "name": "test2", + "type": "test-builder" + } + ], + + "provisioners": [ + { + "type": "test-prov", + "except": ["test1"], + "only": ["test1"] + } + ] + } + ` + + _, err := ParseTemplate([]byte(data)) + if err == nil { + t.Fatal("should have error") + } +} + +func TestTemplateBuild_exceptPPInvalid(t *testing.T) { data := ` { "builders": [ From 15b48bb71e42cbeaa2ac4af7784c29ca14a27f29 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 20 Sep 2013 11:21:03 -0700 Subject: [PATCH 30/46] Update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c5cb3d60..1835745ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ FEATURES: +* core: You can now specify `only` and `except` configurations on any + provisioner or post-processor to specify a list of builds that they + are valid for. [GH-438] * builders/virtualbox: Guest additions can be attached rather than uploaded, easier to handle for Windows guests. [GH-405] * provisioner/chef-solo: Ability to specify a custom Chef configuration From 5c02bd3d26dfed96d8afe746d0af2adbdc593f44 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 20 Sep 2013 11:26:56 -0700 Subject: [PATCH 31/46] packer: better error message for bad provisioner only/except --- packer/template.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packer/template.go b/packer/template.go index b398b54cc..547348d91 100644 --- a/packer/template.go +++ b/packer/template.go @@ -273,7 +273,10 @@ func ParseTemplate(data []byte) (t *Template, err error) { // Verify that the only settings are good if errs := raw.TemplateOnlyExcept.Validate(t.Builders); len(errs) > 0 { - errors = append(errors, errs...) + for _, err := range errs { + errors = append(errors, + fmt.Errorf("provisioner %d: %s", i+1, err)) + } } raw.RawConfig = v From 44d1f3d9ccf176f8f15deb2e23a4a11b3a0bdf04 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 20 Sep 2013 11:42:25 -0700 Subject: [PATCH 32/46] website: document `only` and `except` --- website/Gemfile | 8 +- website/Gemfile.lock | 159 ++++++++---------- .../templates/post-processors.html.markdown | 25 +++ .../docs/templates/provisioners.html.markdown | 24 +++ 4 files changed, 120 insertions(+), 96 deletions(-) diff --git a/website/Gemfile b/website/Gemfile index d270a5ae4..2b3a472d5 100644 --- a/website/Gemfile +++ b/website/Gemfile @@ -2,11 +2,11 @@ source 'https://rubygems.org' ruby '1.9.3' -gem "middleman", "~> 3.0.6" -gem "middleman-minify-html", "~> 3.0.0" +gem "middleman", "~> 3.1.5" +gem "middleman-minify-html", "~> 3.1.1" gem "rack-contrib", "~> 1.1.0" -gem "redcarpet", "~> 2.2.2" -gem "therubyracer", "~> 0.10.2" +gem "redcarpet", "~> 3.0.0" +gem "therubyracer", "~> 0.12.0" gem "thin", "~> 1.5.0" group :development do diff --git a/website/Gemfile.lock b/website/Gemfile.lock index d79220dde..a25bfa939 100644 --- a/website/Gemfile.lock +++ b/website/Gemfile.lock @@ -1,134 +1,109 @@ GEM remote: https://rubygems.org/ specs: - POpen4 (0.1.4) - Platform (>= 0.4.0) - open4 - Platform (0.4.0) - activesupport (3.2.9) - i18n (~> 0.6) + activesupport (3.2.14) + i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - chunky_png (1.2.6) + chunky_png (1.2.8) coffee-script (2.2.0) coffee-script-source execjs - coffee-script-source (1.3.3) + coffee-script-source (1.6.3) compass (0.12.2) chunky_png (~> 1.2) fssm (>= 0.2.7) sass (~> 3.1) daemons (1.1.9) - eventmachine (1.0.0) + eventmachine (1.0.3) execjs (1.4.0) multi_json (~> 1.0) - ffi (1.2.0) - fssm (0.2.9) - haml (3.1.7) - highline (1.6.15) - hike (1.2.1) - htmlcompressor (0.0.3) - yui-compressor (~> 0.9.6) - http_router (0.10.2) - rack (>= 1.0.0) - url_mount (~> 0.2.1) - i18n (0.6.1) - libv8 (3.3.10.4) - listen (0.5.3) - maruku (0.6.1) - syntax (>= 1.0.0) - middleman (3.0.6) - middleman-core (= 3.0.6) - middleman-more (= 3.0.6) - middleman-sprockets (~> 3.0.2) - middleman-core (3.0.6) - activesupport (~> 3.2.6) - bundler (~> 1.1) - listen (~> 0.5.2) - rack (~> 1.4.1) - rack-test (~> 0.6.1) - rb-fsevent (~> 0.9.1) - rb-inotify (~> 0.8.8) - thor (~> 0.15.4) - tilt (~> 1.3.1) - middleman-minify-html (3.0.0) - htmlcompressor - middleman-core (~> 3.0.0) - middleman-more (3.0.6) + ffi (1.9.0) + fssm (0.2.10) + haml (4.0.3) + tilt + highline (1.6.19) + hike (1.2.3) + i18n (0.6.5) + kramdown (1.1.0) + libv8 (3.16.14.3) + listen (1.2.3) + rb-fsevent (>= 0.9.3) + rb-inotify (>= 0.9) + rb-kqueue (>= 0.2) + middleman (3.1.5) coffee-script (~> 2.2.0) - coffee-script-source (~> 1.3.3) compass (>= 0.12.2) execjs (~> 1.4.0) haml (>= 3.1.6) - i18n (~> 0.6.0) - maruku (~> 0.6.0) - middleman-core (= 3.0.6) - padrino-helpers (= 0.10.7) + kramdown (~> 1.1.0) + middleman-core (= 3.1.5) + middleman-more (= 3.1.5) + middleman-sprockets (>= 3.1.2) sass (>= 3.1.20) - uglifier (~> 1.2.6) - middleman-sprockets (3.0.4) - middleman-more (~> 3.0.1) - sprockets (~> 2.1, < 2.5) - sprockets-sass (~> 0.8.0) - multi_json (1.4.0) - open4 (1.3.0) - padrino-core (0.10.7) - activesupport (~> 3.2.0) - http_router (~> 0.10.2) - sinatra (~> 1.3.1) - thor (~> 0.15.2) - tilt (~> 1.3.0) - padrino-helpers (0.10.7) - i18n (~> 0.6) - padrino-core (= 0.10.7) - rack (1.4.1) + uglifier (~> 2.1.0) + middleman-core (3.1.5) + activesupport (~> 3.2.6) + bundler (~> 1.1) + i18n (~> 0.6.1) + listen (~> 1.2.2) + rack (>= 1.4.5) + rack-test (~> 0.6.1) + thor (>= 0.15.2, < 2.0) + tilt (~> 1.3.6) + middleman-minify-html (3.1.1) + middleman-core (~> 3.0) + middleman-more (3.1.5) + middleman-sprockets (3.1.4) + middleman-core (>= 3.0.14) + middleman-more (>= 3.0.14) + sprockets (~> 2.1) + sprockets-helpers (~> 1.0.0) + sprockets-sass (~> 1.0.0) + multi_json (1.8.0) + rack (1.5.2) rack-contrib (1.1.0) rack (>= 0.9.1) - rack-protection (1.2.0) - rack rack-test (0.6.2) rack (>= 1.0) - rb-fsevent (0.9.2) - rb-inotify (0.8.8) + rb-fsevent (0.9.3) + rb-inotify (0.9.2) ffi (>= 0.5.0) - redcarpet (2.2.2) - sass (3.2.3) - sinatra (1.3.3) - rack (~> 1.3, >= 1.3.6) - rack-protection (~> 1.2) - tilt (~> 1.3, >= 1.3.3) - sprockets (2.4.5) + rb-kqueue (0.2.0) + ffi (>= 0.5.0) + redcarpet (3.0.0) + ref (1.0.5) + sass (3.2.10) + sprockets (2.10.0) hike (~> 1.2) multi_json (~> 1.0) rack (~> 1.0) tilt (~> 1.1, != 1.3.0) - sprockets-sass (0.8.0) + sprockets-helpers (1.0.1) + sprockets (~> 2.0) + sprockets-sass (1.0.1) sprockets (~> 2.0) tilt (~> 1.1) - syntax (1.0.0) - therubyracer (0.10.2) - libv8 (~> 3.3.10) - thin (1.5.0) + therubyracer (0.12.0) + libv8 (~> 3.16.14.0) + ref + thin (1.5.1) daemons (>= 1.0.9) eventmachine (>= 0.12.6) rack (>= 1.0.0) - thor (0.15.4) - tilt (1.3.3) - uglifier (1.2.7) + thor (0.18.1) + tilt (1.3.7) + uglifier (2.1.2) execjs (>= 0.3.0) - multi_json (~> 1.3) - url_mount (0.2.1) - rack - yui-compressor (0.9.6) - POpen4 (>= 0.1.4) + multi_json (~> 1.0, >= 1.0.2) PLATFORMS ruby DEPENDENCIES highline (~> 1.6.15) - middleman (~> 3.0.6) - middleman-minify-html (~> 3.0.0) + middleman (~> 3.1.5) + middleman-minify-html (~> 3.1.1) rack-contrib (~> 1.1.0) - redcarpet (~> 2.2.2) - therubyracer (~> 0.10.2) + redcarpet (~> 3.0.0) + therubyracer (~> 0.12.0) thin (~> 1.5.0) diff --git a/website/source/docs/templates/post-processors.html.markdown b/website/source/docs/templates/post-processors.html.markdown index e4c1bfa8c..cda12e2a2 100644 --- a/website/source/docs/templates/post-processors.html.markdown +++ b/website/source/docs/templates/post-processors.html.markdown @@ -125,3 +125,28 @@ The answer is no, of course not. Packer is smart enough to figure out that at least one post-processor requested that the input be kept, so it will keep it around. + +## Run on Specific Builds + +You can use the `only` or `except` configurations to run a post-processor +only with specific builds. These two configurations do what you expect: +`only` will only run the post-processor on the specified builds and +`except` will run the post-processor on anything other than the specified +builds. + +An example of `only` being used is shown below, but the usage of `except` +is effectively the same. `only` and `except` can only be specified on "detailed" +configurations. If you have a sequence of post-processors to run, `only` +and `except` will only affect that single post-processor in the sequence. + +
+{
+  "type": "vagrant",
+  "only": ["virtualbox"]
+}
+
+ +The values within `only` or `except` are _build names_, not builder +types. If you recall, build names by default are just their builder type, +but if you specify a custom `name` parameter, then you should use that +as the value instead of the type. diff --git a/website/source/docs/templates/provisioners.html.markdown b/website/source/docs/templates/provisioners.html.markdown index f4263fb9b..eb49788e1 100644 --- a/website/source/docs/templates/provisioners.html.markdown +++ b/website/source/docs/templates/provisioners.html.markdown @@ -53,6 +53,30 @@ provisioner to run a local script within the machines: } +## Run on Specific Builds + +You can use the `only` or `except` configurations to run a provisioner +only with specific builds. These two configurations do what you expect: +`only` will only run the provisioner on the specified builds and +`except` will run the provisioner on anything other than the specified +builds. + +An example of `only` being used is shown below, but the usage of `except` +is effectively the same: + +
+{
+  "type": "shell",
+  "script": "script.sh",
+  "only": ["virtualbox"]
+}
+
+ +The values within `only` or `except` are _build names_, not builder +types. If you recall, build names by default are just their builder type, +but if you specify a custom `name` parameter, then you should use that +as the value instead of the type. + ## Build-Specific Overrides While the goal of Packer is to produce identical machine images, it From 25924a2d2b3917f530fe193818dbed5d604ec56a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 21 Sep 2013 18:28:14 -0700 Subject: [PATCH 33/46] provisioner/puppet-masterless: user variables in puppet vars [GH-448] --- CHANGELOG.md | 2 ++ provisioner/puppet-masterless/provisioner.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1835745ec..f2e696741 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ BUG FIXES: * post-processor/vagrant: Override configurations properly work. [GH-426] * provisioner/puppet-masterless: Fix failure case when both facter vars are used and prevent_sudo. [GH-415] +* provisioner/puppet-masterless: User variables now work properly in + manifest file and hiera path. [GH-448] ## 0.3.7 (September 9, 2013) diff --git a/provisioner/puppet-masterless/provisioner.go b/provisioner/puppet-masterless/provisioner.go index ff8b2112b..4552aef9a 100644 --- a/provisioner/puppet-masterless/provisioner.go +++ b/provisioner/puppet-masterless/provisioner.go @@ -82,7 +82,9 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { // Templates templates := map[string]*string{ - "staging_dir": &p.config.StagingDir, + "hiera_config_path": &p.config.HieraConfigPath, + "manifest_file": &p.config.ManifestFile, + "staging_dir": &p.config.StagingDir, } for n, ptr := range templates { From 6965af291bc766229fdc9feb09ae69b528a5d968 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 22 Sep 2013 09:39:27 -0700 Subject: [PATCH 34/46] packer/plugin: log git commit plugin was built against --- packer/plugin/plugin.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packer/plugin/plugin.go b/packer/plugin/plugin.go index f9fca8cdd..9afba0285 100644 --- a/packer/plugin/plugin.go +++ b/packer/plugin/plugin.go @@ -33,6 +33,8 @@ const MagicCookieValue = "d602bf8f470bc67ca7faa0386276bbdd4330efaf76d1a219cb4d69 // This serves a single RPC connection on the given RPC server on // a random port. func serve(server *rpc.Server) (err error) { + log.Printf("Plugin build against Packer '%s'", packer.GitCommit) + if os.Getenv(MagicCookieKey) != MagicCookieValue { return errors.New("Please do not execute plugins directly. Packer will execute these for you.") } From c7b10cb2cf84cd67023bf88ce56f1ac567649649 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 22 Sep 2013 09:51:14 -0700 Subject: [PATCH 35/46] packer/plugin: detect invalid versions --- CHANGELOG.md | 1 + packer/plugin/client.go | 22 ++++++++++++++++++---- packer/plugin/client_test.go | 15 +++++++++++++++ packer/plugin/plugin.go | 7 ++++++- packer/plugin/plugin_test.go | 9 ++++++--- 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2e696741..2bd9fc504 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ IMPROVEMENTS: * core: User variables can now be used for integer, boolean, etc. values. [GH-418] +* core: Plugins made with incompatible versions will no longer load. * builder/amazon/all: Interrupts work while waiting for AMI to be ready. * provisioner/shell: Script line-endings are automatically converted to Unix-style line-endings. Can be disabled by setting "binary" to "true". diff --git a/packer/plugin/client.go b/packer/plugin/client.go index d3a302977..792f9d99d 100644 --- a/packer/plugin/client.go +++ b/packer/plugin/client.go @@ -317,10 +317,24 @@ func (c *Client) Start() (address string, err error) { err = errors.New("timeout while waiting for plugin to start") case <-exitCh: err = errors.New("plugin exited before we could connect") - case line := <-linesCh: - // Trim the address and reset the err since we were able - // to read some sort of address. - c.address = strings.TrimSpace(string(line)) + case lineBytes := <-linesCh: + // Trim the line and split by "|" in order to get the parts of + // the output. + line := strings.TrimSpace(string(lineBytes)) + parts := strings.SplitN(line, "|", 2) + if len(parts) < 2 { + err = fmt.Errorf("Unrecognized remote plugin message: %s", line) + return + } + + // Test the API version + if parts[0] != APIVersion { + err = fmt.Errorf("Incompatible API version with plugin. "+ + "Plugin version: %s, Ours: %s", parts[0], APIVersion) + return + } + + c.address = parts[1] address = c.address } diff --git a/packer/plugin/client_test.go b/packer/plugin/client_test.go index ae71c3362..f9257034e 100644 --- a/packer/plugin/client_test.go +++ b/packer/plugin/client_test.go @@ -37,6 +37,21 @@ func TestClient(t *testing.T) { } } +func TestClientStart_badVersion(t *testing.T) { + config := &ClientConfig{ + Cmd: helperProcess("bad-version"), + StartTimeout: 50 * time.Millisecond, + } + + c := NewClient(config) + defer c.Kill() + + _, err := c.Start() + if err == nil { + t.Fatal("err should not be nil") + } +} + func TestClient_Start_Timeout(t *testing.T) { config := &ClientConfig{ Cmd: helperProcess("start-timeout"), diff --git a/packer/plugin/plugin.go b/packer/plugin/plugin.go index 9afba0285..a91fcc3ce 100644 --- a/packer/plugin/plugin.go +++ b/packer/plugin/plugin.go @@ -30,6 +30,11 @@ var Interrupts int32 = 0 const MagicCookieKey = "PACKER_PLUGIN_MAGIC_COOKIE" const MagicCookieValue = "d602bf8f470bc67ca7faa0386276bbdd4330efaf76d1a219cb4d6991ca9872b2" +// The APIVersion is outputted along with the RPC address. The plugin +// client validates this API version and will show an error if it doesn't +// know how to speak it. +const APIVersion = "1" + // This serves a single RPC connection on the given RPC server on // a random port. func serve(server *rpc.Server) (err error) { @@ -77,7 +82,7 @@ func serve(server *rpc.Server) (err error) { // Output the address to stdout log.Printf("Plugin address: %s\n", address) - fmt.Println(address) + fmt.Printf("%s|%s\n", APIVersion, address) os.Stdout.Sync() // Accept a connection diff --git a/packer/plugin/plugin_test.go b/packer/plugin/plugin_test.go index 17018f82d..10c3f9d5c 100644 --- a/packer/plugin/plugin_test.go +++ b/packer/plugin/plugin_test.go @@ -50,6 +50,9 @@ func TestHelperProcess(*testing.T) { cmd, args := args[0], args[1:] switch cmd { + case "bad-version": + fmt.Printf("%s1|:1234\n", APIVersion) + <-make(chan int) case "builder": ServeBuilder(new(helperBuilder)) case "command": @@ -59,7 +62,7 @@ func TestHelperProcess(*testing.T) { case "invalid-rpc-address": fmt.Println("lolinvalid") case "mock": - fmt.Println(":1234") + fmt.Printf("%s|:1234\n", APIVersion) <-make(chan int) case "post-processor": ServePostProcessor(new(helperPostProcessor)) @@ -69,11 +72,11 @@ func TestHelperProcess(*testing.T) { time.Sleep(1 * time.Minute) os.Exit(1) case "stderr": - fmt.Println(":1234") + fmt.Printf("%s|:1234\n", APIVersion) log.Println("HELLO") log.Println("WORLD") case "stdin": - fmt.Println(":1234") + fmt.Printf("%s|:1234\n", APIVersion) data := make([]byte, 5) if _, err := os.Stdin.Read(data); err != nil { log.Printf("stdin read error: %s", err) From a2e6e8b3986bed88ec71e9a372104db11102cc90 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 22 Sep 2013 09:53:53 -0700 Subject: [PATCH 36/46] Add updatedeps to .PHONY list in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 934a12852..aa7bbae2e 100644 --- a/Makefile +++ b/Makefile @@ -28,4 +28,4 @@ test: deps @echo "$(OK_COLOR)==> Testing Packer...$(NO_COLOR)" go test ./... -.PHONY: all deps format test +.PHONY: all deps format test updatedeps From 3f4c9a500eb8d920839681da1f4c50a3dbe16fd8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 22 Sep 2013 09:54:21 -0700 Subject: [PATCH 37/46] Add clean to .PHONY list in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index aa7bbae2e..e3d7e9fcd 100644 --- a/Makefile +++ b/Makefile @@ -28,4 +28,4 @@ test: deps @echo "$(OK_COLOR)==> Testing Packer...$(NO_COLOR)" go test ./... -.PHONY: all deps format test updatedeps +.PHONY: all clean deps format test updatedeps From 3eb744249f6bbbe88018dc7d06806c6e7c6a5874 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 22 Sep 2013 09:55:01 -0700 Subject: [PATCH 38/46] make updatedeps updates the core deps too --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index e3d7e9fcd..a74b30740 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ deps: updatedeps: @echo "$(OK_COLOR)==> Updating all dependencies$(NO_COLOR)" + @go get -d -v -u ./... @echo $(DEPS) | xargs -n1 go get -d -u clean: From 6511b1892c0f92e3b71fe77956d515753a774df7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 22 Sep 2013 09:58:34 -0700 Subject: [PATCH 39/46] scripts: build.sh compiles all Packer components in parallel --- scripts/build.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/scripts/build.sh b/scripts/build.sh index 95ba90abf..8d0976b6b 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -35,21 +35,44 @@ fi echo -e "${OK_COLOR}--> Installing dependencies to speed up builds...${NO_COLOR}" go get ./... +# This function waits for all background tasks to complete +waitAll() { + RESULT=0 + for job in `jobs -p`; do + wait $job + if [ $? -ne 0 ]; then + RESULT=1 + fi + done + + if [ $RESULT -ne 0 ]; then + exit $RESULT + fi +} + +echo -e "${OK_COLOR}--> NOTE: Compilation of components will be done in parallel.${NO_COLOR}" + # Compile the main Packer app echo -e "${OK_COLOR}--> Compiling Packer${NO_COLOR}" +( go build \ ${PACKER_RACE} \ -ldflags "-X github.com/mitchellh/packer/packer.GitCommit ${GIT_COMMIT}${GIT_DIRTY}" \ -v \ -o bin/packer${EXTENSION} . +) & # Go over each plugin and build it for PLUGIN in $(find ./plugin -mindepth 1 -maxdepth 1 -type d); do PLUGIN_NAME=$(basename ${PLUGIN}) echo -e "${OK_COLOR}--> Compiling Plugin: ${PLUGIN_NAME}${NO_COLOR}" + ( go build \ ${PACKER_RACE} \ -ldflags "-X github.com/mitchellh/packer/packer.GitCommit ${GIT_COMMIT}${GIT_DIRTY}" \ -v \ -o bin/packer-${PLUGIN_NAME}${EXTENSION} ${PLUGIN} + ) & done + +waitAll From 1e309b660925ea615025cec119c0ab3b7432771f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 22 Sep 2013 10:03:04 -0700 Subject: [PATCH 40/46] scripts: PACKER_NO_BUILD_PARALLEL to build in sequence --- scripts/build.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index 8d0976b6b..8ba8d3d4f 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -26,6 +26,9 @@ if [ "$(go env GOOS)" = "windows" ]; then EXTENSION=".exe" fi +# Make sure that if we're killed, we kill all our subprocseses +trap "kill 0" SIGINT SIGTERM EXIT + # If we're building a race-enabled build, then set that up. if [ ! -z $PACKER_RACE ]; then echo -e "${OK_COLOR}--> Building with race detection enabled${NO_COLOR}" @@ -50,7 +53,16 @@ waitAll() { fi } -echo -e "${OK_COLOR}--> NOTE: Compilation of components will be done in parallel.${NO_COLOR}" +waitSingle() { + if [ ! -z $PACKER_NO_BUILD_PARALLEL ]; then + waitAll + fi +} + +if [ -z $PACKER_NO_BUILD_PARALLEL ]; then + echo -e "${OK_COLOR}--> NOTE: Compilation of components " \ + "will be done in parallel.${NO_COLOR}" +fi # Compile the main Packer app echo -e "${OK_COLOR}--> Compiling Packer${NO_COLOR}" @@ -62,6 +74,8 @@ go build \ -o bin/packer${EXTENSION} . ) & +waitSingle + # Go over each plugin and build it for PLUGIN in $(find ./plugin -mindepth 1 -maxdepth 1 -type d); do PLUGIN_NAME=$(basename ${PLUGIN}) @@ -73,6 +87,8 @@ for PLUGIN in $(find ./plugin -mindepth 1 -maxdepth 1 -type d); do -v \ -o bin/packer-${PLUGIN_NAME}${EXTENSION} ${PLUGIN} ) & + + waitSingle done waitAll From dca427ce303b03e3182f83be33e615886643d2f3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 22 Sep 2013 15:56:11 -0500 Subject: [PATCH 41/46] v0.3.8 --- CHANGELOG.md | 2 +- packer/version.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bd9fc504..da3fe4660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 0.3.8 (unreleased) +## 0.3.8 (September 22, 2013) FEATURES: diff --git a/packer/version.go b/packer/version.go index 8f95a99ff..66129d8f8 100644 --- a/packer/version.go +++ b/packer/version.go @@ -15,7 +15,7 @@ const Version = "0.3.8" // Any pre-release marker for the version. If this is "" (empty string), // then it means that it is a final release. Otherwise, this is the // pre-release marker. -const VersionPrerelease = "dev" +const VersionPrerelease = "" type versionCommand byte From 8a4a08b1430bd579077e1582458b8e21953a5c29 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 22 Sep 2013 16:15:55 -0500 Subject: [PATCH 42/46] Update version for dev --- CHANGELOG.md | 4 ++++ packer/version.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da3fe4660..6ba16b249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.3.9 (unreleased) + + + ## 0.3.8 (September 22, 2013) FEATURES: diff --git a/packer/version.go b/packer/version.go index 66129d8f8..758985d14 100644 --- a/packer/version.go +++ b/packer/version.go @@ -10,12 +10,12 @@ import ( var GitCommit string // The version of packer. -const Version = "0.3.8" +const Version = "0.3.9" // Any pre-release marker for the version. If this is "" (empty string), // then it means that it is a final release. Otherwise, this is the // pre-release marker. -const VersionPrerelease = "" +const VersionPrerelease = "dev" type versionCommand byte From c47f740946d682272ad69d256199befc8431d8dc Mon Sep 17 00:00:00 2001 From: Nathan Sullivan Date: Tue, 24 Sep 2013 16:14:34 +1000 Subject: [PATCH 43/46] fix variable name --- website/source/docs/provisioners/shell.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/provisioners/shell.html.markdown b/website/source/docs/provisioners/shell.html.markdown index 4c5a19992..d47b59733 100644 --- a/website/source/docs/provisioners/shell.html.markdown +++ b/website/source/docs/provisioners/shell.html.markdown @@ -102,7 +102,7 @@ root privileges without worrying about password prompts. ## Default Environmental Variables In addition to being able to specify custom environmental variables using -the `environmental_vars` configuration, the provisioner automatically +the `environment_vars` configuration, the provisioner automatically defines certain commonly useful environmental variables: * `PACKER_BUILD_NAME` is set to the name of the build that Packer is running. From d3c2b3492c572c445dc6eb33a2968872788fdf95 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Tue, 24 Sep 2013 01:40:42 -0700 Subject: [PATCH 44/46] use interactive shell to run build script. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a74b30740..77e9b5a23 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ DEPS = $(go list -f '{{range .TestImports}}{{.}} {{end}}' ./...) all: deps @mkdir -p bin/ @echo "$(OK_COLOR)==> Building$(NO_COLOR)" - @./scripts/build.sh + @bash --norc -i ./scripts/build.sh deps: @echo "$(OK_COLOR)==> Installing dependencies$(NO_COLOR)" From f0d0621855f5b1d0dd3e956030f7d832db30bc91 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 24 Sep 2013 22:59:44 +0200 Subject: [PATCH 45/46] packer: default user var values needn't be strings [GH-456] --- CHANGELOG.md | 2 ++ packer/template.go | 24 +++++++++++++++--------- packer/template_test.go | 13 +++++++++++-- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba16b249..562fe7b09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## 0.3.9 (unreleased) +BUG FIXES: +* core: default user variable values don't need to be strings. [GH-456] ## 0.3.8 (September 22, 2013) diff --git a/packer/template.go b/packer/template.go index 547348d91..063508ba4 100644 --- a/packer/template.go +++ b/packer/template.go @@ -124,18 +124,24 @@ func ParseTemplate(data []byte) (t *Template, err error) { // Gather all the variables for k, v := range rawTpl.Variables { var variable RawVariable - variable.Default = "" variable.Required = v == nil - if v != nil { - def, ok := v.(string) - if !ok { - errors = append(errors, - fmt.Errorf("variable '%s': default value must be string or null", k)) - continue - } + // Create a new mapstructure decoder in order to decode the default + // value since this is the only value in the regular template that + // can be weakly typed. + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: &variable.Default, + WeaklyTypedInput: true, + }) + if err != nil { + // This should never happen. + panic(err) + } - variable.Default = def + err = decoder.Decode(v) + if err != nil { + errors = append(errors, + fmt.Errorf("Error decoding default value for user var '%s': %s", k, err)) } t.Variables[k] = variable diff --git a/packer/template_test.go b/packer/template_test.go index aa531ad26..d457e7932 100644 --- a/packer/template_test.go +++ b/packer/template_test.go @@ -391,7 +391,8 @@ func TestParseTemplate_Variables(t *testing.T) { { "variables": { "foo": "bar", - "bar": null + "bar": null, + "baz": 27 }, "builders": [{"type": "something"}] @@ -403,7 +404,7 @@ func TestParseTemplate_Variables(t *testing.T) { t.Fatalf("err: %s", err) } - if result.Variables == nil || len(result.Variables) != 2 { + if result.Variables == nil || len(result.Variables) != 3 { t.Fatalf("bad vars: %#v", result.Variables) } @@ -422,6 +423,14 @@ func TestParseTemplate_Variables(t *testing.T) { if !result.Variables["bar"].Required { t.Fatal("bar should be required") } + + if result.Variables["baz"].Default != "27" { + t.Fatal("default should be empty") + } + + if result.Variables["baz"].Required { + t.Fatal("baz should not be required") + } } func TestParseTemplate_variablesBadDefault(t *testing.T) { From 6face65ecc9fae78ddbc12c81b0ee25f6b32da76 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 24 Sep 2013 23:01:16 +0200 Subject: [PATCH 46/46] packer: skip the user var if there was an error --- packer/template.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packer/template.go b/packer/template.go index 063508ba4..180b02c0f 100644 --- a/packer/template.go +++ b/packer/template.go @@ -142,6 +142,7 @@ func ParseTemplate(data []byte) (t *Template, err error) { if err != nil { errors = append(errors, fmt.Errorf("Error decoding default value for user var '%s': %s", k, err)) + continue } t.Variables[k] = variable