From cce13e3877074b9f0ec2037750533213fb45aa9d Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Wed, 29 Aug 2018 10:44:21 -0700 Subject: [PATCH 1/4] shell provisioner: add option to source env vars from a file --- provisioner/shell/provisioner.go | 91 +++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 6 deletions(-) diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index 8dd5796ce..ad68d1739 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -45,6 +45,10 @@ type Config struct { // your command(s) are executed. Vars []string `mapstructure:"environment_vars"` + // Write the Vars to a file and source them from there rather than declaring + // inline + UseEnvVarFile bool `mapstructure:"use_env_var_file"` + // The remote folder where the local shell script will be uploaded to. // This should be set to a pre-existing directory, it defaults to /tmp RemoteFolder string `mapstructure:"remote_folder"` @@ -75,6 +79,8 @@ type Config struct { startRetryTimeout time.Duration ctx interpolate.Context + // name of the tmp environment variable file, if UseEnvVarFile is true + envVarFile string } type Provisioner struct { @@ -82,8 +88,9 @@ type Provisioner struct { } type ExecuteCommandTemplate struct { - Vars string - Path string + Vars string + EnvVarFile string + Path string } func (p *Provisioner) Prepare(raws ...interface{}) error { @@ -102,6 +109,9 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { if p.config.ExecuteCommand == "" { p.config.ExecuteCommand = "chmod +x {{.Path}}; {{.Vars}} {{.Path}}" + if p.config.UseEnvVarFile == true { + p.config.ExecuteCommand = "chmod +x {{.Path}}; . {{.EnvVarFile}} && {{.Path}}" + } } if p.config.Inline != nil && len(p.config.Inline) == 0 { @@ -218,6 +228,57 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { tf.Close() } + if p.config.UseEnvVarFile == true { + tf, err := ioutil.TempFile("", "packer-shell-vars") + if err != nil { + return fmt.Errorf("Error preparing shell script: %s", err) + } + defer os.Remove(tf.Name()) + + // Write our contents to it + writer := bufio.NewWriter(tf) + if _, err := writer.WriteString(p.createEnvVarFileContent()); err != nil { + return fmt.Errorf("Error preparing shell script: %s", err) + } + + if err := writer.Flush(); err != nil { + return fmt.Errorf("Error preparing shell script: %s", err) + } + + p.config.envVarFile = tf.Name() + defer os.Remove(p.config.envVarFile) + + // upload the var file + var cmd *packer.RemoteCmd + err = p.retryable(func() error { + if _, err := tf.Seek(0, 0); err != nil { + return err + } + + var r io.Reader = tf + if !p.config.Binary { + r = &UnixReader{Reader: r} + } + remoteVFName := fmt.Sprintf("%s/%s", p.config.RemoteFolder, "varfile") + if err := comm.Upload(remoteVFName, r, nil); err != nil { + return fmt.Errorf("Error uploading envVarFile: %s", err) + } + tf.Close() + + cmd = &packer.RemoteCmd{ + Command: fmt.Sprintf("chmod 0600 %s", remoteVFName), + } + if err := comm.Start(cmd); err != nil { + return fmt.Errorf( + "Error chmodding script file to 0755 in remote "+ + "machine: %s", err) + } + cmd.Wait() + p.config.envVarFile = remoteVFName + return nil + }) + } + // Create environment variables to set before executing the command flattenedEnvVars := p.createFlattenedEnvVars() @@ -233,8 +294,9 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { // Compile the command p.config.ctx.Data = &ExecuteCommandTemplate{ - Vars: flattenedEnvVars, - Path: p.config.RemotePath, + Vars: flattenedEnvVars, + EnvVarFile: p.config.envVarFile, + Path: p.config.RemotePath, } command, err := interpolate.Render(p.config.ExecuteCommand, &p.config.ctx) if err != nil { @@ -360,8 +422,7 @@ func (p *Provisioner) retryable(f func() error) error { } } -func (p *Provisioner) createFlattenedEnvVars() (flattened string) { - flattened = "" +func (p *Provisioner) escapeEnvVars() ([]string, map[string]string) { envVars := make(map[string]string) // Always available Packer provided env vars @@ -387,6 +448,24 @@ func (p *Provisioner) createFlattenedEnvVars() (flattened string) { } sort.Strings(keys) + return keys, envVars +} + +func (p *Provisioner) createEnvVarFileContent() string { + keys, envVars := p.escapeEnvVars() + + flattened := "" + // Re-assemble vars surrounding value with single quotes and flatten + for _, key := range keys { + flattened += fmt.Sprintf("export %s='%s'\n", key, envVars[key]) + } + + return flattened +} + +func (p *Provisioner) createFlattenedEnvVars() (flattened string) { + keys, envVars := p.escapeEnvVars() + // Re-assemble vars surrounding value with single quotes and flatten for _, key := range keys { flattened += fmt.Sprintf("%s='%s' ", key, envVars[key]) From 35406bbfc270588ba69aa60c7beb960ce50b3161 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Wed, 29 Aug 2018 11:03:42 -0700 Subject: [PATCH 2/4] update shell provisioner docs with new use_env_var_file option and clear descriptions of quoting --- .../source/docs/provisioners/shell.html.md | 65 +++++++++++++++++-- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/website/source/docs/provisioners/shell.html.md b/website/source/docs/provisioners/shell.html.md index e5c8f48b9..7f885137a 100644 --- a/website/source/docs/provisioners/shell.html.md +++ b/website/source/docs/provisioners/shell.html.md @@ -65,13 +65,25 @@ Optional parameters: Packer injects some environmental variables by default into the environment, as well, which are covered in the section below. -- `execute_command` (string) - The command to use to execute the script. By - default this is `chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}`. The value - of this is treated as [configuration - template](/docs/templates/engine.html). There are two - available variables: `Path`, which is the path to the script to run, and - `Vars`, which is the list of `environment_vars`, if configured. +- `use_env_var_file` (boolean) - If true, Packer will write your environment + variables to a tempfile and source them from that file, rather than + declaring them inline in our execute_command. The default `execute_command` + will be `chmod +x {{.Path}}; . {{.EnvVarFile}} && {{.Path}}`. This option is + unnecessary for most cases, but if you have extra quoting in your custom + `execute_command`, then this may be neccecary for proper script execution. + Default: false. +- `execute_command` (string) - The command to use to execute the script. By + default this is `chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}`, unless the + user has set `"use_env_var_file": true` -- in that case, the default + `execute_command` is `chmod +x {{.Path}}; . {{.EnvVarFile}} && {{.Path}}`. + The value of this is treated as a + [configuration template](/docs/templates/engine.html). There are three + available variables: + * `Path` is the path to the script to run + * `Vars` is the list of `environment_vars`, if configured. + * `EnvVarFile` is the path to the file containing env vars, if + `use_env_var_file` is true. - `expect_disconnect` (boolean) - Defaults to `false`. Whether to error if the server disconnects us. A disconnect might happen if you restart the ssh server or reboot the host. @@ -256,9 +268,48 @@ would be: create race conditions. Your first provisioner can tell the machine to wait until it completely boots. -``` json +```json { "type": "shell", "inline": [ "sleep 10" ] } ``` + +## Quoting Environment Variables + +Packer manages quoting for you, so you should't have to worry about it. +Below is an example of packer template inputs and what you should expect to get +out: + +```json + "provisioners": [ + { + "type": "shell", + "environment_vars": ["FOO=foo", + "BAR=bar's", + "BAZ=baz=baz", + "QUX==qux", + "FOOBAR=foo bar", + "FOOBARBAZ='foo bar baz'", + "QUX2=\"qux\""], + "inline": ["echo \"FOO is $FOO\"", + "echo \"BAR is $BAR\"", + "echo \"BAZ is $BAZ\"", + "echo \"QUX is $QUX\"", + "echo \"FOOBAR is $FOOBAR\"", + "echo \"FOOBARBAZ is $FOOBARBAZ\"", + "echo \"QUX2 is $QUX2\""] + } +``` + +Output: + +``` + docker: FOO is foo + docker: BAR is bar's + docker: BAZ is baz=baz + docker: QUX is =qux + docker: FOOBAR is foo bar + docker: FOOBARBAZ is 'foo bar baz' + docker: QUX2 is "qux" +``` From 2c9a205f11902bc03278d4c70fc3a4041d94c412 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Wed, 29 Aug 2018 11:10:45 -0700 Subject: [PATCH 3/4] update shell provisioner tests --- provisioner/shell/provisioner_test.go | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/provisioner/shell/provisioner_test.go b/provisioner/shell/provisioner_test.go index c190f8c52..fd3914383 100644 --- a/provisioner/shell/provisioner_test.go +++ b/provisioner/shell/provisioner_test.go @@ -286,6 +286,61 @@ func TestProvisioner_createFlattenedEnvVars(t *testing.T) { } } +func TestProvisioner_createEnvVarFileContent(t *testing.T) { + var flattenedEnvVars string + config := testConfig() + + userEnvVarTests := [][]string{ + {}, // No user env var + {"FOO=bar"}, // Single user env var + {"FOO=bar's"}, // User env var with single quote in value + {"FOO=bar", "BAZ=qux"}, // Multiple user env vars + {"FOO=bar=baz"}, // User env var with value containing equals + {"FOO==bar"}, // User env var with value starting with equals + } + expected := []string{ + `export PACKER_BUILDER_TYPE='iso' +export PACKER_BUILD_NAME='vmware' +`, + `export FOO='bar' +export PACKER_BUILDER_TYPE='iso' +export PACKER_BUILD_NAME='vmware' +`, + `export FOO='bar'"'"'s' +export PACKER_BUILDER_TYPE='iso' +export PACKER_BUILD_NAME='vmware' +`, + `export BAZ='qux' +export FOO='bar' +export PACKER_BUILDER_TYPE='iso' +export PACKER_BUILD_NAME='vmware' +`, + `export FOO='bar=baz' +export PACKER_BUILDER_TYPE='iso' +export PACKER_BUILD_NAME='vmware' +`, + `export FOO='=bar' +export PACKER_BUILDER_TYPE='iso' +export PACKER_BUILD_NAME='vmware' +`, + } + + p := new(Provisioner) + p.Prepare(config) + + // Defaults provided by Packer + p.config.PackerBuildName = "vmware" + p.config.PackerBuilderType = "iso" + + for i, expectedValue := range expected { + p.config.Vars = userEnvVarTests[i] + flattenedEnvVars = p.createEnvVarFileContent() + if flattenedEnvVars != expectedValue { + t.Fatalf("expected flattened env vars to be: %s, got %s.", expectedValue, flattenedEnvVars) + } + } +} + func TestProvisioner_RemoteFolderSetSuccessfully(t *testing.T) { config := testConfig() From ab13c73277285dee8bd3a7b22b4fceced4c912c8 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Thu, 30 Aug 2018 11:02:56 -0700 Subject: [PATCH 4/4] make varfile name unique and make sure to remove it from guest system if cleanup is true. --- provisioner/shell/provisioner.go | 58 ++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index ad68d1739..3e431c390 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -259,7 +259,8 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { if !p.config.Binary { r = &UnixReader{Reader: r} } - remoteVFName := fmt.Sprintf("%s/%s", p.config.RemoteFolder, "varfile") + remoteVFName := fmt.Sprintf("%s/%s", p.config.RemoteFolder, + fmt.Sprintf("varfile_%d.sh", rand.Intn(9999))) if err := comm.Upload(remoteVFName, r, nil); err != nil { return fmt.Errorf("Error uploading envVarFile: %s", err) } @@ -359,30 +360,13 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { // Delete the temporary file we created. We retry this a few times // since if the above rebooted we have to wait until the reboot // completes. - err = p.retryable(func() error { - cmd = &packer.RemoteCmd{ - Command: fmt.Sprintf("rm -f %s", p.config.RemotePath), - } - if err := comm.Start(cmd); err != nil { - return fmt.Errorf( - "Error removing temporary script at %s: %s", - p.config.RemotePath, err) - } - cmd.Wait() - // treat disconnects as retryable by returning an error - if cmd.ExitStatus == packer.CmdDisconnect { - return fmt.Errorf("Disconnect while removing temporary script.") - } - return nil - }) + err = p.cleanupRemoteFile(p.config.RemotePath, comm) if err != nil { return err } - - if cmd.ExitStatus != 0 { - return fmt.Errorf( - "Error removing temporary script at %s!", - p.config.RemotePath) + err = p.cleanupRemoteFile(p.config.envVarFile, comm) + if err != nil { + return err } } } @@ -390,6 +374,36 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { return nil } +func (p *Provisioner) cleanupRemoteFile(path string, comm packer.Communicator) error { + err := p.retryable(func() error { + cmd := &packer.RemoteCmd{ + Command: fmt.Sprintf("rm -f %s", path), + } + if err := comm.Start(cmd); err != nil { + return fmt.Errorf( + "Error removing temporary script at %s: %s", + path, err) + } + cmd.Wait() + // treat disconnects as retryable by returning an error + if cmd.ExitStatus == packer.CmdDisconnect { + return fmt.Errorf("Disconnect while removing temporary script.") + } + if cmd.ExitStatus != 0 { + return fmt.Errorf( + "Error removing temporary script at %s!", + path) + } + return nil + }) + + if err != nil { + return err + } + + return nil +} + func (p *Provisioner) Cancel() { // Just hard quit. It isn't a big deal if what we're doing keeps // running on the other side.