Merge pull request #6610 from hashicorp/filter_logs

Filter logs

builder/alicloud/ecs/builder.go

@@ -67,7 +67,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 		return nil, errs
 	}
 
-	log.Println(common.ScrubConfig(b.config, b.config.AlicloudAccessKey, b.config.AlicloudSecretKey))
+	packer.LogSecretFilter.Set(b.config.AlicloudAccessKey, b.config.AlicloudSecretKey)
+	log.Println(b.config)
 	return nil, nil
 }
 

builder/amazon/chroot/builder.go

@@ -176,7 +176,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 		return warns, errs
 	}
 
-	log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
+	packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
+	log.Println(b.config)
 	return warns, nil
 }
 

builder/amazon/common/step_get_password.go

@@ -95,7 +95,9 @@ WaitLoop:
 			"Password (since debug is enabled): %s", s.Comm.WinRMPassword))
 	}
 	// store so that we can access this later during provisioning
+
 	commonhelper.SetSharedState("winrm_password", s.Comm.WinRMPassword, s.BuildName)
+	packer.LogSecretFilter.Set(s.Comm.WinRMPassword)
 
 	return multistep.ActionContinue
 }

builder/amazon/ebs/builder.go

@@ -81,7 +81,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 		return nil, errs
 	}
 
-	log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
+	packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
+	log.Println(b.config)
 	return nil, nil
 }
 
@@ -260,7 +261,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	// Run!
 	b.runner = common.NewRunner(steps, b.config.PackerConfig, ui)
 	b.runner.Run(state)
-
 	// If there was an error, return that
 	if rawErr, ok := state.GetOk("error"); ok {
 		return nil, rawErr.(error)

builder/amazon/ebssurrogate/builder.go

@@ -96,7 +96,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 		return nil, errs
 	}
 
-	log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
+	packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
+	log.Println(b.config)
 	return nil, nil
 }
 

builder/amazon/ebsvolume/builder.go

@@ -81,7 +81,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 		return nil, errs
 	}
 
-	log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
+	packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
+	log.Println(b.config)
 	return nil, nil
 }
 

builder/amazon/instance/builder.go

@@ -166,8 +166,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 	if errs != nil && len(errs.Errors) > 0 {
 		return nil, errs
 	}
-
+	packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
-	log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
+	log.Println(b.config)
 	return nil, nil
 }
 

builder/azure/arm/config.go

@@ -363,6 +363,7 @@ func setRuntimeValues(c *Config) {
 	c.tmpAdminPassword = tempName.AdminPassword
 	// store so that we can access this later during provisioning
 	commonhelper.SetSharedState("winrm_password", c.tmpAdminPassword, c.PackerConfig.PackerBuildName)
+	packer.LogSecretFilter.Set(c.tmpAdminPassword)
 
 	c.tmpCertificatePassword = tempName.CertificatePassword
 	if c.TempComputeName == "" {

builder/azure/arm/step_save_winrm_password.go

@@ -5,6 +5,7 @@ import (
 
 	commonhelper "github.com/hashicorp/packer/helper/common"
 	"github.com/hashicorp/packer/helper/multistep"
+	"github.com/hashicorp/packer/packer"
 )
 
 type StepSaveWinRMPassword struct {
@@ -15,6 +16,7 @@ type StepSaveWinRMPassword struct {
 func (s *StepSaveWinRMPassword) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
 	// store so that we can access this later during provisioning
 	commonhelper.SetSharedState("winrm_password", s.Password, s.BuildName)
+	packer.LogSecretFilter.Set(s.Password)
 	return multistep.ActionContinue
 }
 

builder/digitalocean/config.go

@@ -138,6 +138,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 		return nil, nil, errs
 	}
 
-	common.ScrubConfig(c, c.APIToken)
+	packer.LogSecretFilter.Set(c.APIToken)
 	return c, nil, nil
 }

builder/googlecompute/step_create_windows_password.go

@@ -33,6 +33,7 @@ func (s *StepCreateWindowsPassword) Run(_ context.Context, state multistep.State
 
 	if c.Comm.WinRMPassword != "" {
 		state.Put("winrm_password", c.Comm.WinRMPassword)
+		packer.LogSecretFilter.Set(c.Comm.WinRMPassword)
 		return multistep.ActionContinue
 	}
 
@@ -114,6 +115,7 @@ func (s *StepCreateWindowsPassword) Run(_ context.Context, state multistep.State
 
 	state.Put("winrm_password", data.password)
 	commonhelper.SetSharedState("winrm_password", data.password, c.PackerConfig.PackerBuildName)
+	packer.LogSecretFilter.Set(data.password)
 
 	return multistep.ActionContinue
 }

builder/oneandone/config.go

@@ -109,7 +109,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 	if errs != nil && len(errs.Errors) > 0 {
 		return nil, nil, errs
 	}
-	common.ScrubConfig(c, c.Token)
+	packer.LogSecretFilter.Set(c.Token)
-
 	return &c, nil, nil
 }

builder/openstack/builder.go

@@ -57,7 +57,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 		b.config.InstanceName = b.config.ImageName
 	}
 
-	log.Println(common.ScrubConfig(b.config, b.config.Password))
+	packer.LogSecretFilter.Set(b.config.Password)
+	log.Println(b.config)
 	return nil, nil
 }
 

builder/oracle/oci/step_get_default_credentials.go

@@ -53,7 +53,7 @@ func (s *stepGetDefaultCredentials) Run(ctx context.Context, state multistep.Sta
 
 	// store so that we can access this later during provisioning
 	commonhelper.SetSharedState("winrm_password", s.Comm.WinRMPassword, s.BuildName)
-
+	packer.LogSecretFilter.Set(s.Comm.WinRMPassword)
 	return multistep.ActionContinue
 }
 

builder/profitbricks/config.go

@@ -122,7 +122,7 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 	if errs != nil && len(errs.Errors) > 0 {
 		return nil, nil, errs
 	}
-	common.ScrubConfig(c, c.PBUsername)
+	packer.LogSecretFilter.Set(c.PBUsername)
 
 	return &c, nil, nil
 }

builder/scaleway/config.go

@@ -119,6 +119,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 		return nil, nil, errs
 	}
 
-	common.ScrubConfig(c, c.Token)
+	packer.LogSecretFilter.Set(c.Token)
 	return c, nil, nil
 }

common/config.go

@@ -20,19 +20,6 @@ const PackerKeyEnv = "PACKER_KEY_INTERVAL"
 // shorter delay (e.g. 10ms) can be used on a workstation. See PackerKeyEnv.
 const PackerKeyDefault = 100 * time.Millisecond
 
-// ScrubConfig is a helper that returns a string representation of
-// any struct with the given values stripped out.
-func ScrubConfig(target interface{}, values ...string) string {
-	conf := fmt.Sprintf("Config: %+v", target)
-	for _, value := range values {
-		if value == "" {
-			continue
-		}
-		conf = strings.Replace(conf, value, "<Filtered>", -1)
-	}
-	return conf
-}
-
 // ChooseString returns the first non-empty value.
 func ChooseString(vals ...string) string {
 	for _, el := range vals {

common/config_test.go

@@ -310,20 +310,3 @@ func TestFileExistsLocally(t *testing.T) {
 		}
 	}
 }
-
-func TestScrubConfig(t *testing.T) {
-	type Inner struct {
-		Baz string
-	}
-	type Local struct {
-		Foo string
-		Bar string
-		Inner
-	}
-	c := Local{"foo", "bar", Inner{"bar"}}
-	expect := "Config: {Foo:foo Bar:<Filtered> Inner:{Baz:<Filtered>}}"
-	conf := ScrubConfig(c, c.Bar)
-	if conf != expect {
-		t.Fatalf("got %s, expected %s", conf, expect)
-	}
-}

common/packer_config.go

@@ -10,4 +10,5 @@ type PackerConfig struct {
 	PackerForce         bool              `mapstructure:"packer_force"`
 	PackerOnError       string            `mapstructure:"packer_on_error"`
 	PackerUserVars      map[string]string `mapstructure:"packer_user_variables"`
+	PackerSensitiveVars []string          `mapstructure:"packer_sensitive_variables"`
 }

common/shell-local/run.go

@@ -70,12 +70,7 @@ func Run(ui packer.Ui, config *Config) (bool, error) {
 		// buffers and for reading the final exit status.
 		flattenedCmd := strings.Join(interpolatedCmds, " ")
 		cmd := &packer.RemoteCmd{Command: flattenedCmd}
-		sanitized := flattenedCmd
+		log.Printf("[INFO] (shell-local): starting local command: %s", flattenedCmd)
-		if len(getWinRMPassword(config.PackerBuildName)) > 0 {
-			sanitized = strings.Replace(flattenedCmd,
-				getWinRMPassword(config.PackerBuildName), "*****", -1)
-		}
-		log.Printf("[INFO] (shell-local): starting local command: %s", sanitized)
 		if err := cmd.StartWithUi(comm, ui); err != nil {
 			return false, fmt.Errorf(
 				"Error executing script: %s\n\n"+
@@ -204,5 +199,6 @@ func createFlattenedEnvVars(config *Config) (string, error) {
 
 func getWinRMPassword(buildName string) string {
 	winRMPass, _ := commonhelper.RetrieveSharedState("winrm_password", buildName)
+	packer.LogSecretFilter.Set(winRMPass)
 	return winRMPass
 }

helper/config/decode.go

@@ -112,6 +112,7 @@ func DetectContext(raws ...interface{}) (*interpolate.Context, error) {
 		BuildType     string            `mapstructure:"packer_builder_type"`
 		TemplatePath  string            `mapstructure:"packer_template_path"`
 		Vars          map[string]string `mapstructure:"packer_user_variables"`
+		SensitiveVars []string          `mapstructure:"packer_sensitive_variables"`
 	}
 
 	for _, r := range raws {
@@ -125,6 +126,7 @@ func DetectContext(raws ...interface{}) (*interpolate.Context, error) {
 		BuildType:          s.BuildType,
 		TemplatePath:       s.TemplatePath,
 		UserVariables:      s.Vars,
+		SensitiveVariables: s.SensitiveVars,
 	}, nil
 }
 

main.go

@@ -55,6 +55,10 @@ func realMain() int {
 			logWriter = ioutil.Discard
 		}
 
+		packer.LogSecretFilter.SetOutput(logWriter)
+
+		//packer.LogSecrets.
+
 		// Disable logging here
 		log.SetOutput(ioutil.Discard)
 
@@ -87,7 +91,7 @@ func realMain() int {
 
 		// Create the configuration for panicwrap and wrap our executable
 		wrapConfig.Handler = panicHandler(logTempFile)
-		wrapConfig.Writer = io.MultiWriter(logTempFile, logWriter)
+		wrapConfig.Writer = io.MultiWriter(logTempFile, &packer.LogSecretFilter)
 		wrapConfig.Stdout = outW
 		wrapConfig.DetectDuration = 500 * time.Millisecond
 		wrapConfig.ForwardSignals = []os.Signal{syscall.SIGTERM}
@@ -125,7 +129,8 @@ func wrappedMain() int {
 		runtime.GOMAXPROCS(runtime.NumCPU())
 	}
 
-	log.SetOutput(os.Stderr)
+	packer.LogSecretFilter.SetOutput(os.Stderr)
+	log.SetOutput(&packer.LogSecretFilter)
 
 	log.Printf("[INFO] Packer version: %s", version.FormattedVersion())
 	log.Printf("Packer Target OS/Arch: %s %s", runtime.GOOS, runtime.GOARCH)

packer/core.go

@@ -19,6 +19,7 @@ type Core struct {
 	variables  map[string]string
 	builds     map[string]*template.Builder
 	version    string
+	secrets    []string
 }
 
 // CoreConfig is the structure for initializing a new Core. Once a CoreConfig
@@ -27,6 +28,7 @@ type CoreConfig struct {
 	Components         ComponentFinder
 	Template           *template.Template
 	Variables          map[string]string
+	SensitiveVariables []string
 	Version            string
 }
 
@@ -60,12 +62,16 @@ func NewCore(c *CoreConfig) (*Core, error) {
 		variables:  c.Variables,
 		version:    c.Version,
 	}
+
 	if err := result.validate(); err != nil {
 		return nil, err
 	}
 	if err := result.init(); err != nil {
 		return nil, err
 	}
+	for _, secret := range result.secrets {
+		LogSecretFilter.Set(secret)
+	}
 
 	// Go through and interpolate all the build names. We should be able
 	// to do this at this point with the variables.
@@ -302,6 +308,16 @@ func (c *Core) init() error {
 		c.variables[k] = def
 	}
 
+	for _, v := range c.Template.SensitiveVariables {
+		def, err := interpolate.Render(v.Default, ctx)
+		if err != nil {
+			return fmt.Errorf(
+				"error interpolating default value for '%#v': %s",
+				v, err)
+		}
+		c.secrets = append(c.secrets, def)
+	}
+
 	// Interpolate the push configuration
 	if _, err := interpolate.RenderInterface(&c.Template.Push, c.Context()); err != nil {
 		return fmt.Errorf("Error interpolating 'push': %s", err)

packer/core_test.go

@@ -516,12 +516,67 @@ func TestCoreValidate(t *testing.T) {
 			Variables: tc.Vars,
 			Version:   "1.0.0",
 		})
+
 		if (err != nil) != tc.Err {
 			t.Fatalf("err: %s\n\n%s", tc.File, err)
 		}
 	}
 }
 
+func TestSensitiveVars(t *testing.T) {
+	cases := []struct {
+		File          string
+		Vars          map[string]string
+		SensitiveVars []string
+		Expected      string
+		Err           bool
+	}{
+		// hardcoded
+		{
+			"sensitive-variables.json",
+			map[string]string{"foo": "bar"},
+			[]string{"foo"},
+			"bar",
+			false,
+		},
+		// interpolated
+		{
+			"sensitive-variables.json",
+			map[string]string{"foo": "{{build_name}}"},
+			[]string{"foo"},
+			"test",
+			false,
+		},
+	}
+
+	for _, tc := range cases {
+		f, err := os.Open(fixtureDir(tc.File))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+
+		tpl, err := template.Parse(f)
+		f.Close()
+		if err != nil {
+			t.Fatalf("err: %s\n\n%s", tc.File, err)
+		}
+
+		_, err = NewCore(&CoreConfig{
+			Template:  tpl,
+			Variables: tc.Vars,
+			Version:   "1.0.0",
+		})
+
+		if (err != nil) != tc.Err {
+			t.Fatalf("err: %s\n\n%s", tc.File, err)
+		}
+		filtered := LogSecretFilter.get()
+		if filtered[0] != tc.Expected && len(filtered) != 1 {
+			t.Fatalf("not filtering sensitive vars; filtered is %#v", filtered)
+		}
+	}
+}
+
 func testComponentFinder() *ComponentFinder {
 	builderFactory := func(n string) (Builder, error) { return new(MockBuilder), nil }
 	ppFactory := func(n string) (PostProcessor, error) { return new(MockPostProcessor), nil }

packer/logs.go

@@ -0,0 +1,51 @@
+package packer
+
+import (
+	"bytes"
+	"io"
+	"sync"
+)
+
+type secretFilter struct {
+	s map[string]struct{}
+	m sync.Mutex
+	w io.Writer
+}
+
+func (l *secretFilter) Set(secrets ...string) {
+	l.m.Lock()
+	defer l.m.Unlock()
+	for _, s := range secrets {
+		l.s[s] = struct{}{}
+	}
+}
+
+func (l *secretFilter) SetOutput(output io.Writer) {
+	l.m.Lock()
+	defer l.m.Unlock()
+	l.w = output
+}
+
+func (l *secretFilter) Write(p []byte) (n int, err error) {
+	for s := range l.s {
+		if s != "" {
+			p = bytes.Replace(p, []byte(s), []byte("<sensitive>"), -1)
+		}
+	}
+	return l.w.Write(p)
+}
+
+func (l *secretFilter) get() (s []string) {
+	l.m.Lock()
+	defer l.m.Unlock()
+	for k := range l.s {
+		s = append(s, k)
+	}
+	return
+}
+
+var LogSecretFilter secretFilter
+
+func init() {
+	LogSecretFilter.s = make(map[string]struct{})
+}

packer/test-fixtures/sensitive-variables.json

@@ -0,0 +1,12 @@
+{
+    "variables": {
+        "foo": "bar"
+    },
+    "sensitive-variables": [
+    	"foo"
+    ],
+    "builders": [{
+        "type": "test",
+        "value": "{{build_name}}"
+    }]
+}

post-processor/alicloud-import/post-processor.go

@@ -113,7 +113,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 		return errs
 	}
 
-	log.Println(common.ScrubConfig(p.config, p.config.AlicloudAccessKey, p.config.AlicloudSecretKey))
+	packer.LogSecretFilter.Set(p.config.AlicloudAccessKey, p.config.AlicloudSecretKey)
+	log.Println(p.config)
 	return nil
 }
 

post-processor/amazon-import/post-processor.go

@@ -92,7 +92,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 		return errs
 	}
 
-	log.Println(common.ScrubConfig(p.config, p.config.AccessKey, p.config.SecretKey, p.config.Token))
+	packer.LogSecretFilter.Set(p.config.AccessKey, p.config.SecretKey, p.config.Token)
+	log.Println(p.config)
 	return nil
 }
 

provisioner/ansible/provisioner.go

@@ -555,6 +555,7 @@ func newSigner(privKeyFile string) (*signer, error) {
 
 func getWinRMPassword(buildName string) string {
 	winRMPass, _ := commonhelper.RetrieveSharedState("winrm_password", buildName)
+	packer.LogSecretFilter.Set(winRMPass)
 	return winRMPass
 }
 

provisioner/powershell/provisioner.go

@@ -481,6 +481,7 @@ func (p *Provisioner) createCommandTextNonPrivileged() (command string, err erro
 
 func getWinRMPassword(buildName string) string {
 	winRMPass, _ := commonhelper.RetrieveSharedState("winrm_password", buildName)
+	packer.LogSecretFilter.Set(winRMPass)
 	return winRMPass
 }
 

template/interpolate/i.go

@@ -18,6 +18,9 @@ type Context struct {
 	// "user" function reads from.
 	UserVariables map[string]string
 
+	// SensitiveVariables is a list of variables to sanitize.
+	SensitiveVariables []string
+
 	// EnableEnv enables the env function
 	EnableEnv bool
 

template/parse.go

@@ -28,6 +28,7 @@ type rawTemplate struct {
 	PostProcessors     []interface{} `mapstructure:"post-processors"`
 	Provisioners       []map[string]interface{}
 	Variables          map[string]interface{}
+	SensitiveVariables []string `mapstructure:"sensitive-variables"`
 
 	RawContents []byte
 }
@@ -60,6 +61,12 @@ func (r *rawTemplate) Template() (*Template, error) {
 			continue
 		}
 
+		for _, sVar := range r.SensitiveVariables {
+			if sVar == k {
+				result.SensitiveVariables = append(result.SensitiveVariables, &v)
+			}
+		}
+
 		result.Variables[k] = &v
 	}
 

template/template.go

@@ -19,6 +19,7 @@ type Template struct {
 	MinVersion  string
 
 	Variables          map[string]*Variable
+	SensitiveVariables []*Variable
 	Builders           map[string]*Builder
 	Provisioners       []*Provisioner
 	PostProcessors     [][]*PostProcessor

website/source/docs/templates/user-variables.html.md

@@ -206,6 +206,32 @@ Results in the following variables:
 | aws\_access\_key | foo   |
 | aws\_secret\_key | baz   |
 
+# Sensitive Variables
+
+If you use the environment to set a variable that is sensitive, you probably
+don't want that variable printed to the Packer logs. You can make sure that
+sensitive variables won't get printed to the logs by adding them to the
+"sensitive-variables" list within the Packer template:
+
+``` json
+{
+  "variables": {
+    "my_secret": "{{env `MY_SECRET`}}",
+    "not_a_secret": "plaintext",
+    "foo": "bar"
+  },
+
+  "sensitive-variables": ["my_secret", "foo"],
+  ...
+}
+```
+
+The above snippet of code will function exactly the same as if you did not set
+"sensitive-variables", except that the Packer UI and logs will replace all
+instances of "bar" and of whatever the value of "my_secret" is with
+`<sensitive>`. This allows you to be confident that you are not printing
+secrets in plaintext to our logs by accident.
+
 # Recipes
 
 ## Making a provisioner step conditional on the value of a variable