Add packer fmt command (#10225)

* Add packer fmt command

This change adds a new command that allows users to format one or more
HCL2 Packer configuration template files.

Related to: #9174

* command/fmt: Add check flag

Packer's fmt command now supports a check flag that will output the name
of any file that would be changed by the HCL2 formatting engine. The
check flag is mutually exclusive with the write flag and will only check
if formatting is needed.

The update write flag will now overwrite the source files with the newly
formatted HCL2 source unless the `-write=false` or `-check` is passed at
the command line.

* Returns a diagnostic error if Format is unable to show a diff - equivalent to `terraform fmt`
* Updates testing to run against #Format and not the private methods of the HCL2Formatter; fixes ShowDiff test failure on Windows
* Updates comments for exported functions

* Add docs for fmt command

command/cli.go

@@ -137,8 +137,22 @@ func (va *HCL2UpgradeArgs) AddFlagSets(flags *flag.FlagSet) {
 	va.MetaArgs.AddFlagSets(flags)
 }
 
-// HCL2UpgradeArgs represents a parsed cli line for a `packer build`
+// HCL2UpgradeArgs represents a parsed cli line for a `packer hcl2_upgrade`
 type HCL2UpgradeArgs struct {
 	MetaArgs
 	OutputFile string
 }
+
+func (va *FormatArgs) AddFlagSets(flags *flag.FlagSet) {
+	flags.BoolVar(&va.Check, "check", false, "check if the input is formatted")
+	flags.BoolVar(&va.Diff, "diff", false, "display the diff of formatting changes")
+	flags.BoolVar(&va.Write, "write", true, "overwrite source files instead of writing to stdout")
+
+	va.MetaArgs.AddFlagSets(flags)
+}
+
+// FormatArgs represents a parsed cli line for `packer fmt`
+type FormatArgs struct {
+	MetaArgs
+	Check, Diff, Write bool
+}

command/fmt.go

@@ -0,0 +1,109 @@
+package command
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	hclutils "github.com/hashicorp/packer/hcl2template"
+	"github.com/posener/complete"
+)
+
+type FormatCommand struct {
+	Meta
+}
+
+func (c *FormatCommand) Run(args []string) int {
+	ctx := context.Background()
+	cfg, ret := c.ParseArgs(args)
+	if ret != 0 {
+		return ret
+	}
+
+	return c.RunContext(ctx, cfg)
+}
+
+func (c *FormatCommand) ParseArgs(args []string) (*FormatArgs, int) {
+	var cfg FormatArgs
+	flags := c.Meta.FlagSet("format", FlagSetNone)
+	flags.Usage = func() { c.Ui.Say(c.Help()) }
+	cfg.AddFlagSets(flags)
+	if err := flags.Parse(args); err != nil {
+		return &cfg, 1
+	}
+
+	args = flags.Args()
+	if len(args) != 1 {
+		flags.Usage()
+		return &cfg, 1
+	}
+
+	cfg.Path = args[0]
+	return &cfg, 0
+}
+
+func (c *FormatCommand) RunContext(ctx context.Context, cla *FormatArgs) int {
+	if cla.Check {
+		cla.Write = false
+	}
+
+	formatter := hclutils.HCL2Formatter{
+		ShowDiff: cla.Diff,
+		Write:    cla.Write,
+		Output:   os.Stdout,
+	}
+
+	bytesModified, diags := formatter.Format(cla.Path)
+	ret := writeDiags(c.Ui, nil, diags)
+	if ret != 0 {
+		return ret
+	}
+
+	if cla.Check && bytesModified > 0 {
+		// exit code taken from `terraform fmt` command
+		return 3
+	}
+
+	return 0
+}
+
+func (*FormatCommand) Help() string {
+	helpText := `
+Usage: packer fmt [options] [TEMPLATE]
+
+  Rewrites all Packer configuration files to a canonical format. Both
+  configuration files (.pkr.hcl) and variable files (.pkrvars) are updated.
+  JSON files (.json) are not modified.
+
+  If TEMPATE is "." the current directory will be used. The given content must
+  be in Packer's HCL2 configuration language; JSON is not supported.
+
+Options:
+  -check        Check if the input is formatted. Exit status will be 0 if all
+                 input is properly formatted and non-zero otherwise.
+
+  -diff         Display diffs of formatting change
+
+  -write=false  Don't write to source files
+                (always disabled if using -check)
+
+`
+
+	return strings.TrimSpace(helpText)
+}
+
+func (*FormatCommand) Synopsis() string {
+	return "Rewrites HCL2 config files to canonical format"
+}
+
+func (*FormatCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictNothing
+}
+
+func (*FormatCommand) AutocompleteFlags() complete.Flags {
+	return complete.Flags{
+		"-check": complete.PredictNothing,
+		"-diff":  complete.PredictNothing,
+		"-write": complete.PredictNothing,
+	}
+}

commands.go

@@ -18,9 +18,7 @@ const OutputPrefix = "o:"
 func init() {
 	Commands = map[string]cli.CommandFactory{
 		"build": func() (cli.Command, error) {
-			return &command.BuildCommand{
-				Meta: *CommandMeta,
-			}, nil
+			return &command.BuildCommand{Meta: *CommandMeta}, nil
 		},
 		"console": func() (cli.Command, error) {
 			return &command.ConsoleCommand{
@@ -34,12 +32,30 @@ func init() {
 			}, nil
 		},
 
+		"fmt": func() (cli.Command, error) {
+			return &command.FormatCommand{
+				Meta: *CommandMeta,
+			}, nil
+		},
+
+		"hcl2_upgrade": func() (cli.Command, error) {
+			return &command.HCL2UpgradeCommand{
+				Meta: *CommandMeta,
+			}, nil
+		},
+
 		"inspect": func() (cli.Command, error) {
 			return &command.InspectCommand{
 				Meta: *CommandMeta,
 			}, nil
 		},
 
+		"plugin": func() (cli.Command, error) {
+			return &command.PluginCommand{
+				Meta: *CommandMeta,
+			}, nil
+		},
+
 		"validate": func() (cli.Command, error) {
 			return &command.ValidateCommand{
 				Meta: *CommandMeta,
@@ -52,17 +68,5 @@ func init() {
 				CheckFunc: commandVersionCheck,
 			}, nil
 		},
-
-		"plugin": func() (cli.Command, error) {
-			return &command.PluginCommand{
-				Meta: *CommandMeta,
-			}, nil
-		},
-
-		"hcl2_upgrade": func() (cli.Command, error) {
-			return &command.HCL2UpgradeCommand{
-				Meta: *CommandMeta,
-			}, nil
-		},
 	}
 }

hcl2template/formatter.go

@@ -0,0 +1,151 @@
+package hcl2template
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclparse"
+	"github.com/hashicorp/hcl/v2/hclwrite"
+)
+
+type HCL2Formatter struct {
+	ShowDiff, Write bool
+	Output          io.Writer
+	parser          *hclparse.Parser
+}
+
+// NewHCL2Formatter creates a new formatter, ready to format configuration files.
+func NewHCL2Formatter() *HCL2Formatter {
+	return &HCL2Formatter{
+		parser: hclparse.NewParser(),
+	}
+}
+
+// Format all HCL2 files in path and return the total bytes formatted.
+// If any error is encountered, zero bytes will be returned.
+//
+// Path can be a directory or a file.
+func (f *HCL2Formatter) Format(path string) (int, hcl.Diagnostics) {
+	hclFiles, _, diags := GetHCL2Files(path, hcl2FileExt, hcl2JsonFileExt)
+	if diags.HasErrors() {
+		return 0, diags
+	}
+
+	hclVarFiles, _, diags := GetHCL2Files(path, hcl2VarFileExt, hcl2VarJsonFileExt)
+	if diags.HasErrors() {
+		return 0, diags
+	}
+
+	allHclFiles := append(hclFiles, hclVarFiles...)
+
+	if len(allHclFiles) == 0 {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  fmt.Sprintf("Cannot tell whether %s contains HCL2 configuration data", path),
+		})
+
+		return 0, diags
+	}
+
+	if f.parser == nil {
+		f.parser = hclparse.NewParser()
+	}
+
+	var bytesModified int
+	for _, fn := range allHclFiles {
+		data, err := f.processFile(fn)
+		if err != nil {
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  fmt.Sprintf("encountered an error while formatting %s", fn),
+				Detail:   err.Error(),
+			})
+		}
+		bytesModified += len(data)
+	}
+
+	return bytesModified, diags
+}
+
+// processFile formats the source contents of filename and return the formatted data.
+// overwriting the contents of the original when the f.Write is true; a diff of the changes
+// will be outputted if f.ShowDiff is true.
+func (f *HCL2Formatter) processFile(filename string) ([]byte, error) {
+	if f.Output == nil {
+		f.Output = os.Stdout
+	}
+
+	in, err := os.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open %s: %s", filename, err)
+	}
+
+	inSrc, err := ioutil.ReadAll(in)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %s: %s", filename, err)
+	}
+
+	_, diags := f.parser.ParseHCL(inSrc, filename)
+	if diags.HasErrors() {
+		return nil, fmt.Errorf("failed to parse HCL %s", filename)
+	}
+
+	outSrc := hclwrite.Format(inSrc)
+
+	if bytes.Equal(inSrc, outSrc) {
+		return nil, nil
+	}
+
+	s := []byte(fmt.Sprintf("%s\n", filename))
+	_, _ = f.Output.Write(s)
+
+	if f.Write {
+		if err := ioutil.WriteFile(filename, outSrc, 0644); err != nil {
+			return nil, err
+		}
+	}
+
+	if f.ShowDiff {
+		diff, err := bytesDiff(inSrc, outSrc, filename)
+		if err != nil {
+			return outSrc, fmt.Errorf("failed to generate diff for %s: %s", filename, err)
+		}
+		_, _ = f.Output.Write(diff)
+	}
+
+	return outSrc, nil
+}
+
+// bytesDiff returns the unified diff of b1 and b2
+// Shamelessly copied from Terraform's fmt command.
+func bytesDiff(b1, b2 []byte, path string) (data []byte, err error) {
+	f1, err := ioutil.TempFile("", "")
+	if err != nil {
+		return
+	}
+	defer os.Remove(f1.Name())
+	defer f1.Close()
+
+	f2, err := ioutil.TempFile("", "")
+	if err != nil {
+		return
+	}
+	defer os.Remove(f2.Name())
+	defer f2.Close()
+
+	_, _ = f1.Write(b1)
+	_, _ = f2.Write(b2)
+
+	data, err = exec.Command("diff", "--label=old/"+path, "--label=new/"+path, "-u", f1.Name(), f2.Name()).CombinedOutput()
+	if len(data) > 0 {
+		// diff exits with a non-zero status when the files don't match.
+		// Ignore that failure as long as we get output.
+		err = nil
+	}
+	return
+}

hcl2template/formatter_test.go

@@ -0,0 +1,110 @@
+package hcl2template
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestHCL2Formatter_Format(t *testing.T) {
+	tt := []struct {
+		Name           string
+		Path           string
+		FormatExpected bool
+	}{
+		{Name: "Unformatted file", Path: "testdata/format/unformatted.pkr.hcl", FormatExpected: true},
+		{Name: "Formatted file", Path: "testdata/format/formatted.pkr.hcl"},
+		{Name: "Directory", Path: "testdata/format", FormatExpected: true},
+	}
+
+	for _, tc := range tt {
+		tc := tc
+		var buf bytes.Buffer
+		f := NewHCL2Formatter()
+		f.Output = &buf
+		_, diags := f.Format(tc.Path)
+		if diags.HasErrors() {
+			t.Fatalf("the call to Format failed unexpectedly %s", diags.Error())
+		}
+
+		if buf.String() != "" && tc.FormatExpected == false {
+			t.Errorf("Format(%q) should contain the name of the formatted file(s), but got %q", tc.Path, buf.String())
+		}
+
+	}
+}
+
+func TestHCL2Formatter_Format_Write(t *testing.T) {
+
+	var buf bytes.Buffer
+	f := NewHCL2Formatter()
+	f.Output = &buf
+	f.Write = true
+
+	unformattedData, err := ioutil.ReadFile("testdata/format/unformatted.pkr.hcl")
+	if err != nil {
+		t.Fatalf("failed to open the unformatted fixture %s", err)
+	}
+
+	tf, err := ioutil.TempFile("", "*.pkr.hcl")
+	if err != nil {
+		t.Fatalf("failed to create tempfile for test %s", err)
+	}
+	defer os.Remove(tf.Name())
+
+	_, _ = tf.Write(unformattedData)
+	tf.Close()
+
+	_, diags := f.Format(tf.Name())
+	if diags.HasErrors() {
+		t.Fatalf("the call to Format failed unexpectedly %s", diags.Error())
+	}
+
+	//lets re-read the tempfile which should now be formatted
+	data, err := ioutil.ReadFile(tf.Name())
+	if err != nil {
+		t.Fatalf("failed to open the newly formatted fixture %s", err)
+	}
+
+	formattedData, err := ioutil.ReadFile("testdata/format/formatted.pkr.hcl")
+	if err != nil {
+		t.Fatalf("failed to open the formatted fixture %s", err)
+	}
+
+	if diff := cmp.Diff(string(data), string(formattedData)); diff != "" {
+		t.Errorf("Unexpected format output %s", diff)
+	}
+}
+
+func TestHCL2Formatter_Format_ShowDiff(t *testing.T) {
+
+	if _, err := exec.LookPath("diff"); err != nil {
+		t.Skip("Skipping test because diff is not in the executable PATH")
+	}
+
+	var buf bytes.Buffer
+	f := HCL2Formatter{
+		Output:   &buf,
+		ShowDiff: true,
+	}
+
+	_, diags := f.Format("testdata/format/unformatted.pkr.hcl")
+	if diags.HasErrors() {
+		t.Fatalf("the call to Format failed unexpectedly %s", diags.Error())
+	}
+
+	diffHeader := `
+--- old/testdata/format/unformatted.pkr.hcl
++++ new/testdata/format/unformatted.pkr.hcl
+@@ -1,149 +1,149 @@
+`
+	if !strings.Contains(buf.String(), diffHeader) {
+		t.Errorf("expected buf to contain a file diff, but instead we got %s", buf.String())
+	}
+
+}

hcl2template/testdata/format/formatted.pkr.hcl

@@ -0,0 +1,149 @@
+
+// starts resources to provision them.
+build {
+  sources = [
+    "source.amazon-ebs.ubuntu-1604",
+    "source.virtualbox-iso.ubuntu-1204",
+  ]
+
+  provisioner "shell" {
+    string   = coalesce(null, "", "string")
+    int      = "${41 + 1}"
+    int64    = "${42 + 1}"
+    bool     = "true"
+    trilean  = true
+    duration = "${9 + 1}s"
+    map_string_string = {
+      a = "b"
+      c = "d"
+    }
+    slice_string = [
+      "a",
+      "b",
+      "c",
+    ]
+    slice_slice_string = [
+      ["a", "b"],
+      ["c", "d"]
+    ]
+
+    nested {
+      string   = "string"
+      int      = 42
+      int64    = 43
+      bool     = true
+      trilean  = true
+      duration = "10s"
+      map_string_string = {
+        a = "b"
+        c = "d"
+      }
+      slice_string = [
+        "a",
+        "b",
+        "c",
+      ]
+      slice_slice_string = [
+        ["a", "b"],
+        ["c", "d"]
+      ]
+    }
+
+    nested_slice {
+    }
+  }
+
+  provisioner "file" {
+    string   = "string"
+    int      = 42
+    int64    = 43
+    bool     = true
+    trilean  = true
+    duration = "10s"
+    map_string_string = {
+      a = "b"
+      c = "d"
+    }
+    slice_string = [
+      "a",
+      "b",
+      "c",
+    ]
+    slice_slice_string = [
+      ["a", "b"],
+      ["c", "d"]
+    ]
+
+    nested {
+      string   = "string"
+      int      = 42
+      int64    = 43
+      bool     = true
+      trilean  = true
+      duration = "10s"
+      map_string_string = {
+        a = "b"
+        c = "d"
+      }
+      slice_string = [
+        "a",
+        "b",
+        "c",
+      ]
+      slice_slice_string = [
+        ["a", "b"],
+        ["c", "d"]
+      ]
+    }
+
+    nested_slice {
+    }
+  }
+
+  post-processor "amazon-import" {
+    string   = "string"
+    int      = 42
+    int64    = 43
+    bool     = true
+    trilean  = true
+    duration = "10s"
+    map_string_string = {
+      a = "b"
+      c = "d"
+    }
+    slice_string = [
+      "a",
+      "b",
+      "c",
+    ]
+    slice_slice_string = [
+      ["a", "b"],
+      ["c", "d"]
+    ]
+
+    nested {
+      string   = "string"
+      int      = 42
+      int64    = 43
+      bool     = true
+      trilean  = true
+      duration = "10s"
+      map_string_string = {
+        a = "b"
+        c = "d"
+      }
+      slice_string = [
+        "a",
+        "b",
+        "c",
+      ]
+      slice_slice_string = [
+        ["a", "b"],
+        ["c", "d"]
+      ]
+    }
+
+    nested_slice {
+    }
+  }
+}

hcl2template/testdata/format/unformatted.pkr.hcl

@@ -0,0 +1,149 @@
+
+// starts resources to provision them.
+build {
+    sources = [
+        "source.amazon-ebs.ubuntu-1604",
+        "source.virtualbox-iso.ubuntu-1204",
+    ]
+
+    provisioner "shell" {
+        string  = coalesce(null, "", "string")
+        int     = "${41 + 1}"
+        int64   = "${42 + 1}"
+        bool    = "true"
+        trilean = true
+        duration = "${9 + 1}s"
+        map_string_string = {
+            a = "b"
+            c = "d"
+        }
+        slice_string = [
+            "a",
+            "b",
+            "c",
+        ]
+        slice_slice_string = [
+            ["a","b"],
+            ["c","d"]
+        ]
+
+        nested {
+            string  = "string"
+            int     = 42
+            int64   = 43
+            bool    = true
+            trilean = true
+            duration = "10s"
+            map_string_string = {
+                a = "b"
+                c = "d"
+            }
+            slice_string = [
+                "a",
+                "b",
+                "c",
+            ]
+            slice_slice_string = [
+                ["a","b"],
+                ["c","d"]
+            ]
+        }
+
+        nested_slice {
+        }
+    }
+
+    provisioner "file" {
+        string  = "string"
+        int     = 42
+        int64   = 43
+        bool    = true
+        trilean = true
+        duration          = "10s"
+        map_string_string = {
+            a = "b"
+            c = "d"
+        }
+        slice_string = [
+            "a",
+            "b",
+            "c",
+        ]
+        slice_slice_string = [
+            ["a","b"],
+            ["c","d"]
+        ]
+
+        nested {
+            string   = "string"
+            int      = 42
+            int64    = 43
+            bool     = true
+            trilean  = true
+            duration = "10s"
+            map_string_string = {
+                a = "b"
+                c = "d"
+            }
+            slice_string = [
+                "a",
+                "b",
+                "c",
+            ]
+            slice_slice_string = [
+                ["a","b"],
+                ["c","d"]
+            ]
+        }
+
+        nested_slice {
+        }
+    }
+
+    post-processor "amazon-import" {
+        string   = "string"
+        int      = 42
+        int64    = 43
+        bool     = true
+        trilean  = true
+        duration = "10s"
+        map_string_string = {
+            a = "b"
+            c = "d"
+        }
+        slice_string = [
+            "a",
+            "b",
+            "c",
+        ]
+        slice_slice_string = [
+            ["a","b"],
+            ["c","d"]
+        ]
+
+        nested {
+            string   = "string"
+            int      = 42
+            int64    = 43
+            bool     = true
+            trilean  = true
+            duration = "10s"
+            map_string_string = {
+                a = "b"
+                c = "d"
+            }
+            slice_string = [
+                "a",
+                "b",
+                "c",
+            ]
+            slice_slice_string = [
+                ["a","b"],
+                ["c","d"]
+            ]
+        }
+
+        nested_slice {
+        }
+    }
+}

website/data/docs-navigation.js

@@ -170,7 +170,7 @@ export default [
   'terminology',
   {
     category: 'commands',
-    content: ['build', 'console', 'fix', 'inspect', 'validate', 'hcl2_upgrade'],
+    content: ['build', 'console', 'fix', 'fmt', 'inspect', 'validate', 'hcl2_upgrade'],
   },
   {
     category: 'templates',

website/pages/docs/commands/fmt.mdx

@@ -0,0 +1,46 @@
+---
+description: |
+  The `packer fmt` Packer command is used to format HCL2
+  configuration files to a canonical format and style.
+layout: docs
+page_title: packer fmt - Commands
+sidebar_title: <tt>fmt</tt>
+---
+
+# `fmt` Command
+
+The `packer fmt` Packer command is used to format HCL2 configuration files to
+a canonical format and style. JSON files (.json) are not modified. This command
+applies a subset of HCL language style conventions, along with other minor
+adjustments for readability.
+
+`packer fmt` will display the name of the configuration file(s) that need formatting,
+and write any formatted changes back to the original configuration file(s).
+
+Example usage:
+
+Check if configuration file(s) need to be formatted, but don't write the changes.
+
+```shell-session
+$ packer fmt -check .
+my-template.json
+
+```
+
+Format a configuration file, writing the changes back to the original file.
+
+```shell-session
+$ packer fmt my-template.json
+my-template.json
+
+```
+
+## Options
+
+- `-check` - Checks if the input is formatted. Exit status will be 0 if all
+input is properly formatted and non-zero otherwise.
+
+- `-diff` - Display diffs of any formatting change
+
+- `-write=false` - Don't write formatting changes to source files
+(always disabled if using -check)