Merge pull request #7339 from devenney/feature/7322

Add functionality to marshal a Template to valid Packer JSON

template/parse.go

@@ -20,17 +20,40 @@ import (
 // This is what is decoded directly from the file, and then it is turned
 // into a Template object thereafter.
 type rawTemplate struct {
-	MinVersion  string `mapstructure:"min_packer_version"`
-	Description string
+	MinVersion  string `mapstructure:"min_packer_version" json:"min_packer_version,omitempty"`
+	Description string `json:"description,omitempty"`
 
-	Builders           []map[string]interface{}
-	Push               map[string]interface{}
-	PostProcessors     []interface{} `mapstructure:"post-processors"`
-	Provisioners       []map[string]interface{}
-	Variables          map[string]interface{}
-	SensitiveVariables []string `mapstructure:"sensitive-variables"`
+	Builders           []interface{}          `mapstructure:"builders" json:"builders,omitempty"`
+	Comments           []map[string]string    `json:"comments,omitempty"`
+	Push               map[string]interface{} `json:"push,omitempty"`
+	PostProcessors     []interface{}          `mapstructure:"post-processors" json:"post-processors,omitempty"`
+	Provisioners       []interface{}          `json:"provisioners,omitempty"`
+	Variables          map[string]interface{} `json:"variables,omitempty"`
+	SensitiveVariables []string               `mapstructure:"sensitive-variables" json:"sensitive-variables,omitempty"`
 
-	RawContents []byte
+	RawContents []byte `json:"-"`
+}
+
+// MarshalJSON conducts the necessary flattening of the rawTemplate struct
+// to provide valid Packer template JSON
+func (r *rawTemplate) MarshalJSON() ([]byte, error) {
+	// Avoid recursion
+	type rawTemplate_ rawTemplate
+	out, _ := json.Marshal(rawTemplate_(*r))
+
+	var m map[string]json.RawMessage
+	_ = json.Unmarshal(out, &m)
+
+	// Flatten Comments
+	delete(m, "comments")
+	for _, comment := range r.Comments {
+		for k, v := range comment {
+			out, _ = json.Marshal(v)
+			m[k] = out
+		}
+	}
+
+	return json.Marshal(m)
 }
 
 // Template returns the actual Template object built from this raw
@@ -44,12 +67,25 @@ func (r *rawTemplate) Template() (*Template, error) {
 	result.MinVersion = r.MinVersion
 	result.RawContents = r.RawContents
 
+	// Gather the comments
+	if len(r.Comments) > 0 {
+		result.Comments = make(map[string]string, len(r.Comments))
+
+		for _, c := range r.Comments {
+			for k, v := range c {
+				result.Comments[k] = v
+			}
+		}
+	}
+
 	// Gather the variables
 	if len(r.Variables) > 0 {
 		result.Variables = make(map[string]*Variable, len(r.Variables))
 	}
+
 	for k, rawV := range r.Variables {
 		var v Variable
+		v.Key = k
 
 		// Variable is required if the value is exactly nil
 		v.Required = rawV == nil
@@ -83,9 +119,11 @@ func (r *rawTemplate) Template() (*Template, error) {
 		}
 
 		// Set the raw configuration and delete any special keys
-		b.Config = rawB
+		b.Config = rawB.(map[string]interface{})
+
 		delete(b.Config, "name")
 		delete(b.Config, "type")
+
 		if len(b.Config) == 0 {
 			b.Config = nil
 		}
@@ -144,19 +182,22 @@ func (r *rawTemplate) Template() (*Template, error) {
 				continue
 			}
 
+			// Set the raw configuration and delete any special keys
+			pp.Config = c
+
 			// The name defaults to the type if it isn't set
 			if pp.Name == "" {
 				pp.Name = pp.Type
 			}
 
-			// Set the configuration
-			delete(c, "except")
-			delete(c, "only")
-			delete(c, "keep_input_artifact")
-			delete(c, "type")
-			delete(c, "name")
-			if len(c) > 0 {
-				pp.Config = c
+			delete(pp.Config, "except")
+			delete(pp.Config, "only")
+			delete(pp.Config, "keep_input_artifact")
+			delete(pp.Config, "type")
+			delete(pp.Config, "name")
+
+			if len(pp.Config) == 0 {
+				pp.Config = nil
 			}
 
 			pps = append(pps, &pp)
@@ -184,17 +225,19 @@ func (r *rawTemplate) Template() (*Template, error) {
 			continue
 		}
 
-		// Copy the configuration
-		delete(v, "except")
-		delete(v, "only")
-		delete(v, "override")
-		delete(v, "pause_before")
-		delete(v, "type")
-		if len(v) > 0 {
-			p.Config = v
+		// Set the raw configuration and delete any special keys
+		p.Config = v.(map[string]interface{})
+
+		delete(p.Config, "except")
+		delete(p.Config, "only")
+		delete(p.Config, "override")
+		delete(p.Config, "pause_before")
+		delete(p.Config, "type")
+
+		if len(p.Config) == 0 {
+			p.Config = nil
 		}
 
-		// TODO: stuff
 		result.Provisioners = append(result.Provisioners, &p)
 	}
 
@@ -309,9 +352,24 @@ func Parse(r io.Reader) (*Template, error) {
 	// Build an error if there are unused root level keys
 	if len(md.Unused) > 0 {
 		sort.Strings(md.Unused)
+
+		unusedMap, ok := raw.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("Failed to convert unused root level keys to map")
+		}
+
 		for _, unused := range md.Unused {
-			// Ignore keys starting with '_' as comments
 			if unused[0] == '_' {
+				commentVal, ok := unusedMap[unused].(string)
+				if !ok {
+					return nil, fmt.Errorf("Failed to cast root level comment value to string")
+				}
+
+				comment := map[string]string{
+					unused: commentVal,
+				}
+
+				rawTpl.Comments = append(rawTpl.Comments, comment)
 				continue
 			}
 

template/parse_test.go

@@ -3,7 +3,10 @@
 package template
 
 import (
+	"bytes"
+	"encoding/json"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -32,6 +35,21 @@ func TestParse(t *testing.T) {
 			},
 			false,
 		},
+		{
+			"parse-basic-config.json",
+			&Template{
+				Builders: map[string]*Builder{
+					"something": {
+						Name: "something",
+						Type: "something",
+						Config: map[string]interface{}{
+							"foo": "bar",
+						},
+					},
+				},
+			},
+			false,
+		},
 		{
 			"parse-builder-no-type.json",
 			nil,
@@ -57,7 +75,20 @@ func TestParse(t *testing.T) {
 			},
 			false,
 		},
-
+		{
+			"parse-provisioner-config.json",
+			&Template{
+				Provisioners: []*Provisioner{
+					{
+						Type: "something",
+						Config: map[string]interface{}{
+							"inline": "echo 'foo'",
+						},
+					},
+				},
+			},
+			false,
+		},
 		{
 			"parse-provisioner-pause-before.json",
 			&Template{
@@ -128,6 +159,7 @@ func TestParse(t *testing.T) {
 				Variables: map[string]*Variable{
 					"foo": {
 						Default: "foo",
+						Key:     "foo",
 					},
 				},
 			},
@@ -140,6 +172,7 @@ func TestParse(t *testing.T) {
 				Variables: map[string]*Variable{
 					"foo": {
 						Required: true,
+						Key:      "foo",
 					},
 				},
 			},
@@ -317,7 +350,6 @@ func TestParse(t *testing.T) {
 			},
 			false,
 		},
-
 		{
 			"parse-comment.json",
 			&Template{
@@ -327,6 +359,113 @@ func TestParse(t *testing.T) {
 						Type: "something",
 					},
 				},
+				Comments: map[string]string{
+					"_info": "foo",
+				},
+			},
+			false,
+		},
+		{
+			"parse-monolithic.json",
+			&Template{
+				Comments: map[string]string{
+					"_comment": "comment",
+				},
+				Description: "Description Test",
+				MinVersion:  "1.3.0",
+				SensitiveVariables: []*Variable{
+					{
+						Required: false,
+						Key:      "one",
+						Default:  "1",
+					},
+				},
+				Variables: map[string]*Variable{
+					"one": {
+						Required: false,
+						Key:      "one",
+						Default:  "1",
+					},
+					"two": {
+						Required: false,
+						Key:      "two",
+						Default:  "2",
+					},
+					"three": {
+						Required: true,
+						Key:      "three",
+						Default:  "",
+					},
+				},
+				Builders: map[string]*Builder{
+					"amazon-ebs": {
+						Name: "amazon-ebs",
+						Type: "amazon-ebs",
+						Config: map[string]interface{}{
+							"ami_name":      "AMI Name",
+							"instance_type": "t2.micro",
+							"ssh_username":  "ec2-user",
+							"source_ami":    "ami-aaaaaaaaaaaaaa",
+						},
+					},
+					"docker": {
+						Name: "docker",
+						Type: "docker",
+						Config: map[string]interface{}{
+							"image":       "ubuntu",
+							"export_path": "image.tar",
+						},
+					},
+				},
+				Provisioners: []*Provisioner{
+					{
+						Type: "shell",
+						Config: map[string]interface{}{
+							"script": "script.sh",
+						},
+					},
+					{
+						Type: "shell",
+						Config: map[string]interface{}{
+							"script": "script.sh",
+						},
+						Override: map[string]interface{}{
+							"docker": map[string]interface{}{
+								"execute_command": "echo 'override'",
+							},
+						},
+					},
+				},
+				PostProcessors: [][]*PostProcessor{
+					{
+						{
+							Name: "compress",
+							Type: "compress",
+						},
+						{
+							Name: "vagrant",
+							Type: "vagrant",
+							OnlyExcept: OnlyExcept{
+								Only: []string{"docker"},
+							},
+						},
+					},
+					{
+						{
+							Name: "shell-local",
+							Type: "shell-local",
+							Config: map[string]interface{}{
+								"inline": []interface{}{"echo foo"},
+							},
+							OnlyExcept: OnlyExcept{
+								Except: []string{"amazon-ebs"},
+							},
+						},
+					},
+				},
+				Push: Push{
+					Name: "push test",
+				},
 			},
 			false,
 		},
@@ -336,7 +475,7 @@ func TestParse(t *testing.T) {
 		path, _ := filepath.Abs(fixtureDir(tc.File))
 		tpl, err := ParseFile(fixtureDir(tc.File))
 		if (err != nil) != tc.Err {
-			t.Fatalf("err: %s", err)
+			t.Fatalf("%s\n\nerr: %s", tc.File, err)
 		}
 
 		if tc.Result != nil {
@@ -348,6 +487,38 @@ func TestParse(t *testing.T) {
 		if diff := cmp.Diff(tpl, tc.Result); diff != "" {
 			t.Fatalf("[%d]bad: %s\n%v", i, tc.File, diff)
 		}
+
+		// Only test template writing if the template is valid
+		if tc.Err == false {
+			// Get rawTemplate
+			raw, err := tpl.Raw()
+			if err != nil {
+				t.Fatalf("Failed to convert back to raw template: %s\n\n%v\n\n%s", tc.File, tpl, err)
+			}
+
+			out, _ := json.MarshalIndent(raw, "", "  ")
+			if err != nil {
+				t.Fatalf("Failed to marshal raw template: %s\n\n%v\n\n%s", tc.File, raw, err)
+			}
+
+			// Write JSON to a buffer (emulates filesystem write without dirtying the workspace)
+			fileBuf := bytes.NewBuffer(out)
+
+			// Parse the JSON template we wrote to our buffer
+			tplRewritten, err := Parse(fileBuf)
+			if err != nil {
+				t.Fatalf("Failed to re-read marshalled template: %s\n\n%v\n\n%s\n\n%s", tc.File, tpl, out, err)
+			}
+
+			// Override the metadata we don't care about (file path, raw file contents)
+			tplRewritten.Path = path
+			tplRewritten.RawContents = nil
+
+			// Test that our output raw template is functionally equal
+			if !reflect.DeepEqual(tpl, tplRewritten) {
+				t.Fatalf("Data lost when writing template to file: %s\n\n%v\n\n%v\n\n%s", tc.File, tpl, tplRewritten, out)
+			}
+		}
 	}
 }
 

template/template.go

@@ -1,6 +1,7 @@
 package template
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"time"
@@ -18,6 +19,7 @@ type Template struct {
 	Description string
 	MinVersion  string
 
+	Comments           map[string]string
 	Variables          map[string]*Variable
 	SensitiveVariables []*Variable
 	Builders           map[string]*Builder
@@ -29,31 +31,143 @@ type Template struct {
 	RawContents []byte
 }
 
+// Raw converts a Template struct back into the raw Packer template structure
+func (t *Template) Raw() (*rawTemplate, error) {
+	var out rawTemplate
+
+	out.MinVersion = t.MinVersion
+	out.Description = t.Description
+
+	for k, v := range t.Comments {
+		out.Comments = append(out.Comments, map[string]string{k: v})
+	}
+
+	for _, b := range t.Builders {
+		out.Builders = append(out.Builders, b)
+	}
+
+	for _, p := range t.Provisioners {
+		out.Provisioners = append(out.Provisioners, p)
+	}
+
+	for _, pp := range t.PostProcessors {
+		out.PostProcessors = append(out.PostProcessors, pp)
+	}
+
+	for _, v := range t.SensitiveVariables {
+		out.SensitiveVariables = append(out.SensitiveVariables, v.Key)
+	}
+
+	for k, v := range t.Variables {
+		if out.Variables == nil {
+			out.Variables = make(map[string]interface{})
+		}
+
+		out.Variables[k] = v
+	}
+
+	if t.Push.Name != "" {
+		b, _ := json.Marshal(t.Push)
+
+		var m map[string]interface{}
+		_ = json.Unmarshal(b, &m)
+
+		out.Push = m
+	}
+
+	return &out, nil
+}
+
 // Builder represents a builder configured in the template
 type Builder struct {
-	Name   string
-	Type   string
-	Config map[string]interface{}
+	Name   string                 `json:"name,omitempty"`
+	Type   string                 `json:"type"`
+	Config map[string]interface{} `json:"config,omitempty"`
+}
+
+// MarshalJSON conducts the necessary flattening of the Builder struct
+// to provide valid Packer template JSON
+func (b *Builder) MarshalJSON() ([]byte, error) {
+	// Avoid recursion
+	type Builder_ Builder
+	out, _ := json.Marshal(Builder_(*b))
+
+	var m map[string]json.RawMessage
+	_ = json.Unmarshal(out, &m)
+
+	// Flatten Config
+	delete(m, "config")
+	for k, v := range b.Config {
+		out, _ = json.Marshal(v)
+		m[k] = out
+	}
+
+	return json.Marshal(m)
 }
 
 // PostProcessor represents a post-processor within the template.
 type PostProcessor struct {
-	OnlyExcept `mapstructure:",squash"`
+	OnlyExcept `mapstructure:",squash" json:",omitempty"`
 
-	Name              string
-	Type              string
-	KeepInputArtifact bool `mapstructure:"keep_input_artifact"`
-	Config            map[string]interface{}
+	Name              string                 `json:"name,omitempty"`
+	Type              string                 `json:"type"`
+	KeepInputArtifact bool                   `mapstructure:"keep_input_artifact" json:"keep_input_artifact,omitempty"`
+	Config            map[string]interface{} `json:"config,omitempty"`
+}
+
+// MarshalJSON conducts the necessary flattening of the PostProcessor struct
+// to provide valid Packer template JSON
+func (p *PostProcessor) MarshalJSON() ([]byte, error) {
+	// Early exit for simple definitions
+	if len(p.Config) == 0 && len(p.OnlyExcept.Only) == 0 && len(p.OnlyExcept.Except) == 0 && !p.KeepInputArtifact {
+		return json.Marshal(p.Type)
+	}
+
+	// Avoid recursion
+	type PostProcessor_ PostProcessor
+	out, _ := json.Marshal(PostProcessor_(*p))
+
+	var m map[string]json.RawMessage
+	_ = json.Unmarshal(out, &m)
+
+	// Flatten Config
+	delete(m, "config")
+	for k, v := range p.Config {
+		out, _ = json.Marshal(v)
+		m[k] = out
+	}
+
+	return json.Marshal(m)
 }
 
 // Provisioner represents a provisioner within the template.
 type Provisioner struct {
-	OnlyExcept `mapstructure:",squash"`
+	OnlyExcept `mapstructure:",squash" json:",omitempty"`
 
-	Type        string
-	Config      map[string]interface{}
-	Override    map[string]interface{}
-	PauseBefore time.Duration `mapstructure:"pause_before"`
+	Type        string                 `json:"type"`
+	Config      map[string]interface{} `json:"config,omitempty"`
+	Override    map[string]interface{} `json:"override,omitempty"`
+	PauseBefore time.Duration          `mapstructure:"pause_before" json:"pause_before,omitempty"`
+}
+
+// MarshalJSON conducts the necessary flattening of the Provisioner struct
+// to provide valid Packer template JSON
+func (p *Provisioner) MarshalJSON() ([]byte, error) {
+	// Avoid recursion
+	type Provisioner_ Provisioner
+	out, _ := json.Marshal(Provisioner_(*p))
+
+	var m map[string]json.RawMessage
+	_ = json.Unmarshal(out, &m)
+
+	// Flatten Config
+	delete(m, "config")
+	for k, v := range p.Config {
+		out, _ = json.Marshal(v)
+		m[k] = out
+	}
+
+	return json.Marshal(m)
 }
 
 // Push represents the configuration for pushing the template to Atlas.
@@ -69,15 +183,26 @@ type Push struct {
 
 // Variable represents a variable within the template
 type Variable struct {
+	Key      string
 	Default  string
 	Required bool
 }
 
+func (v *Variable) MarshalJSON() ([]byte, error) {
+	if v.Required {
+		// We use a nil pointer to coax Go into marshalling it as a JSON null
+		var ret *string
+		return json.Marshal(ret)
+	}
+
+	return json.Marshal(v.Default)
+}
+
 // OnlyExcept is a struct that is meant to be embedded that contains the
 // logic required for "only" and "except" meta-parameters.
 type OnlyExcept struct {
-	Only   []string
-	Except []string
+	Only   []string `json:"only,omitempty"`
+	Except []string `json:"except,omitempty"`
 }
 
 //-------------------------------------------------------------------

template/test-fixtures/parse-basic-config.json

@@ -0,0 +1,3 @@
+{
+    "builders": [{"type": "something", "foo": "bar"}]
+}

template/test-fixtures/parse-monolithic.json

@@ -0,0 +1,61 @@
+{
+  "_comment": "comment",
+  "description": "Description Test",
+  "min_packer_version": "1.3.0",
+  "variables": {
+    "one": "1",
+    "two": "2",
+    "three": null
+  },
+  "sensitive-variables": ["one"],
+  "builders": [
+    {
+      "type": "amazon-ebs",
+
+      "ami_name": "AMI Name",
+      "instance_type": "t2.micro",
+      "ssh_username": "ec2-user",
+      "source_ami": "ami-aaaaaaaaaaaaaa"
+    },
+    {
+      "type": "docker",
+
+      "image": "ubuntu",
+      "export_path": "image.tar"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "script": "script.sh"
+    },
+    {
+      "type": "shell",
+      "script": "script.sh",
+      "override": {
+        "docker": {
+          "execute_command": "echo 'override'"
+        }
+      }
+    }
+  ],
+  "post-processors": [
+    [
+      "compress",
+      {
+        "type": "vagrant",
+        "only": ["docker"]
+      }
+    ],
+    [
+      {
+        "type": "shell-local",
+        "inline": ["echo foo"],
+        "except": ["amazon-ebs"]
+      }
+    ]
+  ],
+  "push": {
+    "name": "push test"
+  }
+}

template/test-fixtures/parse-provisioner-config.json

@@ -0,0 +1,5 @@
+{
+    "provisioners": [
+        {"type": "something","inline":"echo 'foo'"}
+    ]
+}