packer-cn/template/parse.go

579 lines
15 KiB
Go

package template
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/packer/packer/tmp"
"github.com/mitchellh/mapstructure"
)
// rawTemplate is the direct JSON document format of the template file.
// This is what is decoded directly from the file, and then it is turned
// into a Template object thereafter.
type rawTemplate struct {
MinVersion string `mapstructure:"min_packer_version" json:"min_packer_version,omitempty"`
Description string `json:"description,omitempty"`
Builders []interface{} `mapstructure:"builders" json:"builders,omitempty"`
Comments []map[string]string `json:"comments,omitempty"`
Push map[string]interface{} `json:"push,omitempty"`
PostProcessors []interface{} `mapstructure:"post-processors" json:"post-processors,omitempty"`
Provisioners []interface{} `json:"provisioners,omitempty"`
CleanupProvisioner interface{} `mapstructure:"error-cleanup-provisioner" json:"error-cleanup-provisioner,omitempty"`
Variables map[string]interface{} `json:"variables,omitempty"`
SensitiveVariables []string `mapstructure:"sensitive-variables" json:"sensitive-variables,omitempty"`
RawContents []byte `json:"-"`
}
// MarshalJSON conducts the necessary flattening of the rawTemplate struct
// to provide valid Packer template JSON
func (r *rawTemplate) MarshalJSON() ([]byte, error) {
// Avoid recursion
type rawTemplate_ rawTemplate
out, _ := json.Marshal(rawTemplate_(*r))
var m map[string]json.RawMessage
_ = json.Unmarshal(out, &m)
// Flatten Comments
delete(m, "comments")
for _, comment := range r.Comments {
for k, v := range comment {
out, _ = json.Marshal(v)
m[k] = out
}
}
return json.Marshal(m)
}
func (r *rawTemplate) decodeProvisioner(raw interface{}) (Provisioner, error) {
var p Provisioner
if err := r.weakDecoder(&p, nil).Decode(raw); err != nil {
return p, fmt.Errorf("Error decoding provisioner: %s", err)
}
// Type is required before any richer validation
if p.Type == "" {
return p, fmt.Errorf("Provisioner missing 'type'")
}
// Set the raw configuration and delete any special keys
p.Config = raw.(map[string]interface{})
delete(p.Config, "except")
delete(p.Config, "only")
delete(p.Config, "override")
delete(p.Config, "pause_before")
delete(p.Config, "max_retries")
delete(p.Config, "type")
delete(p.Config, "timeout")
if len(p.Config) == 0 {
p.Config = nil
}
return p, nil
}
// Template returns the actual Template object built from this raw
// structure.
func (r *rawTemplate) Template() (*Template, error) {
var result Template
var errs error
// Copy some literals
result.Description = r.Description
result.MinVersion = r.MinVersion
result.RawContents = r.RawContents
// Gather the comments
if len(r.Comments) > 0 {
result.Comments = make(map[string]string, len(r.Comments))
for _, c := range r.Comments {
for k, v := range c {
result.Comments[k] = v
}
}
}
// Gather the variables
if len(r.Variables) > 0 {
result.Variables = make(map[string]*Variable, len(r.Variables))
}
for k, rawV := range r.Variables {
var v Variable
v.Key = k
// Variable is required if the value is exactly nil
v.Required = rawV == nil
// Weak decode the default if we have one
if err := r.decoder(&v.Default, nil).Decode(rawV); err != nil {
errs = multierror.Append(errs, fmt.Errorf(
"variable %s: %s", k, err))
continue
}
for _, sVar := range r.SensitiveVariables {
if sVar == k {
result.SensitiveVariables = append(result.SensitiveVariables, &v)
}
}
result.Variables[k] = &v
}
// Let's start by gathering all the builders
if len(r.Builders) > 0 {
result.Builders = make(map[string]*Builder, len(r.Builders))
}
for i, rawB := range r.Builders {
var b Builder
if err := mapstructure.WeakDecode(rawB, &b); err != nil {
errs = multierror.Append(errs, fmt.Errorf(
"builder %d: %s", i+1, err))
continue
}
// Set the raw configuration and delete any special keys
b.Config = rawB.(map[string]interface{})
delete(b.Config, "name")
delete(b.Config, "type")
if len(b.Config) == 0 {
b.Config = nil
}
// If there is no type set, it is an error
if b.Type == "" {
errs = multierror.Append(errs, fmt.Errorf(
"builder %d: missing 'type'", i+1))
continue
}
// The name defaults to the type if it isn't set
if b.Name == "" {
b.Name = b.Type
}
// If this builder already exists, it is an error
if _, ok := result.Builders[b.Name]; ok {
errs = multierror.Append(errs, fmt.Errorf(
"builder %d: builder with name '%s' already exists",
i+1, b.Name))
continue
}
// Append the builders
result.Builders[b.Name] = &b
}
// Gather all the post-processors
if len(r.PostProcessors) > 0 {
result.PostProcessors = make([][]*PostProcessor, 0, len(r.PostProcessors))
}
for i, v := range r.PostProcessors {
// Parse the configurations. We need to do this because post-processors
// can take three different formats.
configs, err := r.parsePostProcessor(i, v)
if err != nil {
errs = multierror.Append(errs, err)
continue
}
// Parse the PostProcessors out of the configs
pps := make([]*PostProcessor, 0, len(configs))
for j, c := range configs {
var pp PostProcessor
if err := r.decoder(&pp, nil).Decode(c); err != nil {
errs = multierror.Append(errs, fmt.Errorf(
"post-processor %d.%d: %s", i+1, j+1, err))
continue
}
// Type is required
if pp.Type == "" {
errs = multierror.Append(errs, fmt.Errorf(
"post-processor %d.%d: type is required", i+1, j+1))
continue
}
// Set the raw configuration and delete any special keys
pp.Config = c
// The name defaults to the type if it isn't set
if pp.Name == "" {
pp.Name = pp.Type
}
delete(pp.Config, "except")
delete(pp.Config, "only")
delete(pp.Config, "keep_input_artifact")
delete(pp.Config, "type")
delete(pp.Config, "name")
if len(pp.Config) == 0 {
pp.Config = nil
}
pps = append(pps, &pp)
}
result.PostProcessors = append(result.PostProcessors, pps)
}
// Gather all the provisioners
if len(r.Provisioners) > 0 {
result.Provisioners = make([]*Provisioner, 0, len(r.Provisioners))
}
for i, v := range r.Provisioners {
p, err := r.decodeProvisioner(v)
if err != nil {
errs = multierror.Append(errs, fmt.Errorf(
"provisioner %d: %s", i+1, err))
continue
}
result.Provisioners = append(result.Provisioners, &p)
}
// Gather the error-cleanup-provisioner
if r.CleanupProvisioner != nil {
p, err := r.decodeProvisioner(r.CleanupProvisioner)
if err != nil {
errs = multierror.Append(errs,
fmt.Errorf("On Error Cleanup Provisioner error: %s", err))
}
result.CleanupProvisioner = &p
}
// If we have errors, return those with a nil result
if errs != nil {
return nil, errs
}
return &result, nil
}
func (r *rawTemplate) decoder(
result interface{},
md *mapstructure.Metadata) *mapstructure.Decoder {
d, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
Metadata: md,
Result: result,
})
if err != nil {
// This really shouldn't happen since we have firm control over
// all the arguments and they're all unit tested. So we use a
// panic here to note this would definitely be a bug.
panic(err)
}
return d
}
func (r *rawTemplate) weakDecoder(
result interface{},
md *mapstructure.Metadata) *mapstructure.Decoder {
d, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
WeaklyTypedInput: true,
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
Metadata: md,
Result: result,
})
if err != nil {
// This really shouldn't happen since we have firm control over
// all the arguments and they're all unit tested. So we use a
// panic here to note this would definitely be a bug.
panic(err)
}
return d
}
func (r *rawTemplate) parsePostProcessor(
i int, raw interface{}) ([]map[string]interface{}, error) {
switch v := raw.(type) {
case string:
return []map[string]interface{}{
{"type": v},
}, nil
case map[string]interface{}:
return []map[string]interface{}{v}, nil
case []interface{}:
var err error
result := make([]map[string]interface{}, len(v))
for j, innerRaw := range v {
switch innerV := innerRaw.(type) {
case string:
result[j] = map[string]interface{}{"type": innerV}
case map[string]interface{}:
result[j] = innerV
case []interface{}:
err = multierror.Append(err, fmt.Errorf(
"post-processor %d.%d: sequence not allowed to be nested in a sequence",
i+1, j+1))
default:
err = multierror.Append(err, fmt.Errorf(
"post-processor %d.%d: unknown format",
i+1, j+1))
}
}
if err != nil {
return nil, err
}
return result, nil
default:
return nil, fmt.Errorf("post-processor %d: bad format", i+1)
}
}
// Parse takes the given io.Reader and parses a Template object out of it.
func Parse(r io.Reader) (*Template, error) {
// First, decode the object into an interface{} and search for duplicate fields.
// We do this instead of the rawTemplate directly because we'd rather use mapstructure to
// decode since it has richer errors.
var raw interface{}
buf, err := jsonUnmarshal(r, &raw)
if err != nil {
return nil, err
}
// Create our decoder
var md mapstructure.Metadata
var rawTpl rawTemplate
rawTpl.RawContents = buf.Bytes()
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
Metadata: &md,
Result: &rawTpl,
})
if err != nil {
return nil, err
}
// Do the actual decode into our structure
if err := decoder.Decode(raw); err != nil {
return nil, err
}
// Build an error if there are unused root level keys
if len(md.Unused) > 0 {
sort.Strings(md.Unused)
unusedMap, ok := raw.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("Failed to convert unused root level keys to map")
}
for _, unused := range md.Unused {
if unused[0] == '_' {
commentVal, ok := unusedMap[unused].(string)
if !ok {
return nil, fmt.Errorf("Failed to cast root level comment value in comment \"%s\" to string.", unused)
}
comment := map[string]string{
unused: commentVal,
}
rawTpl.Comments = append(rawTpl.Comments, comment)
continue
}
err = multierror.Append(err, fmt.Errorf(
"Unknown root level key in template: '%s'", unused))
}
}
if err != nil {
return nil, err
}
// Return the template parsed from the raw structure
return rawTpl.Template()
}
func jsonUnmarshal(r io.Reader, raw *interface{}) (bytes.Buffer, error) {
// Create a buffer to copy what we read
var buf bytes.Buffer
if _, err := buf.ReadFrom(r); err != nil {
return buf, err
}
// Decode the object into an interface{}
if err := json.Unmarshal(buf.Bytes(), raw); err != nil {
return buf, err
}
// If Json is valid, check for duplicate fields to avoid silent unwanted override
jsonDecoder := json.NewDecoder(strings.NewReader(buf.String()))
if err := checkForDuplicateFields(jsonDecoder); err != nil {
return buf, err
}
return buf, nil
}
func checkForDuplicateFields(d *json.Decoder) error {
// Get next token from JSON
t, err := d.Token()
if err != nil {
return err
}
delim, ok := t.(json.Delim)
// Do nothing if it's not a delimiter
if !ok {
return nil
}
// Check for duplicates inside of a delimiter {} or []
switch delim {
case '{':
keys := make(map[string]bool)
for d.More() {
// Get attribute key
t, err := d.Token()
if err != nil {
return err
}
key := t.(string)
// Check for duplicates
if keys[key] {
return fmt.Errorf("template has duplicate field: %s", key)
}
keys[key] = true
// Check value to find duplicates in nested blocks
if err := checkForDuplicateFields(d); err != nil {
return err
}
}
case '[':
for d.More() {
if err := checkForDuplicateFields(d); err != nil {
return err
}
}
}
// consume closing delimiter } or ]
if _, err := d.Token(); err != nil {
return err
}
return nil
}
// ParseFile is the same as Parse but is a helper to automatically open
// a file for parsing.
func ParseFile(path string) (*Template, error) {
var f *os.File
var err error
if path == "-" {
// Create a temp file for stdin in case of errors
f, err = tmp.File("parse")
if err != nil {
return nil, err
}
defer os.Remove(f.Name())
defer f.Close()
if _, err = io.Copy(f, os.Stdin); err != nil {
return nil, err
}
if _, err = f.Seek(0, io.SeekStart); err != nil {
return nil, err
}
} else {
f, err = os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
}
tpl, err := Parse(f)
if err != nil {
syntaxErr, ok := err.(*json.SyntaxError)
if !ok {
return nil, err
}
// Rewind the file and get a better error
if _, err := f.Seek(0, io.SeekStart); err != nil {
return nil, err
}
// Grab the error location, and return a string to point to offending syntax error
line, col, highlight := highlightPosition(f, syntaxErr.Offset)
err = fmt.Errorf("Error parsing JSON: %s\nAt line %d, column %d (offset %d):\n%s", err, line, col, syntaxErr.Offset, highlight)
return nil, err
}
if !filepath.IsAbs(path) {
path, err = filepath.Abs(path)
if err != nil {
return nil, err
}
}
tpl.Path = path
return tpl, nil
}
// Takes a file and the location in bytes of a parse error
// from json.SyntaxError.Offset and returns the line, column,
// and pretty-printed context around the error with an arrow indicating the exact
// position of the syntax error.
func highlightPosition(f *os.File, pos int64) (line, col int, highlight string) {
// Modified version of the function in Camlistore by Brad Fitzpatrick
// https://github.com/camlistore/camlistore/blob/4b5403dd5310cf6e1ae8feb8533fd59262701ebc/vendor/go4.org/errorutil/highlight.go
line = 1
// New io.Reader for file
br := bufio.NewReader(f)
// Initialize lines
lastLine := ""
thisLine := new(bytes.Buffer)
// Loop through template to find line, column
for n := int64(0); n < pos; n++ {
// read byte from io.Reader
b, err := br.ReadByte()
if err != nil {
break
}
// If end of line, save line as previous line in case next line is offender
if b == '\n' {
lastLine = thisLine.String()
thisLine.Reset()
line++
col = 1
} else {
// Write current line, until line is safe, or error point is encountered
col++
thisLine.WriteByte(b)
}
}
// Populate highlight string to place a '^' char at offending column
if line > 1 {
highlight += fmt.Sprintf("%5d: %s\n", line-1, lastLine)
}
highlight += fmt.Sprintf("%5d: %s\n", line, thisLine.String())
highlight += fmt.Sprintf("%s^\n", strings.Repeat(" ", col+5))
return
}