packer-cn/cmd/mapstructure-to-hcl2/mapstructure-to-hcl2.go

599 lines
20 KiB
Go

// mapstructure-to-hcl2 fills the gaps between hcl2 and mapstructure for Packer
//
// By generating a struct that the HCL2 ecosystem understands making use of
// mapstructure tags.
//
// Packer heavily uses the mapstructure decoding library to load/parse user
// config files. Packer now needs to move to HCL2.
//
// Here are a few differences/gaps betweens hcl2 and mapstructure:
//
// * in HCL2 all basic struct fields (string/int/struct) that are not pointers
// are required ( must be set ). In mapstructure everything is optional.
//
// * mapstructure allows to 'squash' fields
// (ex: Field CommonStructType `mapstructure:",squash"`) this allows to
// decorate structs and reuse configuration code. HCL2 parsing libs don't have
// anything similar.
//
// mapstructure-to-hcl2 will parse Packer's config files and generate the HCL2
// compliant code that will allow to not change any of the current builders in
// order to softly move to HCL2.
package main
import (
"bytes"
"flag"
"fmt"
"go/types"
"io"
"log"
"os"
"regexp"
"sort"
"strings"
"github.com/fatih/structtag"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/zclconf/go-cty/cty"
"golang.org/x/tools/go/packages"
"golang.org/x/tools/imports"
)
const mapstructureToHCL2 = "mapstructure-to-hcl2"
var (
typeNames = flag.String("type", "", "comma-separated list of type names; must be set")
output = flag.String("output", "", "output file name; default srcdir/<type>_hcl2.go")
trimprefix = flag.String("trimprefix", "", "trim the `prefix` from the generated constant names")
)
// Usage is a replacement usage function for the flags package.
func Usage() {
fmt.Fprintf(os.Stderr, "Usage of "+mapstructureToHCL2+":\n")
fmt.Fprintf(os.Stderr, "\t"+mapstructureToHCL2+" [flags] -type T[,T...] pkg\n")
fmt.Fprintf(os.Stderr, "Flags:\n")
flag.PrintDefaults()
}
func main() {
log.SetFlags(0)
log.SetPrefix(mapstructureToHCL2 + ": ")
flag.Usage = Usage
flag.Parse()
if len(*typeNames) == 0 {
flag.Usage()
os.Exit(2)
}
typeNames := strings.Split(*typeNames, ",")
// We accept either one directory or a list of files. Which do we have?
args := flag.Args()
if len(args) == 0 {
// Default: process whole package in current directory.
args = []string{"."}
}
outputPath := strings.ToLower(typeNames[0]) + ".hcl2spec.go"
if goFile := os.Getenv("GOFILE"); goFile != "" {
outputPath = goFile[:len(goFile)-2] + "hcl2spec.go"
}
log.SetPrefix(fmt.Sprintf(mapstructureToHCL2+": %s.%v: ", os.Getenv("GOPACKAGE"), typeNames))
cfg := &packages.Config{
Mode: packages.LoadSyntax,
}
pkgs, err := packages.Load(cfg, args...)
if err != nil {
log.Fatal(err)
}
if len(pkgs) != 1 {
log.Fatalf("error: %d packages found", len(pkgs))
}
topPkg := pkgs[0]
sort.Strings(typeNames)
var structs []StructDef
usedImports := map[NamePath]*types.Package{}
for id, obj := range topPkg.TypesInfo.Defs {
if obj == nil {
continue
}
t := obj.Type()
nt, isANamedType := t.(*types.Named)
if !isANamedType {
continue
}
if nt.Obj().Pkg() != topPkg.Types {
// Sometimes a struct embeds another struct named the same. ex:
// builder/osc/bsuvolume.BlockDevice. This makes sure the type is
// defined in topPkg.
continue
}
ut := nt.Underlying()
utStruct, utOk := ut.(*types.Struct)
if !utOk {
continue
}
pos := sort.SearchStrings(typeNames, id.Name)
if pos >= len(typeNames) || typeNames[pos] != id.Name {
continue // not a struct we care about
}
// make sure each type is found once where somehow sometimes they can be found twice
typeNames = append(typeNames[:pos], typeNames[pos+1:]...)
flatenedStruct := getMapstructureSquashedStruct(obj.Pkg(), utStruct)
flatenedStruct = addCtyTagToStruct(flatenedStruct)
newStructName := "Flat" + id.Name
structs = append(structs, StructDef{
OriginalStructName: id.Name,
FlatStructName: newStructName,
Struct: flatenedStruct,
})
for k, v := range getUsedImports(flatenedStruct) {
if _, found := usedImports[k]; !found {
usedImports[k] = v
}
}
}
out := bytes.NewBuffer(nil)
fmt.Fprintf(out, `// Code generated by "%s %s"; DO NOT EDIT.`, mapstructureToHCL2, strings.Join(os.Args[1:], " "))
fmt.Fprintf(out, "\npackage %s\n", topPkg.Name)
delete(usedImports, NamePath{topPkg.Name, topPkg.PkgPath})
usedImports[NamePath{"hcldec", "github.com/hashicorp/hcl/v2/hcldec"}] = types.NewPackage("hcldec", "github.com/hashicorp/hcl/v2/hcldec")
usedImports[NamePath{"cty", "github.com/zclconf/go-cty/cty"}] = types.NewPackage("cty", "github.com/zclconf/go-cty/cty")
outputImports(out, usedImports)
sort.Slice(structs, func(i int, j int) bool {
return structs[i].OriginalStructName < structs[j].OriginalStructName
})
for _, flatenedStruct := range structs {
fmt.Fprintf(out, "\n// %s is an auto-generated flat version of %s.", flatenedStruct.FlatStructName, flatenedStruct.OriginalStructName)
fmt.Fprintf(out, "\n// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.")
fmt.Fprintf(out, "\ntype %s struct {\n", flatenedStruct.FlatStructName)
outputStructFields(out, flatenedStruct.Struct)
fmt.Fprint(out, "}\n")
fmt.Fprintf(out, "\n// FlatMapstructure returns a new %s.", flatenedStruct.FlatStructName)
fmt.Fprintf(out, "\n// %s is an auto-generated flat version of %s.", flatenedStruct.FlatStructName, flatenedStruct.OriginalStructName)
fmt.Fprintf(out, "\n// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.")
fmt.Fprintf(out, "\nfunc (*%s) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {", flatenedStruct.OriginalStructName)
fmt.Fprintf(out, "\nreturn new(%s)", flatenedStruct.FlatStructName)
fmt.Fprint(out, "\n}\n")
fmt.Fprintf(out, "\n// HCL2Spec returns the hcl spec of a %s.", flatenedStruct.OriginalStructName)
fmt.Fprintf(out, "\n// This spec is used by HCL to read the fields of %s.", flatenedStruct.OriginalStructName)
fmt.Fprintf(out, "\n// The decoded values from this spec will then be applied to a %s.", flatenedStruct.FlatStructName)
fmt.Fprintf(out, "\nfunc (*%s) HCL2Spec() map[string]hcldec.Spec {\n", flatenedStruct.FlatStructName)
outputStructHCL2SpecBody(out, flatenedStruct.Struct)
fmt.Fprint(out, "}\n")
}
for impt := range usedImports {
if strings.ContainsAny(impt.Path, "/") {
out = bytes.NewBuffer(bytes.ReplaceAll(out.Bytes(),
[]byte(impt.Path+"."),
[]byte(impt.Name+".")))
}
}
// avoid needing to import current pkg; there's probably a better way.
out = bytes.NewBuffer(bytes.ReplaceAll(out.Bytes(),
[]byte(topPkg.PkgPath+"."),
nil))
outputFile, err := os.Create(outputPath)
if err != nil {
log.Fatalf("os.Create: %v", err)
}
_, err = outputFile.Write(goFmt(outputFile.Name(), out.Bytes()))
if err != nil {
log.Fatalf("failed to write file: %v", err)
}
}
type StructDef struct {
OriginalStructName string
FlatStructName string
Struct *types.Struct
}
// outputStructHCL2SpecBody writes the map[string]hcldec.Spec that defines the HCL spec of a
// struct. Based on the layout of said struct.
// If a field of s is a struct then the HCL2Spec() function of that struct will be called, otherwise a
// cty.Type is outputed.
func outputStructHCL2SpecBody(w io.Writer, s *types.Struct) {
fmt.Fprintf(w, "s := map[string]hcldec.Spec{\n")
for i := 0; i < s.NumFields(); i++ {
field, tag := s.Field(i), s.Tag(i)
st, _ := structtag.Parse(tag)
ctyTag, _ := st.Get("cty")
fmt.Fprintf(w, " \"%s\": ", ctyTag.Name)
outputHCL2SpecField(w, ctyTag.Name, field.Type(), st)
fmt.Fprintln(w, `,`)
}
fmt.Fprintln(w, `}`)
fmt.Fprintln(w, `return s`)
}
// outputHCL2SpecField is called on each field of a struct.
// outputHCL2SpecField writes the values of the `map[string]hcldec.Spec` map
// supposed to define the HCL spec of a struct.
func outputHCL2SpecField(w io.Writer, accessor string, fieldType types.Type, tag *structtag.Tags) {
if m2h, err := tag.Get(mapstructureToHCL2); err == nil && m2h.HasOption("self-defined") {
fmt.Fprintf(w, `(&%s{}).HCL2Spec()`, fieldType.String())
return
}
spec, _ := goFieldToCtyType(accessor, fieldType)
switch spec := spec.(type) {
case string:
fmt.Fprintf(w, spec)
default:
fmt.Fprintf(w, `%#v`, spec)
}
}
// goFieldToCtyType is a recursive method that returns a cty.Type (or a string) based on the fieldType.
// goFieldToCtyType returns the values of the `map[string]hcldec.Spec` map
// supposed to define the HCL spec of a struct.
// To allow it to be recursive, the method returns two values: an interface that can either be
// a cty.Type or a string. The second argument is used for recursion and is the
// type that will be used by the parent. For example when fieldType is a []string; a
// recursive goFieldToCtyType call will return a cty.String.
func goFieldToCtyType(accessor string, fieldType types.Type) (interface{}, cty.Type) {
switch f := fieldType.(type) {
case *types.Pointer:
return goFieldToCtyType(accessor, f.Elem())
case *types.Basic:
ctyType := basicKindToCtyType(f.Kind())
return &hcldec.AttrSpec{
Name: accessor,
Type: ctyType,
Required: false,
}, ctyType
case *types.Map:
return &hcldec.AttrSpec{
Name: accessor,
Type: cty.Map(cty.String), // for now everything can be simplified to a map[string]string
}, cty.Map(cty.String)
case *types.Named:
// Named is the relative type when of a field with a struct.
// E.g. SourceAmiFilter *common.FlatAmiFilterOptions
// SourceAmiFilter will become a block with nested elements from the struct itself.
underlyingType := f.Underlying()
switch underlyingType.(type) {
case *types.Struct:
// A struct returns NilType because its HCL2Spec is written in the related file
// and we don't need to write it again.
return fmt.Sprintf(`&hcldec.BlockSpec{TypeName: "%s",`+
` Nested: hcldec.ObjectSpec((*%s)(nil).HCL2Spec())}`, accessor, f.String()), cty.NilType
default:
return goFieldToCtyType(accessor, underlyingType)
}
case *types.Slice:
elem := f.Elem()
if ptr, isPtr := elem.(*types.Pointer); isPtr {
elem = ptr.Elem()
}
switch elem := elem.(type) {
case *types.Named:
// A Slice of Named is the relative type of a filed with a slice of structs.
// E.g. LaunchMappings []common.FlatBlockDevice
// LaunchMappings will validate more than one block with nested elements.
b := bytes.NewBuffer(nil)
underlyingType := elem.Underlying()
switch underlyingType.(type) {
case *types.Struct:
fmt.Fprintf(b, `hcldec.ObjectSpec((*%s)(nil).HCL2Spec())`, elem.String())
}
return fmt.Sprintf(`&hcldec.BlockListSpec{TypeName: "%s", Nested: %s}`, accessor, b.String()), cty.NilType
default:
_, specType := goFieldToCtyType(accessor, elem)
if specType == cty.NilType {
return goFieldToCtyType(accessor, elem.Underlying())
}
return &hcldec.AttrSpec{
Name: accessor,
Type: cty.List(specType),
Required: false,
}, cty.List(specType)
}
}
b := bytes.NewBuffer(nil)
fmt.Fprintf(b, `%#v`, &hcldec.AttrSpec{
Name: accessor,
Type: basicKindToCtyType(types.Bool),
Required: false,
})
fmt.Fprintf(b, `/* TODO(azr): could not find type */`)
return b.String(), cty.NilType
}
func basicKindToCtyType(kind types.BasicKind) cty.Type {
switch kind {
case types.Bool:
return cty.Bool
case types.String:
return cty.String
case types.Int, types.Int8, types.Int16, types.Int32, types.Int64,
types.Uint, types.Uint8, types.Uint16, types.Uint32, types.Uint64,
types.Float32, types.Float64,
types.Complex64, types.Complex128:
return cty.Number
case types.Invalid:
return cty.String // TODO(azr): fix that beforehand ?
default:
log.Printf("Un handled basic kind: %d", kind)
return cty.String
}
}
func outputStructFields(w io.Writer, s *types.Struct) {
for i := 0; i < s.NumFields(); i++ {
field, tag := s.Field(i), s.Tag(i)
fieldNameStr := field.String()
fieldNameStr = strings.Replace(fieldNameStr, "field ", "", 1)
fmt.Fprintf(w, " %s `%s`\n", fieldNameStr, tag)
}
}
type NamePath struct {
Name, Path string
}
func outputImports(w io.Writer, imports map[NamePath]*types.Package) {
if len(imports) == 0 {
return
}
// naive implementation
pkgs := []NamePath{}
for k := range imports {
pkgs = append(pkgs, k)
}
sort.Slice(pkgs, func(i int, j int) bool {
return pkgs[i].Path < pkgs[j].Path
})
fmt.Fprint(w, "import (\n")
for _, pkg := range pkgs {
if pkg.Name == pkg.Path || strings.HasSuffix(pkg.Path, "/"+pkg.Name) {
fmt.Fprintf(w, " \"%s\"\n", pkg.Path)
} else {
fmt.Fprintf(w, " %s \"%s\"\n", pkg.Name, pkg.Path)
}
}
fmt.Fprint(w, ")\n")
}
func getUsedImports(s *types.Struct) map[NamePath]*types.Package {
res := map[NamePath]*types.Package{}
for i := 0; i < s.NumFields(); i++ {
fieldType := s.Field(i).Type()
if p, ok := fieldType.(*types.Pointer); ok {
fieldType = p.Elem()
}
if p, ok := fieldType.(*types.Slice); ok {
fieldType = p.Elem()
}
namedType, ok := fieldType.(*types.Named)
if !ok {
continue
}
pkg := namedType.Obj().Pkg()
if pkg == nil {
continue
}
res[NamePath{pkg.Name(), pkg.Path()}] = pkg
}
return res
}
func addCtyTagToStruct(s *types.Struct) *types.Struct {
vars, tags := structFields(s)
for i := range tags {
field, tag := vars[i], tags[i]
ctyAccessor := ToSnakeCase(field.Name())
st, err := structtag.Parse(tag)
if err == nil {
if ms, err := st.Get("mapstructure"); err == nil && ms.Name != "" {
ctyAccessor = ms.Name
}
}
st.Set(&structtag.Tag{Key: "cty", Name: ctyAccessor})
_ = st.Set(&structtag.Tag{Key: "hcl", Name: ctyAccessor})
tags[i] = st.String()
}
return types.NewStruct(uniqueTags("cty", vars, tags))
}
func uniqueTags(tagName string, fields []*types.Var, tags []string) ([]*types.Var, []string) {
outVars := []*types.Var{}
outTags := []string{}
uniqueTags := map[string]bool{}
for i := range fields {
field, tag := fields[i], tags[i]
structtag, _ := structtag.Parse(tag)
h, err := structtag.Get(tagName)
if err == nil {
if uniqueTags[h.Name] {
log.Printf("skipping field %s ( duplicate `%s` %s tag )", field.Name(), h.Name, tagName)
continue
}
uniqueTags[h.Name] = true
}
outVars = append(outVars, field)
outTags = append(outTags, tag)
}
return outVars, outTags
}
// getMapstructureSquashedStruct will return the same struct but embedded
// fields with a `mapstructure:",squash"` tag will be un-nested.
func getMapstructureSquashedStruct(topPkg *types.Package, utStruct *types.Struct) *types.Struct {
res := &types.Struct{}
for i := 0; i < utStruct.NumFields(); i++ {
field, tag := utStruct.Field(i), utStruct.Tag(i)
if !field.Exported() {
continue
}
if _, ok := field.Type().(*types.Signature); ok {
continue // ignore funcs
}
structtag, err := structtag.Parse(tag)
if err != nil {
log.Printf("could not parse field tag %s of : %v", tag, err)
continue
}
// Contains mapstructure-to-hcl2 tag
if ms, err := structtag.Get("mapstructure-to-hcl2"); err == nil {
// Stop if is telling to skip it
if ms.HasOption("skip") {
continue
}
}
// Contains mapstructure tag
if ms, err := structtag.Get("mapstructure"); err == nil {
// Squash structs
if ms.HasOption("squash") {
ot := field.Type()
uot := ot.Underlying()
utStruct, utOk := uot.(*types.Struct)
if !utOk {
continue
}
res = squashStructs(res, getMapstructureSquashedStruct(topPkg, utStruct))
continue
}
}
if field.Pkg() != topPkg {
field = types.NewField(field.Pos(), topPkg, field.Name(), field.Type(), field.Embedded())
}
if p, isPointer := field.Type().(*types.Pointer); isPointer {
// in order to make the following switch simpler we 'unwrap' this
// pointer all structs are going to be made pointers anyways.
field = types.NewField(field.Pos(), field.Pkg(), field.Name(), p.Elem(), field.Embedded())
}
switch f := field.Type().(type) {
case *types.Named:
switch f.String() {
case "time.Duration":
field = types.NewField(field.Pos(), field.Pkg(), field.Name(), types.NewPointer(types.Typ[types.String]), field.Embedded())
case "github.com/hashicorp/packer/helper/config.Trilean": // TODO(azr): unhack this situation
field = types.NewField(field.Pos(), field.Pkg(), field.Name(), types.NewPointer(types.Typ[types.Bool]), field.Embedded())
case "github.com/hashicorp/packer/provisioner/powershell.ExecutionPolicy": // TODO(azr): unhack this situation
field = types.NewField(field.Pos(), field.Pkg(), field.Name(), types.NewPointer(types.Typ[types.String]), field.Embedded())
default:
if str, isStruct := f.Underlying().(*types.Struct); isStruct {
obj := flattenNamed(f, str)
field = types.NewField(field.Pos(), field.Pkg(), field.Name(), obj, field.Embedded())
field = makePointer(field)
}
if slice, isSlice := f.Underlying().(*types.Slice); isSlice {
if f, fNamed := slice.Elem().(*types.Named); fNamed {
if str, isStruct := f.Underlying().(*types.Struct); isStruct {
// this is a slice of named structs; we want to change
// the struct ref to a 'FlatStruct'.
obj := flattenNamed(f, str)
slice := types.NewSlice(obj)
field = types.NewField(field.Pos(), field.Pkg(), field.Name(), slice, field.Embedded())
}
}
}
if _, isBasic := f.Underlying().(*types.Basic); isBasic {
field = makePointer(field)
}
}
case *types.Slice:
if f, fNamed := f.Elem().(*types.Named); fNamed {
if str, isStruct := f.Underlying().(*types.Struct); isStruct {
obj := flattenNamed(f, str)
field = types.NewField(field.Pos(), field.Pkg(), field.Name(), types.NewSlice(obj), field.Embedded())
}
}
case *types.Basic:
// since everything is optional, everything must be a pointer
// non optional fields should be non pointers.
field = makePointer(field)
}
res = addFieldToStruct(res, field, tag)
}
return res
}
func flattenNamed(f *types.Named, underlying types.Type) *types.Named {
obj := f.Obj()
obj = types.NewTypeName(obj.Pos(), obj.Pkg(), "Flat"+obj.Name(), obj.Type())
return types.NewNamed(obj, underlying, nil)
}
func makePointer(field *types.Var) *types.Var {
return types.NewField(field.Pos(), field.Pkg(), field.Name(), types.NewPointer(field.Type()), field.Embedded())
}
func addFieldToStruct(s *types.Struct, field *types.Var, tag string) *types.Struct {
sf, st := structFields(s)
return types.NewStruct(uniqueFields(append(sf, field), append(st, tag)))
}
func squashStructs(a, b *types.Struct) *types.Struct {
va, ta := structFields(a)
vb, tb := structFields(b)
return types.NewStruct(uniqueFields(append(va, vb...), append(ta, tb...)))
}
func uniqueFields(fields []*types.Var, tags []string) ([]*types.Var, []string) {
outVars := []*types.Var{}
outTags := []string{}
fieldNames := map[string]bool{}
for i := range fields {
field, tag := fields[i], tags[i]
if fieldNames[field.Name()] {
log.Printf("skipping duplicate %s field", field.Name())
continue
}
fieldNames[field.Name()] = true
outVars = append(outVars, field)
outTags = append(outTags, tag)
}
return outVars, outTags
}
func structFields(s *types.Struct) (vars []*types.Var, tags []string) {
for i := 0; i < s.NumFields(); i++ {
field, tag := s.Field(i), s.Tag(i)
vars = append(vars, field)
tags = append(tags, tag)
}
return vars, tags
}
var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])")
func ToSnakeCase(str string) string {
snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}")
snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}")
return strings.ToLower(snake)
}
func goFmt(filename string, b []byte) []byte {
fb, err := imports.Process(filename, b, nil)
if err != nil {
log.Printf("formatting err: %v", err)
return b
}
return fb
}