Merge pull request #2854 from mitchellh/f-integrated-plugins-rebase

Implemented internal plugins
This commit is contained in:
Chris Bednarski 2015-10-21 17:04:14 -07:00
commit 8e63ce1302
8 changed files with 637 additions and 13 deletions

181
command/plugin.go Normal file
View File

@ -0,0 +1,181 @@
//
// This file is automatically generated by scripts/generate-plugins.go -- Do not edit!
//
package command
import (
"fmt"
"log"
"regexp"
"strings"
"github.com/mitchellh/packer/packer"
"github.com/mitchellh/packer/packer/plugin"
amazonchrootbuilder "github.com/mitchellh/packer/builder/amazon/chroot"
amazonebsbuilder "github.com/mitchellh/packer/builder/amazon/ebs"
amazoninstancebuilder "github.com/mitchellh/packer/builder/amazon/instance"
ansiblelocalprovisioner "github.com/mitchellh/packer/provisioner/ansible-local"
artificepostprocessor "github.com/mitchellh/packer/post-processor/artifice"
atlaspostprocessor "github.com/mitchellh/packer/post-processor/atlas"
chefclientprovisioner "github.com/mitchellh/packer/provisioner/chef-client"
chefsoloprovisioner "github.com/mitchellh/packer/provisioner/chef-solo"
compresspostprocessor "github.com/mitchellh/packer/post-processor/compress"
digitaloceanbuilder "github.com/mitchellh/packer/builder/digitalocean"
dockerbuilder "github.com/mitchellh/packer/builder/docker"
dockerimportpostprocessor "github.com/mitchellh/packer/post-processor/docker-import"
dockerpushpostprocessor "github.com/mitchellh/packer/post-processor/docker-push"
dockersavepostprocessor "github.com/mitchellh/packer/post-processor/docker-save"
dockertagpostprocessor "github.com/mitchellh/packer/post-processor/docker-tag"
filebuilder "github.com/mitchellh/packer/builder/file"
fileprovisioner "github.com/mitchellh/packer/provisioner/file"
googlecomputebuilder "github.com/mitchellh/packer/builder/googlecompute"
nullbuilder "github.com/mitchellh/packer/builder/null"
openstackbuilder "github.com/mitchellh/packer/builder/openstack"
parallelsisobuilder "github.com/mitchellh/packer/builder/parallels/iso"
parallelspvmbuilder "github.com/mitchellh/packer/builder/parallels/pvm"
powershellprovisioner "github.com/mitchellh/packer/provisioner/powershell"
puppetmasterlessprovisioner "github.com/mitchellh/packer/provisioner/puppet-masterless"
puppetserverprovisioner "github.com/mitchellh/packer/provisioner/puppet-server"
qemubuilder "github.com/mitchellh/packer/builder/qemu"
saltmasterlessprovisioner "github.com/mitchellh/packer/provisioner/salt-masterless"
shelllocalprovisioner "github.com/mitchellh/packer/provisioner/shell-local"
shellprovisioner "github.com/mitchellh/packer/provisioner/shell"
vagrantcloudpostprocessor "github.com/mitchellh/packer/post-processor/vagrant-cloud"
vagrantpostprocessor "github.com/mitchellh/packer/post-processor/vagrant"
virtualboxisobuilder "github.com/mitchellh/packer/builder/virtualbox/iso"
virtualboxovfbuilder "github.com/mitchellh/packer/builder/virtualbox/ovf"
vmwareisobuilder "github.com/mitchellh/packer/builder/vmware/iso"
vmwarevmxbuilder "github.com/mitchellh/packer/builder/vmware/vmx"
vspherepostprocessor "github.com/mitchellh/packer/post-processor/vsphere"
windowsrestartprovisioner "github.com/mitchellh/packer/provisioner/windows-restart"
windowsshellprovisioner "github.com/mitchellh/packer/provisioner/windows-shell"
)
type PluginCommand struct {
Meta
}
var Builders = map[string]packer.Builder{
"amazon-chroot": new(amazonchrootbuilder.Builder),
"amazon-ebs": new(amazonebsbuilder.Builder),
"amazon-instance": new(amazoninstancebuilder.Builder),
"digitalocean": new(digitaloceanbuilder.Builder),
"docker": new(dockerbuilder.Builder),
"file": new(filebuilder.Builder),
"googlecompute": new(googlecomputebuilder.Builder),
"null": new(nullbuilder.Builder),
"openstack": new(openstackbuilder.Builder),
"parallels-iso": new(parallelsisobuilder.Builder),
"parallels-pvm": new(parallelspvmbuilder.Builder),
"qemu": new(qemubuilder.Builder),
"virtualbox-iso": new(virtualboxisobuilder.Builder),
"virtualbox-ovf": new(virtualboxovfbuilder.Builder),
"vmware-iso": new(vmwareisobuilder.Builder),
"vmware-vmx": new(vmwarevmxbuilder.Builder),
}
var Provisioners = map[string]packer.Provisioner{
"ansible-local": new(ansiblelocalprovisioner.Provisioner),
"chef-client": new(chefclientprovisioner.Provisioner),
"chef-solo": new(chefsoloprovisioner.Provisioner),
"file": new(fileprovisioner.Provisioner),
"powershell": new(powershellprovisioner.Provisioner),
"puppet-masterless": new(puppetmasterlessprovisioner.Provisioner),
"puppet-server": new(puppetserverprovisioner.Provisioner),
"salt-masterless": new(saltmasterlessprovisioner.Provisioner),
"shell": new(shellprovisioner.Provisioner),
"shell-local": new(shelllocalprovisioner.Provisioner),
"windows-restart": new(windowsrestartprovisioner.Provisioner),
"windows-shell": new(windowsshellprovisioner.Provisioner),
}
var PostProcessors = map[string]packer.PostProcessor{
"artifice": new(artificepostprocessor.PostProcessor),
"atlas": new(atlaspostprocessor.PostProcessor),
"compress": new(compresspostprocessor.PostProcessor),
"docker-import": new(dockerimportpostprocessor.PostProcessor),
"docker-push": new(dockerpushpostprocessor.PostProcessor),
"docker-save": new(dockersavepostprocessor.PostProcessor),
"docker-tag": new(dockertagpostprocessor.PostProcessor),
"vagrant": new(vagrantpostprocessor.PostProcessor),
"vagrant-cloud": new(vagrantcloudpostprocessor.PostProcessor),
"vsphere": new(vspherepostprocessor.PostProcessor),
}
var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")
func (c *PluginCommand) Run(args []string) int {
// This is an internal call (users should not call this directly) so we're
// not going to do much input validation. If there's a problem we'll often
// just crash. Error handling should be added to facilitate debugging.
log.Printf("args: %#v", args)
if len(args) != 1 {
c.Ui.Error("Wrong number of args")
return 1
}
// Plugin will match something like "packer-builder-amazon-ebs"
parts := pluginRegexp.FindStringSubmatch(args[0])
if len(parts) != 3 {
c.Ui.Error(fmt.Sprintf("Error parsing plugin argument [DEBUG]: %#v", parts))
return 1
}
pluginType := parts[1] // capture group 1 (builder|post-processor|provisioner)
pluginName := parts[2] // capture group 2 (.+)
server, err := plugin.Server()
if err != nil {
c.Ui.Error(fmt.Sprintf("Error starting plugin server: %s", err))
return 1
}
switch pluginType {
case "builder":
builder, found := Builders[pluginName]
if !found {
c.Ui.Error(fmt.Sprintf("Could not load builder: %s", pluginName))
return 1
}
server.RegisterBuilder(builder)
case "provisioner":
provisioner, found := Provisioners[pluginName]
if !found {
c.Ui.Error(fmt.Sprintf("Could not load provisioner: %s", pluginName))
return 1
}
server.RegisterProvisioner(provisioner)
case "post-processor":
postProcessor, found := PostProcessors[pluginName]
if !found {
c.Ui.Error(fmt.Sprintf("Could not load post-processor: %s", pluginName))
return 1
}
server.RegisterPostProcessor(postProcessor)
}
server.Serve()
return 0
}
func (*PluginCommand) Help() string {
helpText := `
Usage: packer plugin PLUGIN
Runs an internally-compiled version of a plugin from the packer binary.
NOTE: this is an internal command and you should not call it yourself.
`
return strings.TrimSpace(helpText)
}
func (c *PluginCommand) Synopsis() string {
return "internal plugin command"
}

View File

@ -1,5 +1,7 @@
package command
//go:generate go run ../scripts/generate-plugins.go
import (
"bytes"
"fmt"

View File

@ -59,6 +59,12 @@ func init() {
CheckFunc: commandVersionCheck,
}, nil
},
"plugin": func() (cli.Command, error) {
return &command.PluginCommand{
Meta: *CommandMeta,
}, nil
},
}
}

View File

@ -2,6 +2,7 @@ package main
import (
"encoding/json"
"fmt"
"io"
"log"
"os/exec"
@ -10,10 +11,15 @@ import (
"strings"
"github.com/mitchellh/osext"
"github.com/mitchellh/packer/command"
"github.com/mitchellh/packer/packer"
"github.com/mitchellh/packer/packer/plugin"
)
// PACKERSPACE is used to represent the spaces that separate args for a command
// without being confused with spaces in the path to the command itself.
const PACKERSPACE = "-PACKERSPACE-"
type config struct {
DisableCheckpoint bool `json:"disable_checkpoint"`
DisableCheckpointSignature bool `json:"disable_checkpoint_signature"`
@ -73,11 +79,17 @@ func (c *config) Discover() error {
}
}
// Last, look in the CWD.
// Next, look in the CWD.
if err := c.discover("."); err != nil {
return err
}
// Finally, try to use an internal plugin. Note that this will not override
// any previously-loaded plugins.
if err := c.discoverInternal(); err != nil {
return err
}
return nil
}
@ -196,6 +208,46 @@ func (c *config) discoverSingle(glob string, m *map[string]string) error {
return nil
}
func (c *config) discoverInternal() error {
// Get the packer binary path
packerPath, err := osext.Executable()
if err != nil {
log.Printf("[ERR] Error loading exe directory: %s", err)
return err
}
for builder := range command.Builders {
_, found := (c.Builders)[builder]
if !found {
log.Printf("Using internal plugin for %s", builder)
(c.Builders)[builder] = fmt.Sprintf("%s%splugin%spacker-builder-%s",
packerPath, PACKERSPACE, PACKERSPACE, builder)
}
}
for provisioner := range command.Provisioners {
_, found := (c.Provisioners)[provisioner]
if !found {
log.Printf("Using internal plugin for %s", provisioner)
(c.Provisioners)[provisioner] = fmt.Sprintf(
"%s%splugin%spacker-provisioner-%s",
packerPath, PACKERSPACE, PACKERSPACE, provisioner)
}
}
for postProcessor := range command.PostProcessors {
_, found := (c.PostProcessors)[postProcessor]
if !found {
log.Printf("Using internal plugin for %s", postProcessor)
(c.PostProcessors)[postProcessor] = fmt.Sprintf(
"%s%splugin%spacker-post-processor-%s",
packerPath, PACKERSPACE, PACKERSPACE, postProcessor)
}
}
return nil
}
func (c *config) pluginClient(path string) *plugin.Client {
originalPath := path
@ -214,6 +266,14 @@ func (c *config) pluginClient(path string) *plugin.Client {
}
}
// Check for special case using `packer plugin PLUGIN`
args := []string{}
if strings.Contains(path, PACKERSPACE) {
parts := strings.Split(path, PACKERSPACE)
path = parts[0]
args = parts[1:]
}
// If everything failed, just use the original path and let the error
// bubble through.
if path == "" {
@ -222,7 +282,7 @@ func (c *config) pluginClient(path string) *plugin.Client {
log.Printf("Creating plugin client for path: %s", path)
var config plugin.ClientConfig
config.Cmd = exec.Command(path)
config.Cmd = exec.Command(path, args...)
config.Managed = true
config.MinPort = c.PluginMinPort
config.MaxPort = c.PluginMaxPort

23
main.go
View File

@ -181,7 +181,7 @@ func wrappedMain() int {
cli := &cli.CLI{
Args: args,
Commands: Commands,
HelpFunc: cli.BasicHelpFunc("packer"),
HelpFunc: excludeHelpFunc(Commands, []string{"plugin"}),
HelpWriter: os.Stdout,
Version: Version,
}
@ -195,6 +195,27 @@ func wrappedMain() int {
return exitCode
}
// excludeHelpFunc filters commands we don't want to show from the list of
// commands displayed in packer's help text.
func excludeHelpFunc(commands map[string]cli.CommandFactory, exclude []string) cli.HelpFunc {
// Make search slice into a map so we can use use the `if found` idiom
// instead of a nested loop.
var excludes = make(map[string]interface{}, len(exclude))
for _, item := range exclude {
excludes[item] = nil
}
// Create filtered list of commands
helpCommands := []string{}
for command := range commands {
if _, found := excludes[command]; !found {
helpCommands = append(helpCommands, command)
}
}
return cli.FilteredHelpFunc(helpCommands, cli.BasicHelpFunc("packer"))
}
// extractMachineReadable checks the args for the machine readable
// flag and returns whether or not it is on. It modifies the args
// to remove this flag.

View File

@ -3,9 +3,36 @@ package main
import (
"math/rand"
"reflect"
"strings"
"testing"
"github.com/mitchellh/cli"
"github.com/mitchellh/packer/command"
)
func TestExcludeHelpFunc(t *testing.T) {
commands := map[string]cli.CommandFactory{
"build": func() (cli.Command, error) {
return &command.BuildCommand{
Meta: command.Meta{},
}, nil
},
"fix": func() (cli.Command, error) {
return &command.FixCommand{
Meta: command.Meta{},
}, nil
},
}
helpFunc := excludeHelpFunc(commands, []string{"fix"})
helpText := helpFunc(commands)
if strings.Contains(helpText, "fix") {
t.Fatalf("Found fix in help text even though we excluded it: \n\n%s\n\n", helpText)
}
}
func TestExtractMachineReadable(t *testing.T) {
var args, expected, result []string
var mr bool

View File

@ -42,18 +42,10 @@ gox \
-os="${XC_OS}" \
-arch="${XC_ARCH}" \
-ldflags "-X main.GitCommit ${GIT_COMMIT}${GIT_DIRTY}" \
-output "pkg/{{.OS}}_{{.Arch}}/packer-{{.Dir}}" \
./...
-output "pkg/{{.OS}}_{{.Arch}}/packer" \
.
set -e
# Make sure "packer-packer" is renamed properly
for PLATFORM in $(find ./pkg -mindepth 1 -maxdepth 1 -type d); do
set +e
mv ${PLATFORM}/packer-packer.exe ${PLATFORM}/packer.exe 2>/dev/null
mv ${PLATFORM}/packer-packer ${PLATFORM}/packer 2>/dev/null
set -e
done
# Move all the compiled things to the $GOPATH/bin
GOPATH=${GOPATH:-$(go env GOPATH)}
case $(uname) in

335
scripts/generate-plugins.go Normal file
View File

@ -0,0 +1,335 @@
// Generate Plugins is a small program that updates the lists of plugins in
// command/plugin.go so they will be compiled into the main packer binary.
//
// See https://github.com/mitchellh/packer/pull/2608 for details.
package main
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strings"
)
const target = "command/plugin.go"
func main() {
// Normally this is run via go:generate from the command folder so we need
// to cd .. first. But when developing it's easier to use go run, so we'll
// support that too.
wd, _ := os.Getwd()
if filepath.Base(wd) != "packer" {
os.Chdir("..")
wd, _ = os.Getwd()
if filepath.Base(wd) != "packer" {
log.Fatalf("This program must be invoked in the packer project root; in %s", wd)
}
}
// Collect all of the data we need about plugins we have in the project
builders, err := discoverBuilders()
if err != nil {
log.Fatalf("Failed to discover builders: %s", err)
}
provisioners, err := discoverProvisioners()
if err != nil {
log.Fatalf("Failed to discover provisioners: %s", err)
}
postProcessors, err := discoverPostProcessors()
if err != nil {
log.Fatalf("Failed to discover post processors: %s", err)
}
// Do some simple code generation and templating
output := source
output = strings.Replace(output, "IMPORTS", makeImports(builders, provisioners, postProcessors), 1)
output = strings.Replace(output, "BUILDERS", makeMap("Builders", "Builder", builders), 1)
output = strings.Replace(output, "PROVISIONERS", makeMap("Provisioners", "Provisioner", provisioners), 1)
output = strings.Replace(output, "POSTPROCESSORS", makeMap("PostProcessors", "PostProcessor", postProcessors), 1)
// TODO sort the lists of plugins so we are not subjected to random OS ordering of the plugin lists
// TODO format the file
// Write our generated code to the command/plugin.go file
file, err := os.Create(target)
defer file.Close()
if err != nil {
log.Fatalf("Failed to open %s for writing: %s", target, err)
}
_, err = file.WriteString(output)
if err != nil {
log.Fatalf("Failed writing to %s: %s", target, err)
}
log.Printf("Generated %s", target)
}
type plugin struct {
Package string // This plugin's package name (iso)
PluginName string // Name of plugin (vmware-iso)
TypeName string // Type of plugin (builder)
Path string // Path relative to packer root (builder/vmware/iso)
ImportName string // PluginName+TypeName (vmwareisobuilder)
}
// makeMap creates a map named Name with type packer.Name that looks something
// like this:
//
// var Builders = map[string]packer.Builder{
// "amazon-chroot": new(chroot.Builder),
// "amazon-ebs": new(ebs.Builder),
// "amazon-instance": new(instance.Builder),
func makeMap(varName, varType string, items []plugin) string {
output := ""
output += fmt.Sprintf("var %s = map[string]packer.%s{\n", varName, varType)
for _, item := range items {
output += fmt.Sprintf("\t\"%s\": new(%s.%s),\n", item.PluginName, item.ImportName, item.TypeName)
}
output += "}\n"
return output
}
func makeImports(builders, provisioners, postProcessors []plugin) string {
plugins := []string{}
for _, builder := range builders {
plugins = append(plugins, fmt.Sprintf("\t%s \"github.com/mitchellh/packer/%s\"\n", builder.ImportName, builder.Path))
}
for _, provisioner := range provisioners {
plugins = append(plugins, fmt.Sprintf("\t%s \"github.com/mitchellh/packer/%s\"\n", provisioner.ImportName, provisioner.Path))
}
for _, postProcessor := range postProcessors {
plugins = append(plugins, fmt.Sprintf("\t%s \"github.com/mitchellh/packer/%s\"\n", postProcessor.ImportName, postProcessor.Path))
}
// Make things pretty
sort.Strings(plugins)
return strings.Join(plugins, "")
}
// listDirectories recursively lists directories under the specified path
func listDirectories(path string) ([]string, error) {
names := []string{}
items, err := ioutil.ReadDir(path)
if err != nil {
return names, err
}
for _, item := range items {
// We only want directories
if item.IsDir() {
currentDir := filepath.Join(path, item.Name())
names = append(names, currentDir)
// Do some recursion
subNames, err := listDirectories(currentDir)
if err == nil {
names = append(names, subNames...)
}
}
}
return names, nil
}
// deriveName determines the name of the plugin (what you'll see in a packer
// template) based on the filesystem path. We use two rules:
//
// Start with -> builder/virtualbox/iso
//
// 1. Strip the root -> virtualbox/iso
// 2. Switch slash / to dash - -> virtualbox-iso
func deriveName(root, full string) string {
short, _ := filepath.Rel(root, full)
bits := strings.Split(short, string(os.PathSeparator))
return strings.Join(bits, "-")
}
// deriveImport will build a unique import identifier based on packageName and
// the result of deriveName()
//
// This will be something like -> virtualboxisobuilder
//
// Which is long, but deterministic and unique.
func deriveImport(typeName, derivedName string) string {
return strings.Replace(derivedName, "-", "", -1) + strings.ToLower(typeName)
}
// discoverTypesInPath searches for types of typeID in path and returns a list
// of plugins it finds.
func discoverTypesInPath(path, typeID string) ([]plugin, error) {
postProcessors := []plugin{}
dirs, err := listDirectories(path)
if err != nil {
return postProcessors, err
}
for _, dir := range dirs {
fset := token.NewFileSet()
goPackages, err := parser.ParseDir(fset, dir, nil, parser.AllErrors)
if err != nil {
return postProcessors, fmt.Errorf("Failed parsing directory %s: %s", dir, err)
}
for _, goPackage := range goPackages {
ast.PackageExports(goPackage)
ast.Inspect(goPackage, func(n ast.Node) bool {
switch x := n.(type) {
case *ast.TypeSpec:
if x.Name.Name == typeID {
derivedName := deriveName(path, dir)
postProcessors = append(postProcessors, plugin{
Package: goPackage.Name,
PluginName: derivedName,
ImportName: deriveImport(x.Name.Name, derivedName),
TypeName: x.Name.Name,
Path: dir,
})
// The AST stops parsing when we return false. Once we
// find the symbol we want we can stop parsing.
// DEBUG:
// fmt.Printf("package %#v\n", goPackage)
return false
}
}
return true
})
}
}
return postProcessors, nil
}
func discoverBuilders() ([]plugin, error) {
path := "./builder"
typeID := "Builder"
return discoverTypesInPath(path, typeID)
}
func discoverProvisioners() ([]plugin, error) {
path := "./provisioner"
typeID := "Provisioner"
return discoverTypesInPath(path, typeID)
}
func discoverPostProcessors() ([]plugin, error) {
path := "./post-processor"
typeID := "PostProcessor"
return discoverTypesInPath(path, typeID)
}
const source = `//
// This file is automatically generated by scripts/generate-plugins.go -- Do not edit!
//
package command
import (
"fmt"
"log"
"regexp"
"strings"
"github.com/mitchellh/packer/packer"
"github.com/mitchellh/packer/packer/plugin"
IMPORTS
)
type PluginCommand struct {
Meta
}
BUILDERS
PROVISIONERS
POSTPROCESSORS
var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")
func (c *PluginCommand) Run(args []string) int {
// This is an internal call (users should not call this directly) so we're
// not going to do much input validation. If there's a problem we'll often
// just crash. Error handling should be added to facilitate debugging.
log.Printf("args: %#v", args)
if len(args) != 1 {
c.Ui.Error("Wrong number of args")
return 1
}
// Plugin will match something like "packer-builder-amazon-ebs"
parts := pluginRegexp.FindStringSubmatch(args[0])
if len(parts) != 3 {
c.Ui.Error(fmt.Sprintf("Error parsing plugin argument [DEBUG]: %#v", parts))
return 1
}
pluginType := parts[1] // capture group 1 (builder|post-processor|provisioner)
pluginName := parts[2] // capture group 2 (.+)
server, err := plugin.Server()
if err != nil {
c.Ui.Error(fmt.Sprintf("Error starting plugin server: %s", err))
return 1
}
switch pluginType {
case "builder":
builder, found := Builders[pluginName]
if !found {
c.Ui.Error(fmt.Sprintf("Could not load builder: %s", pluginName))
return 1
}
server.RegisterBuilder(builder)
case "provisioner":
provisioner, found := Provisioners[pluginName]
if !found {
c.Ui.Error(fmt.Sprintf("Could not load provisioner: %s", pluginName))
return 1
}
server.RegisterProvisioner(provisioner)
case "post-processor":
postProcessor, found := PostProcessors[pluginName]
if !found {
c.Ui.Error(fmt.Sprintf("Could not load post-processor: %s", pluginName))
return 1
}
server.RegisterPostProcessor(postProcessor)
}
server.Serve()
return 0
}
func (*PluginCommand) Help() string {
helpText := ` + "`" + `
Usage: packer plugin PLUGIN
Runs an internally-compiled version of a plugin from the packer binary.
NOTE: this is an internal command and you should not call it yourself.
` + "`" + `
return strings.TrimSpace(helpText)
}
func (c *PluginCommand) Synopsis() string {
return "internal plugin command"
}
`