Merge pull request #10336 from hashicorp/move_plugin_discover

Move plugin discover to plugin package
This commit is contained in:
Megan Marsh 2020-12-03 10:09:37 -08:00 committed by GitHub
commit 2667bf3d3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 403 additions and 225 deletions

133
config.go
View File

@ -6,7 +6,6 @@ import (
"io"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
@ -23,16 +22,15 @@ import (
const PACKERSPACE = "-PACKERSPACE-"
type config struct {
DisableCheckpoint bool `json:"disable_checkpoint"`
DisableCheckpointSignature bool `json:"disable_checkpoint_signature"`
PluginMinPort int
PluginMaxPort int
DisableCheckpoint bool `json:"disable_checkpoint"`
DisableCheckpointSignature bool `json:"disable_checkpoint_signature"`
RawBuilders map[string]string `json:"builders"`
RawProvisioners map[string]string `json:"provisioners"`
RawPostProcessors map[string]string `json:"post-processors"`
Builders packer.MapOfBuilder `json:"-"`
Provisioners packer.MapOfProvisioner `json:"-"`
PostProcessors packer.MapOfPostProcessor `json:"-"`
Plugins plugin.Config
}
// decodeConfig decodes configuration in JSON format from the given io.Reader into
@ -99,87 +97,23 @@ func (c *config) loadSingleComponent(path string) (string, error) {
case strings.HasPrefix(pluginName, "packer-builder-"):
pluginName = pluginName[len("packer-builder-"):]
c.Builders[pluginName] = func() (packersdk.Builder, error) {
return c.pluginClient(path).Builder()
return c.Plugins.Client(path).Builder()
}
case strings.HasPrefix(pluginName, "packer-post-processor-"):
pluginName = pluginName[len("packer-post-processor-"):]
c.PostProcessors[pluginName] = func() (packersdk.PostProcessor, error) {
return c.pluginClient(path).PostProcessor()
return c.Plugins.Client(path).PostProcessor()
}
case strings.HasPrefix(pluginName, "packer-provisioner-"):
pluginName = pluginName[len("packer-provisioner-"):]
c.Provisioners[pluginName] = func() (packersdk.Provisioner, error) {
return c.pluginClient(path).Provisioner()
return c.Plugins.Client(path).Provisioner()
}
}
return pluginName, nil
}
// Discover discovers plugins.
//
// Search the directory of the executable, then the plugins directory, and
// finally the CWD, in that order. Any conflicts will overwrite previously
// found plugins, in that order.
// Hence, the priority order is the reverse of the search order - i.e., the
// CWD has the highest priority.
func (c *config) Discover() error {
// If we are already inside a plugin process we should not need to
// discover anything.
if os.Getenv(plugin.MagicCookieKey) == plugin.MagicCookieValue {
return nil
}
// Next, look in the same directory as the executable.
exePath, err := os.Executable()
if err != nil {
log.Printf("[ERR] Error loading exe directory: %s", err)
} else {
if err := c.discoverExternalComponents(filepath.Dir(exePath)); err != nil {
return err
}
}
// Next, look in the default plugins directory inside the configdir/.packer.d/plugins.
dir, err := packer.ConfigDir()
if err != nil {
log.Printf("[ERR] Error loading config directory: %s", err)
} else {
if err := c.discoverExternalComponents(filepath.Join(dir, "plugins")); err != nil {
return err
}
}
// Next, look in the CWD.
if err := c.discoverExternalComponents("."); err != nil {
return err
}
// Check whether there is a custom Plugin directory defined. This gets
// absolute preference.
if packerPluginPath := os.Getenv("PACKER_PLUGIN_PATH"); packerPluginPath != "" {
sep := ":"
if runtime.GOOS == "windows" {
// on windows, PATH is semicolon-separated
sep = ";"
}
plugPaths := strings.Split(packerPluginPath, sep)
for _, plugPath := range plugPaths {
if err := c.discoverExternalComponents(plugPath); err != nil {
return err
}
}
}
// Finally, try to use an internal plugin. Note that this will not override
// any previously-loaded plugins.
if err := c.discoverInternalComponents(); err != nil {
return err
}
return nil
}
// This is a proper packer.BuilderFunc that can be used to load packersdk.Builder
// implementations from the defined plugins.
func (c *config) StartBuilder(name string) (packersdk.Builder, error) {
@ -191,7 +125,7 @@ func (c *config) StartBuilder(name string) (packersdk.Builder, error) {
// to load packersdk.Hook implementations from the defined plugins.
func (c *config) StarHook(name string) (packersdk.Hook, error) {
log.Printf("Loading hook: %s\n", name)
return c.pluginClient(name).Hook()
return c.Plugins.Client(name).Hook()
}
// This is a proper packersdk.PostProcessorFunc that can be used to load
@ -226,7 +160,7 @@ func (c *config) discoverExternalComponents(path string) error {
for pluginName, pluginPath := range pluginPaths {
newPath := pluginPath // this needs to be stored in a new variable for the func below
c.Builders[pluginName] = func() (packersdk.Builder, error) {
return c.pluginClient(newPath).Builder()
return c.Plugins.Client(newPath).Builder()
}
externallyUsed = append(externallyUsed, pluginName)
}
@ -243,7 +177,7 @@ func (c *config) discoverExternalComponents(path string) error {
for pluginName, pluginPath := range pluginPaths {
newPath := pluginPath // this needs to be stored in a new variable for the func below
c.PostProcessors[pluginName] = func() (packersdk.PostProcessor, error) {
return c.pluginClient(newPath).PostProcessor()
return c.Plugins.Client(newPath).PostProcessor()
}
externallyUsed = append(externallyUsed, pluginName)
}
@ -260,7 +194,7 @@ func (c *config) discoverExternalComponents(path string) error {
for pluginName, pluginPath := range pluginPaths {
newPath := pluginPath // this needs to be stored in a new variable for the func below
c.Provisioners[pluginName] = func() (packersdk.Provisioner, error) {
return c.pluginClient(newPath).Provisioner()
return c.Plugins.Client(newPath).Provisioner()
}
externallyUsed = append(externallyUsed, pluginName)
}
@ -324,7 +258,7 @@ func (c *config) discoverInternalComponents() error {
c.Builders[builder] = func() (packersdk.Builder, error) {
bin := fmt.Sprintf("%s%splugin%spacker-builder-%s",
packerPath, PACKERSPACE, PACKERSPACE, builder)
return c.pluginClient(bin).Builder()
return c.Plugins.Client(bin).Builder()
}
}
}
@ -336,7 +270,7 @@ func (c *config) discoverInternalComponents() error {
c.Provisioners[provisioner] = func() (packersdk.Provisioner, error) {
bin := fmt.Sprintf("%s%splugin%spacker-provisioner-%s",
packerPath, PACKERSPACE, PACKERSPACE, provisioner)
return c.pluginClient(bin).Provisioner()
return c.Plugins.Client(bin).Provisioner()
}
}
}
@ -348,51 +282,10 @@ func (c *config) discoverInternalComponents() error {
c.PostProcessors[postProcessor] = func() (packersdk.PostProcessor, error) {
bin := fmt.Sprintf("%s%splugin%spacker-post-processor-%s",
packerPath, PACKERSPACE, PACKERSPACE, postProcessor)
return c.pluginClient(bin).PostProcessor()
return c.Plugins.Client(bin).PostProcessor()
}
}
}
return nil
}
func (c *config) pluginClient(path string) *plugin.Client {
originalPath := path
// Check for special case using `packer plugin PLUGIN`
args := []string{}
if strings.Contains(path, PACKERSPACE) {
parts := strings.Split(path, PACKERSPACE)
path = parts[0]
args = parts[1:]
}
// First attempt to find the executable by consulting the PATH.
path, err := exec.LookPath(path)
if err != nil {
// If that doesn't work, look for it in the same directory
// as the `packer` executable (us).
log.Printf("Plugin could not be found at %s (%v). Checking same directory as executable.", originalPath, err)
exePath, err := os.Executable()
if err != nil {
log.Printf("Couldn't get current exe path: %s", err)
} else {
log.Printf("Current exe path: %s", exePath)
path = filepath.Join(filepath.Dir(exePath), filepath.Base(originalPath))
}
}
// If everything failed, just use the original path and let the error
// bubble through.
if path == "" {
path = originalPath
}
log.Printf("Creating plugin client for path: %s", path)
var config plugin.ClientConfig
config.Cmd = exec.Command(path, args...)
config.Managed = true
config.MinPort = c.PluginMinPort
config.MaxPort = c.PluginMaxPort
return plugin.NewClient(&config)
}

View File

@ -12,107 +12,8 @@ import (
"testing"
"github.com/hashicorp/packer/packer"
"github.com/hashicorp/packer/packer/plugin"
)
func newConfig() config {
var conf config
conf.PluginMinPort = 10000
conf.PluginMaxPort = 25000
conf.Builders = packer.MapOfBuilder{}
conf.PostProcessors = packer.MapOfPostProcessor{}
conf.Provisioners = packer.MapOfProvisioner{}
return conf
}
func TestDiscoverReturnsIfMagicCookieSet(t *testing.T) {
config := newConfig()
os.Setenv(plugin.MagicCookieKey, plugin.MagicCookieValue)
defer os.Unsetenv(plugin.MagicCookieKey)
err := config.Discover()
if err != nil {
t.Fatalf("Should not have errored: %s", err)
}
if len(config.Builders) != 0 {
t.Fatalf("Should not have tried to find builders")
}
}
func TestEnvVarPackerPluginPath(t *testing.T) {
// Create a temporary directory to store plugins in
dir, _, cleanUpFunc, err := generateFakePlugins("custom_plugin_dir",
[]string{"packer-provisioner-partyparrot"})
if err != nil {
t.Fatalf("Error creating fake custom plugins: %s", err)
}
defer cleanUpFunc()
// Add temp dir to path.
os.Setenv("PACKER_PLUGIN_PATH", dir)
defer os.Unsetenv("PACKER_PLUGIN_PATH")
config := newConfig()
err = config.Discover()
if err != nil {
t.Fatalf("Should not have errored: %s", err)
}
if len(config.Provisioners) == 0 {
t.Fatalf("Should have found partyparrot provisioner")
}
if _, ok := config.Provisioners["partyparrot"]; !ok {
t.Fatalf("Should have found partyparrot provisioner.")
}
}
func TestEnvVarPackerPluginPath_MultiplePaths(t *testing.T) {
// Create a temporary directory to store plugins in
dir, _, cleanUpFunc, err := generateFakePlugins("custom_plugin_dir",
[]string{"packer-provisioner-partyparrot"})
if err != nil {
t.Fatalf("Error creating fake custom plugins: %s", err)
}
defer cleanUpFunc()
pathsep := ":"
if runtime.GOOS == "windows" {
pathsep = ";"
}
// Create a second dir to look in that will be empty
decoyDir, err := ioutil.TempDir("", "decoy")
if err != nil {
t.Fatalf("Failed to create a temporary test dir.")
}
defer os.Remove(decoyDir)
pluginPath := dir + pathsep + decoyDir
// Add temp dir to path.
os.Setenv("PACKER_PLUGIN_PATH", pluginPath)
defer os.Unsetenv("PACKER_PLUGIN_PATH")
config := newConfig()
err = config.Discover()
if err != nil {
t.Fatalf("Should not have errored: %s", err)
}
if len(config.Provisioners) == 0 {
t.Fatalf("Should have found partyparrot provisioner")
}
if _, ok := config.Provisioners["partyparrot"]; !ok {
t.Fatalf("Should have found partyparrot provisioner.")
}
}
func TestDecodeConfig(t *testing.T) {
packerConfig := `

23
main.go
View File

@ -300,12 +300,23 @@ func extractMachineReadable(args []string) ([]string, bool) {
func loadConfig() (*config, error) {
var config config
config.PluginMinPort = 10000
config.PluginMaxPort = 25000
config.Builders = packer.MapOfBuilder{}
config.PostProcessors = packer.MapOfPostProcessor{}
config.Provisioners = packer.MapOfProvisioner{}
if err := config.Discover(); err != nil {
config.Plugins = plugin.Config{
PluginMinPort: 10000,
PluginMaxPort: 25000,
}
if err := config.Plugins.Discover(); err != nil {
return nil, err
}
// Copy plugins to general list
builders, provisioners, postProcessors := config.Plugins.GetPlugins()
config.Builders = builders
config.Provisioners = provisioners
config.PostProcessors = postProcessors
// Finally, try to use an internal plugin. Note that this will not override
// any previously-loaded plugins.
if err := config.discoverInternalComponents(); err != nil {
return nil, err
}

239
packer/plugin/discover.go Normal file
View File

@ -0,0 +1,239 @@
package plugin
import (
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
"strings"
"github.com/hashicorp/packer/packer"
packersdk "github.com/hashicorp/packer/packer-plugin-sdk/packer"
)
// PACKERSPACE is used to represent the spaces that separate args for a command
// without being confused with spaces in the path to the command itself.
const PACKERSPACE = "-PACKERSPACE-"
type Config struct {
PluginMinPort int
PluginMaxPort int
builders packer.MapOfBuilder
provisioners packer.MapOfProvisioner
postProcessors packer.MapOfPostProcessor
}
func (c *Config) GetPlugins() (packer.MapOfBuilder, packer.MapOfProvisioner, packer.MapOfPostProcessor) {
return c.builders, c.provisioners, c.postProcessors
}
// Discover discovers plugins.
//
// Search the directory of the executable, then the plugins directory, and
// finally the CWD, in that order. Any conflicts will overwrite previously
// found plugins, in that order.
// Hence, the priority order is the reverse of the search order - i.e., the
// CWD has the highest priority.
func (c *Config) Discover() error {
if c.builders == nil {
c.builders = packer.MapOfBuilder{}
}
if c.provisioners == nil {
c.provisioners = packer.MapOfProvisioner{}
}
if c.postProcessors == nil {
c.postProcessors = packer.MapOfPostProcessor{}
}
// If we are already inside a plugin process we should not need to
// discover anything.
if os.Getenv(MagicCookieKey) == MagicCookieValue {
return nil
}
// Next, look in the same directory as the executable.
exePath, err := os.Executable()
if err != nil {
log.Printf("[ERR] Error loading exe directory: %s", err)
} else {
if err := c.discoverExternalComponents(filepath.Dir(exePath)); err != nil {
return err
}
}
// Next, look in the default plugins directory inside the configdir/.packer.d/plugins.
dir, err := packer.ConfigDir()
if err != nil {
log.Printf("[ERR] Error loading config directory: %s", err)
} else {
if err := c.discoverExternalComponents(filepath.Join(dir, "plugins")); err != nil {
return err
}
}
// Next, look in the CWD.
if err := c.discoverExternalComponents("."); err != nil {
return err
}
// Check whether there is a custom Plugin directory defined. This gets
// absolute preference.
if packerPluginPath := os.Getenv("PACKER_PLUGIN_PATH"); packerPluginPath != "" {
sep := ":"
if runtime.GOOS == "windows" {
// on windows, PATH is semicolon-separated
sep = ";"
}
plugPaths := strings.Split(packerPluginPath, sep)
for _, plugPath := range plugPaths {
if err := c.discoverExternalComponents(plugPath); err != nil {
return err
}
}
}
return nil
}
func (c *Config) discoverExternalComponents(path string) error {
var err error
if !filepath.IsAbs(path) {
path, err = filepath.Abs(path)
if err != nil {
return err
}
}
var externallyUsed []string
pluginPaths, err := c.discoverSingle(filepath.Join(path, "packer-builder-*"))
if err != nil {
return err
}
for pluginName, pluginPath := range pluginPaths {
newPath := pluginPath // this needs to be stored in a new variable for the func below
c.builders[pluginName] = func() (packersdk.Builder, error) {
return c.Client(newPath).Builder()
}
externallyUsed = append(externallyUsed, pluginName)
}
if len(externallyUsed) > 0 {
sort.Strings(externallyUsed)
log.Printf("using external builders %v", externallyUsed)
externallyUsed = nil
}
pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-post-processor-*"))
if err != nil {
return err
}
for pluginName, pluginPath := range pluginPaths {
newPath := pluginPath // this needs to be stored in a new variable for the func below
c.postProcessors[pluginName] = func() (packersdk.PostProcessor, error) {
return c.Client(newPath).PostProcessor()
}
externallyUsed = append(externallyUsed, pluginName)
}
if len(externallyUsed) > 0 {
sort.Strings(externallyUsed)
log.Printf("using external post-processors %v", externallyUsed)
externallyUsed = nil
}
pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-provisioner-*"))
if err != nil {
return err
}
for pluginName, pluginPath := range pluginPaths {
newPath := pluginPath // this needs to be stored in a new variable for the func below
c.provisioners[pluginName] = func() (packersdk.Provisioner, error) {
return c.Client(newPath).Provisioner()
}
externallyUsed = append(externallyUsed, pluginName)
}
if len(externallyUsed) > 0 {
sort.Strings(externallyUsed)
log.Printf("using external provisioners %v", externallyUsed)
}
return nil
}
func (c *Config) discoverSingle(glob string) (map[string]string, error) {
matches, err := filepath.Glob(glob)
if err != nil {
return nil, err
}
res := make(map[string]string)
prefix := filepath.Base(glob)
prefix = prefix[:strings.Index(prefix, "*")]
for _, match := range matches {
file := filepath.Base(match)
// On Windows, ignore any plugins that don't end in .exe.
// We could do a full PATHEXT parse, but this is probably good enough.
if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" {
log.Printf(
"[DEBUG] Ignoring plugin match %s, no exe extension",
match)
continue
}
// If the filename has a ".", trim up to there
if idx := strings.Index(file, ".exe"); idx >= 0 {
file = file[:idx]
}
// Look for foo-bar-baz. The plugin name is "baz"
pluginName := file[len(prefix):]
log.Printf("[DEBUG] Discovered plugin: %s = %s", pluginName, match)
res[pluginName] = match
}
return res, nil
}
func (c *Config) Client(path string) *Client {
originalPath := path
// Check for special case using `packer plugin PLUGIN`
args := []string{}
if strings.Contains(path, PACKERSPACE) {
parts := strings.Split(path, PACKERSPACE)
path = parts[0]
args = parts[1:]
}
// First attempt to find the executable by consulting the PATH.
path, err := exec.LookPath(path)
if err != nil {
// If that doesn't work, look for it in the same directory
// as the `packer` executable (us).
log.Printf("Plugin could not be found at %s (%v). Checking same directory as executable.", originalPath, err)
exePath, err := os.Executable()
if err != nil {
log.Printf("Couldn't get current exe path: %s", err)
} else {
log.Printf("Current exe path: %s", exePath)
path = filepath.Join(filepath.Dir(exePath), filepath.Base(originalPath))
}
}
// If everything failed, just use the original path and let the error
// bubble through.
if path == "" {
path = originalPath
}
log.Printf("Creating plugin client for path: %s", path)
var config ClientConfig
config.Cmd = exec.Command(path, args...)
config.Managed = true
config.MinPort = c.PluginMinPort
config.MaxPort = c.PluginMaxPort
return NewClient(&config)
}

View File

@ -0,0 +1,134 @@
package plugin
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"testing"
)
func newConfig() Config {
var conf Config
conf.PluginMinPort = 10000
conf.PluginMaxPort = 25000
return conf
}
func TestDiscoverReturnsIfMagicCookieSet(t *testing.T) {
config := newConfig()
os.Setenv(MagicCookieKey, MagicCookieValue)
defer os.Unsetenv(MagicCookieKey)
err := config.Discover()
if err != nil {
t.Fatalf("Should not have errored: %s", err)
}
if len(config.builders) != 0 {
t.Fatalf("Should not have tried to find builders")
}
}
func TestEnvVarPackerPluginPath(t *testing.T) {
// Create a temporary directory to store plugins in
dir, _, cleanUpFunc, err := generateFakePlugins("custom_plugin_dir",
[]string{"packer-provisioner-partyparrot"})
if err != nil {
t.Fatalf("Error creating fake custom plugins: %s", err)
}
defer cleanUpFunc()
// Add temp dir to path.
os.Setenv("PACKER_PLUGIN_PATH", dir)
defer os.Unsetenv("PACKER_PLUGIN_PATH")
config := newConfig()
err = config.Discover()
if err != nil {
t.Fatalf("Should not have errored: %s", err)
}
if len(config.provisioners) == 0 {
t.Fatalf("Should have found partyparrot provisioner")
}
if _, ok := config.provisioners["partyparrot"]; !ok {
t.Fatalf("Should have found partyparrot provisioner.")
}
}
func TestEnvVarPackerPluginPath_MultiplePaths(t *testing.T) {
// Create a temporary directory to store plugins in
dir, _, cleanUpFunc, err := generateFakePlugins("custom_plugin_dir",
[]string{"packer-provisioner-partyparrot"})
if err != nil {
t.Fatalf("Error creating fake custom plugins: %s", err)
}
defer cleanUpFunc()
pathsep := ":"
if runtime.GOOS == "windows" {
pathsep = ";"
}
// Create a second dir to look in that will be empty
decoyDir, err := ioutil.TempDir("", "decoy")
if err != nil {
t.Fatalf("Failed to create a temporary test dir.")
}
defer os.Remove(decoyDir)
pluginPath := dir + pathsep + decoyDir
// Add temp dir to path.
os.Setenv("PACKER_PLUGIN_PATH", pluginPath)
defer os.Unsetenv("PACKER_PLUGIN_PATH")
config := newConfig()
err = config.Discover()
if err != nil {
t.Fatalf("Should not have errored: %s", err)
}
if len(config.provisioners) == 0 {
t.Fatalf("Should have found partyparrot provisioner")
}
if _, ok := config.provisioners["partyparrot"]; !ok {
t.Fatalf("Should have found partyparrot provisioner.")
}
}
func generateFakePlugins(dirname string, pluginNames []string) (string, []string, func(), error) {
dir, err := ioutil.TempDir("", dirname)
if err != nil {
return "", nil, nil, fmt.Errorf("failed to create temporary test directory: %v", err)
}
cleanUpFunc := func() {
os.RemoveAll(dir)
}
var suffix string
if runtime.GOOS == "windows" {
suffix = ".exe"
}
plugins := make([]string, len(pluginNames))
for i, plugin := range pluginNames {
plug := filepath.Join(dir, plugin+suffix)
plugins[i] = plug
_, err := os.Create(plug)
if err != nil {
cleanUpFunc()
return "", nil, nil, fmt.Errorf("failed to create temporary plugin file (%s): %v", plug, err)
}
}
return dir, plugins, cleanUpFunc, nil
}