Merge pull request #2919 from arizvisa/floppy-recurse

Added an option for copying entire subdirectories via floppy_dirs (supplants floppy_files)
This commit is contained in:
Rickard von Essen 2016-10-08 16:51:16 +02:00 committed by GitHub
commit 5e96709ee9
28 changed files with 420 additions and 94 deletions

View File

@ -149,7 +149,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
Path: b.config.OutputDir,
},
&common.StepCreateFloppy{
Files: b.config.FloppyFiles,
Files: b.config.FloppyConfig.FloppyFiles,
Directories: b.config.FloppyConfig.FloppyDirectories,
},
&common.StepHTTPServer{
HTTPDir: b.config.HTTPDir,

View File

@ -59,7 +59,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
Path: b.config.OutputDir,
},
&common.StepCreateFloppy{
Files: b.config.FloppyFiles,
Files: b.config.FloppyConfig.FloppyFiles,
Directories: b.config.FloppyConfig.FloppyDirectories,
},
&StepImport{
Name: b.config.VMName,

View File

@ -93,7 +93,6 @@ type Config struct {
DiskDiscard string `mapstructure:"disk_discard"`
SkipCompaction bool `mapstructure:"skip_compaction"`
DiskCompression bool `mapstructure:"disk_compression"`
FloppyFiles []string `mapstructure:"floppy_files"`
Format string `mapstructure:"format"`
Headless bool `mapstructure:"headless"`
DiskImage bool `mapstructure:"disk_image"`
@ -365,7 +364,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
steps = append(steps, new(stepPrepareOutputDir),
&common.StepCreateFloppy{
Files: b.config.FloppyFiles,
Files: b.config.FloppyConfig.FloppyFiles,
Directories: b.config.FloppyConfig.FloppyDirectories,
},
new(stepCreateDisk),
new(stepCopyDisk),

View File

@ -195,7 +195,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
Path: b.config.OutputDir,
},
&common.StepCreateFloppy{
Files: b.config.FloppyFiles,
Files: b.config.FloppyConfig.FloppyFiles,
Directories: b.config.FloppyConfig.FloppyDirectories,
},
&common.StepHTTPServer{
HTTPDir: b.config.HTTPDir,

View File

@ -56,7 +56,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
},
new(vboxcommon.StepSuppressMessages),
&common.StepCreateFloppy{
Files: b.config.FloppyFiles,
Files: b.config.FloppyConfig.FloppyFiles,
Directories: b.config.FloppyConfig.FloppyDirectories,
},
&common.StepHTTPServer{
HTTPDir: b.config.HTTPDir,

View File

@ -231,7 +231,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
Force: b.config.PackerForce,
},
&common.StepCreateFloppy{
Files: b.config.FloppyFiles,
Files: b.config.FloppyConfig.FloppyFiles,
Directories: b.config.FloppyConfig.FloppyDirectories,
},
&stepRemoteUpload{
Key: "floppy_path",

View File

@ -62,7 +62,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
Force: b.config.PackerForce,
},
&common.StepCreateFloppy{
Files: b.config.FloppyFiles,
Files: b.config.FloppyConfig.FloppyFiles,
Directories: b.config.FloppyConfig.FloppyDirectories,
},
&StepCloneVMX{
OutputDir: b.config.OutputDir,

View File

@ -8,7 +8,8 @@ import (
)
type FloppyConfig struct {
FloppyFiles []string `mapstructure:"floppy_files"`
FloppyFiles []string `mapstructure:"floppy_files"`
FloppyDirectories []string `mapstructure:"floppy_dirs"`
}
func (c *FloppyConfig) Prepare(ctx *interpolate.Context) []error {
@ -24,5 +25,15 @@ func (c *FloppyConfig) Prepare(ctx *interpolate.Context) []error {
}
}
if c.FloppyDirectories == nil {
c.FloppyDirectories = make([]string, 0)
}
for _, path := range c.FloppyDirectories {
if _, err := os.Stat(path); err != nil {
errs = append(errs, fmt.Errorf("Bad Floppy disk directory '%s': %s", path, err))
}
}
return errs
}

View File

@ -10,15 +10,15 @@ import (
"io/ioutil"
"log"
"os"
"path"
"path/filepath"
"strings"
)
// StepCreateFloppy will create a floppy disk with the given files.
// The floppy disk doesn't support sub-directories. Only files at the
// root level are supported.
type StepCreateFloppy struct {
Files []string
Files []string
Directories []string
floppyPath string
@ -26,7 +26,7 @@ type StepCreateFloppy struct {
}
func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction {
if len(s.Files) == 0 {
if len(s.Files) == 0 && len(s.Directories) == 0 {
log.Println("No floppy files specified. Floppy disk will not be made.")
return multistep.ActionContinue
}
@ -84,22 +84,114 @@ func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction {
return multistep.ActionHalt
}
// Get the root directory to the filesystem
// Get the root directory to the filesystem and create a cache for any directories within
log.Println("Reading the root directory from the filesystem")
rootDir, err := fatFs.RootDir()
if err != nil {
state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
return multistep.ActionHalt
}
cache := fsDirectoryCache(rootDir)
// Go over each file and copy it.
for _, filename := range s.Files {
ui.Message(fmt.Sprintf("Copying: %s", filename))
if err := s.addFilespec(rootDir, filename); err != nil {
state.Put("error", fmt.Errorf("Error adding file to floppy: %s", err))
// Utility functions for walking through a directory grabbing all files flatly
globFiles := func(files []string, list chan string) {
for _,filename := range files {
if strings.IndexAny(filename, "*?[") >= 0 {
matches,_ := filepath.Glob(filename)
if err != nil { continue }
for _,match := range matches {
list <- match
}
continue
}
list <- filename
}
close(list)
}
var crawlDirectoryFiles []string
crawlDirectory := func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
crawlDirectoryFiles = append(crawlDirectoryFiles, path)
ui.Message(fmt.Sprintf("Adding file: %s", path))
}
return nil
}
crawlDirectoryFiles = []string{}
// Collect files and copy them flatly...because floppy_files is broken on purpose.
var filelist chan string
filelist = make(chan string)
go globFiles(s.Files, filelist)
ui.Message("Copying files flatly from floppy_files")
for {
filename, ok := <-filelist
if !ok { break }
finfo,err := os.Stat(filename)
if err != nil {
state.Put("error", fmt.Errorf("Error trying to stat : %s : %s", filename, err))
return multistep.ActionHalt
}
// walk through directory adding files to the root of the fs
if finfo.IsDir() {
ui.Message(fmt.Sprintf("Copying directory: %s", filename))
err := filepath.Walk(filename, crawlDirectory)
if err != nil {
state.Put("error", fmt.Errorf("Error adding file from floppy_files : %s : %s", filename, err))
return multistep.ActionHalt
}
for _,crawlfilename := range crawlDirectoryFiles {
s.Add(cache, crawlfilename)
s.FilesAdded[crawlfilename] = true
}
crawlDirectoryFiles = []string{}
continue
}
// add just a single file
ui.Message(fmt.Sprintf("Copying file: %s", filename))
s.Add(cache, filename)
s.FilesAdded[filename] = true
}
ui.Message("Done copying files from floppy_files")
// Collect all paths (expanding wildcards) into pathqueue
ui.Message("Collecting paths from floppy_dirs")
var pathqueue []string
for _,filename := range s.Directories {
if strings.IndexAny(filename, "*?[") >= 0 {
matches,err := filepath.Glob(filename)
if err != nil {
state.Put("error", fmt.Errorf("Error adding path %s to floppy: %s", filename, err))
return multistep.ActionHalt
}
for _,filename := range matches {
pathqueue = append(pathqueue, filename)
}
continue
}
pathqueue = append(pathqueue, filename)
}
ui.Message(fmt.Sprintf("Resulting paths from floppy_dirs : %v", pathqueue))
// Go over each path in pathqueue and copy it.
for _,src := range pathqueue {
ui.Message(fmt.Sprintf("Recursively copying : %s", src))
err = s.Add(cache, src)
if err != nil {
state.Put("error", fmt.Errorf("Error adding path %s to floppy: %s", src, err))
return multistep.ActionHalt
}
}
ui.Message("Done copying paths from floppy_dirs")
// Set the path to the floppy so it can be used later
state.Put("floppy_path", s.floppyPath)
@ -107,6 +199,68 @@ func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction {
return multistep.ActionContinue
}
func (s *StepCreateFloppy) Add(dircache directoryCache, src string) error {
finfo,err := os.Stat(src)
if err != nil {
return fmt.Errorf("Error adding path to floppy: %s", err)
}
// add a file
if !finfo.IsDir() {
inputF, err := os.Open(src)
if err != nil { return err }
defer inputF.Close()
d,err := dircache("")
if err != nil { return err }
entry,err := d.AddFile(path.Base(src))
if err != nil { return err }
fatFile,err := entry.File()
if err != nil { return err }
_,err = io.Copy(fatFile,inputF)
s.FilesAdded[src] = true
return err
}
// add a directory and it's subdirectories
basedirectory := filepath.Join(src, "..")
visit := func(pathname string, fi os.FileInfo, err error) error {
if err != nil { return err }
if fi.Mode().IsDir() {
base,err := removeBase(basedirectory, pathname)
if err != nil { return err }
_,err = dircache(filepath.ToSlash(base))
return err
}
directory,filename := filepath.Split(pathname)
base,err := removeBase(basedirectory, directory)
if err != nil { return err }
inputF, err := os.Open(pathname)
if err != nil { return err }
defer inputF.Close()
wd,err := dircache(filepath.ToSlash(base))
if err != nil { return err }
entry,err := wd.AddFile(filename)
if err != nil { return err }
fatFile,err := entry.File()
if err != nil { return err }
_,err = io.Copy(fatFile,inputF)
s.FilesAdded[pathname] = true
return err
}
return filepath.Walk(src, visit)
}
func (s *StepCreateFloppy) Cleanup(multistep.StateBag) {
if s.floppyPath != "" {
log.Printf("Deleting floppy disk: %s", s.floppyPath)
@ -114,85 +268,98 @@ func (s *StepCreateFloppy) Cleanup(multistep.StateBag) {
}
}
func (s *StepCreateFloppy) addFilespec(dir fs.Directory, src string) error {
// same as http://golang.org/src/pkg/path/filepath/match.go#L308
if strings.IndexAny(src, "*?[") >= 0 {
matches, err := filepath.Glob(src)
if err != nil {
return err
// removeBase will take a regular os.PathSeparator-separated path and remove the
// prefix directory base from it. Both paths are converted to their absolute
// formats before the stripping takes place.
func removeBase(base string, path string) (string,error) {
var idx int
var err error
if res,err := filepath.Abs(path); err == nil {
path = res
}
path = filepath.Clean(path)
if base,err = filepath.Abs(base); err != nil {
return path,err
}
c1,c2 := strings.Split(base, string(os.PathSeparator)), strings.Split(path, string(os.PathSeparator))
for idx = 0; idx < len(c1); idx++ {
if len(c1[idx]) == 0 && len(c2[idx]) != 0 { break }
if c1[idx] != c2[idx] {
return "", fmt.Errorf("Path %s is not prefixed by Base %s", path, base)
}
return s.addFiles(dir, matches)
}
finfo, err := os.Stat(src)
if err != nil {
return err
}
if finfo.IsDir() {
return s.addDirectory(dir, src)
}
return s.addSingleFile(dir, src)
return strings.Join(c2[idx:], string(os.PathSeparator)),nil
}
func (s *StepCreateFloppy) addFiles(dir fs.Directory, files []string) error {
for _, file := range files {
err := s.addFilespec(dir, file)
if err != nil {
return err
// fsDirectoryCache returns a function that can be used to grab the fs.Directory
// entry associated with a given path. If an fs.Directory entry is not found
// then it will be created relative to the rootDirectory argument that is
// passed.
type directoryCache func(string) (fs.Directory,error)
func fsDirectoryCache(rootDirectory fs.Directory) directoryCache {
var cache map[string]fs.Directory
cache = make(map[string]fs.Directory)
cache[""] = rootDirectory
Input,Output,Error := make(chan string),make(chan fs.Directory),make(chan error)
go func(Error chan error) {
for {
input := path.Clean(<-Input)
// found a directory, so yield it
res,ok := cache[input]
if ok {
Output <- res
continue
}
component := strings.Split(input, "/")
// directory not cached, so start at the root and walk each component
// creating them if they're not in cache
var entry fs.Directory
for i,_ := range component {
// join all of our components into a key
path := strings.Join(component[:i], "/")
// check if parent directory is cached
res,ok = cache[path]
if !ok {
// add directory into cache
directory,err := entry.AddDirectory(component[i-1])
if err != nil { Error <- err; continue }
res,err = directory.Dir()
if err != nil { Error <- err; continue }
cache[path] = res
}
// cool, found a directory
entry = res
}
// finally create our directory
directory,err := entry.AddDirectory(component[len(component)-1])
if err != nil { Error <- err; continue }
res,err = directory.Dir()
if err != nil { Error <- err; continue }
cache[input] = res
// ..and yield it
Output <- entry
}
}(Error)
getFilesystemDirectory := func(input string) (fs.Directory,error) {
Input <- input
select {
case res := <-Output:
return res,nil
case err := <-Error:
return *new(fs.Directory),err
}
}
return nil
}
func (s *StepCreateFloppy) addDirectory(dir fs.Directory, src string) error {
log.Printf("Adding directory to floppy: %s", src)
walkFn := func(path string, finfo os.FileInfo, err error) error {
if err != nil {
return err
}
if path == src {
return nil
}
if finfo.IsDir() {
return s.addDirectory(dir, path)
}
return s.addSingleFile(dir, path)
}
return filepath.Walk(src, walkFn)
}
func (s *StepCreateFloppy) addSingleFile(dir fs.Directory, src string) error {
log.Printf("Adding file to floppy: %s", src)
inputF, err := os.Open(src)
if err != nil {
return err
}
defer inputF.Close()
entry, err := dir.AddFile(filepath.Base(src))
if err != nil {
return err
}
fatFile, err := entry.File()
if err != nil {
return err
}
if _, err := io.Copy(fatFile, inputF); err != nil {
return err
}
s.FilesAdded[src] = true
return nil
return getFilesystemDirectory
}

View File

@ -7,10 +7,30 @@ import (
"io/ioutil"
"os"
"path"
"path/filepath"
"strconv"
"testing"
"log"
"strings"
"fmt"
)
const TestFixtures = "test-fixtures"
// utility function for returning a directory structure as a list of strings
func getDirectory(path string) []string {
var result []string
walk := func(path string, info os.FileInfo, err error) error {
if info.IsDir() && !strings.HasSuffix(path, "/") {
path = path + "/"
}
result = append(result, filepath.ToSlash(path))
return nil
}
filepath.Walk(path, walk)
return result
}
func TestStepCreateFloppy_Impl(t *testing.T) {
var raw interface{}
raw = new(StepCreateFloppy)
@ -191,3 +211,83 @@ func xxxTestStepCreateFloppy_notfound(t *testing.T) {
}
}
}
func TestStepCreateFloppyDirectories(t *testing.T) {
const TestName = "floppy-hier"
// file-system hierarchies
var basePath = filepath.Join(".", TestFixtures, TestName)
type contentsTest struct {
dirs []string
result []string
}
// keep in mind that .FilesAdded doesn't keep track of the target filename or directories, but rather the source filename.
directories := [][]contentsTest{
[]contentsTest{
contentsTest{dirs:[]string{"file1","file2","file3"},result:[]string{"file1","file2","file3"}},
contentsTest{dirs:[]string{"file?"},result:[]string{"file1","file2","file3"}},
contentsTest{dirs:[]string{"*"},result:[]string{"file1","file2","file3"}},
},
[]contentsTest{
contentsTest{dirs:[]string{"dir1"},result:[]string{"dir1/file1","dir1/file2","dir1/file3"}},
contentsTest{dirs:[]string{"dir1/file1","dir1/file2","dir1/file3"},result:[]string{"dir1/file1","dir1/file2","dir1/file3"}},
contentsTest{dirs:[]string{"*"},result:[]string{"dir1/file1","dir1/file2","dir1/file3"}},
contentsTest{dirs:[]string{"*/*"},result:[]string{"dir1/file1","dir1/file2","dir1/file3"}},
},
[]contentsTest{
contentsTest{dirs:[]string{"dir1"},result:[]string{"dir1/file1","dir1/subdir1/file1","dir1/subdir1/file2"}},
contentsTest{dirs:[]string{"dir2/*"},result:[]string{"dir2/subdir1/file1","dir2/subdir1/file2"}},
contentsTest{dirs:[]string{"dir2/subdir1"},result:[]string{"dir2/subdir1/file1","dir2/subdir1/file2"}},
contentsTest{dirs:[]string{"dir?"},result:[]string{"dir1/file1","dir1/subdir1/file1","dir1/subdir1/file2","dir2/subdir1/file1","dir2/subdir1/file2"}},
},
}
// create the hierarchy for each file
for i := 0; i < 2; i++ {
dir := filepath.Join(basePath, fmt.Sprintf("test-%d", i))
for _,test := range directories[i] {
// create a new state and step
state := testStepCreateFloppyState(t)
step := new(StepCreateFloppy)
// modify step.Directories with ones from testcase
step.Directories = []string{}
for _,c := range test.dirs {
step.Directories = append(step.Directories, filepath.Join(dir,filepath.FromSlash(c)))
}
log.Println(fmt.Sprintf("Trying against floppy_dirs : %v",step.Directories))
// run the step
if action := step.Run(state); action != multistep.ActionContinue {
t.Fatalf("bad action: %#v for %v : %v", action, step.Directories, state.Get("error"))
}
if _, ok := state.GetOk("error"); ok {
t.Fatalf("state should be ok for %v : %v", step.Directories, state.Get("error"))
}
floppy_path := state.Get("floppy_path").(string)
if _, err := os.Stat(floppy_path); err != nil {
t.Fatalf("file not found: %s for %v : %v", floppy_path, step.Directories, err)
}
// check the FilesAdded array to see if it matches
for _,rpath := range test.result {
fpath := filepath.Join(dir, filepath.FromSlash(rpath))
if !step.FilesAdded[fpath] {
t.Fatalf("unable to find file: %s for %v", fpath, step.Directories)
}
}
// cleanup the step
step.Cleanup(state)
if _, err := os.Stat(floppy_path); err == nil {
t.Fatalf("file found: %s for %v", floppy_path, step.Directories)
}
}
}
}

View File

@ -115,6 +115,12 @@ builder.
and \[\]) are allowed. Directory names are also allowed, which will add all
the files found in the directory to the floppy.
- `floppy_dirs` (array of strings) - A list of directories to place onto
the floppy disk recursively. This is similar to the `floppy_files` option
except that the directory structure is preserved. This is useful for when
your floppy disk includes drivers or if you just want to organize it's
contents as a hierarchy. Wildcard characters (\*, ?, and \[\]) are allowed.
- `guest_os_type` (string) - The guest OS type being installed. By default
this is "other", but you can get *dramatic* performance improvements by
setting this to the proper value. To view all available values for this run

View File

@ -86,6 +86,12 @@ builder.
listed in this configuration will all be put into the root directory of the
floppy disk; sub-directories are not supported.
- `floppy_dirs` (array of strings) - A list of directories to place onto
the floppy disk recursively. This is similar to the `floppy_files` option
except that the directory structure is preserved. This is useful for when
your floppy disk includes drivers or if you just want to organize it's
contents as a hierarchy. Wildcard characters (\*, ?, and \[\]) are allowed.
- `output_directory` (string) - This is the path to the directory where the
resulting virtual machine will be created. This may be relative or absolute.
If relative, the path is relative to the working directory when `packer`

View File

@ -164,6 +164,12 @@ Linux server and have not enabled X11 forwarding (`ssh -X`).
and \[\]) are allowed. Directory names are also allowed, which will add all
the files found in the directory to the floppy.
- `floppy_dirs` (array of strings) - A list of directories to place onto
the floppy disk recursively. This is similar to the `floppy_files` option
except that the directory structure is preserved. This is useful for when
your floppy disk includes drivers or if you just want to organize it's
contents as a hierarchy. Wildcard characters (\*, ?, and \[\]) are allowed.
- `format` (string) - Either "qcow2" or "raw", this specifies the output
format of the virtual machine image. This defaults to `qcow2`.

View File

@ -150,6 +150,12 @@ builder.
and \[\]) are allowed. Directory names are also allowed, which will add all
the files found in the directory to the floppy.
- `floppy_dirs` (array of strings) - A list of directories to place onto
the floppy disk recursively. This is similar to the `floppy_files` option
except that the directory structure is preserved. This is useful for when
your floppy disk includes drivers or if you just want to organize it's
contents as a hierarchy. Wildcard characters (\*, ?, and \[\]) are allowed.
- `format` (string) - Either "ovf" or "ova", this specifies the output format
of the exported virtual machine. This defaults to "ovf".

View File

@ -132,6 +132,12 @@ builder.
and \[\]) are allowed. Directory names are also allowed, which will add all
the files found in the directory to the floppy.
- `floppy_dirs` (array of strings) - A list of directories to place onto
the floppy disk recursively. This is similar to the `floppy_files` option
except that the directory structure is preserved. This is useful for when
your floppy disk includes drivers or if you just want to organize it's
contents as a hierarchy. Wildcard characters (\*, ?, and \[\]) are allowed.
- `format` (string) - Either "ovf" or "ova", this specifies the output format
of the exported virtual machine. This defaults to "ovf".

View File

@ -125,6 +125,12 @@ builder.
and \[\]) are allowed. Directory names are also allowed, which will add all
the files found in the directory to the floppy.
- `floppy_dirs` (array of strings) - A list of directories to place onto
the floppy disk recursively. This is similar to the `floppy_files` option
except that the directory structure is preserved. This is useful for when
your floppy disk includes drivers or if you just want to organize it's
contents as a hierarchy. Wildcard characters (\*, ?, and \[\]) are allowed.
- `fusion_app_path` (string) - Path to "VMware Fusion.app". By default this is
"/Applications/VMware Fusion.app" but this setting allows you to
customize this.

View File

@ -83,6 +83,12 @@ builder.
and \[\]) are allowed. Directory names are also allowed, which will add all
the files found in the directory to the floppy.
- `floppy_dirs` (array of strings) - A list of directories to place onto
the floppy disk recursively. This is similar to the `floppy_files` option
except that the directory structure is preserved. This is useful for when
your floppy disk includes drivers or if you just want to organize it's
contents as a hierarchy. Wildcard characters (\*, ?, and \[\]) are allowed.
- `fusion_app_path` (string) - Path to "VMware Fusion.app". By default this is
"/Applications/VMware Fusion.app" but this setting allows you to
customize this.