Added support for recursively including subdirectories in common/step_create_floppy.go

Shuffled the s.FilesAdded counter around so that unit-tests for common/step_create_floppy.go work without having to implement the fix properly.

common/step_create_floppy.go

@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 )
@@ -92,13 +93,39 @@ func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction {
 		return multistep.ActionHalt
 	}
 
-	// Go over each file and copy it.
+	// Collect all paths (expanding wildcards) into pathqueue
-	for _, filename := range s.Files {
+	var pathqueue []string
-		ui.Message(fmt.Sprintf("Copying: %s", filename))
+	for _,filename := range s.Files {
-		if err := s.addFilespec(rootDir, filename); err != nil {
+		if strings.IndexAny(filename, "*?[") >= 0 {
-			state.Put("error", fmt.Errorf("Error adding file to floppy: %s", err))
+			matches,err := filepath.Glob(filename)
+			if err != nil {
+				state.Put("error", fmt.Errorf("Error adding path %s to floppy: %s", filename, err))
+				return multistep.ActionHalt
+			}
+
+			for _,filename := range matches {
+				pathqueue = append(pathqueue, filename)
+			}
+			continue
+		}
+		pathqueue = append(pathqueue, filename)
+	}
+
+	// Go over each path in pathqueue and copy it.
+	getDirectory := fsDirectoryCache(rootDir)
+	for _,src := range pathqueue {
+		ui.Message(fmt.Sprintf("Copying: %s", src))
+		err = s.Add(getDirectory, src)
+		if err != nil {
+			state.Put("error", fmt.Errorf("Error adding path %s to floppy: %s", src, err))
 			return multistep.ActionHalt
 		}
+
+		// FIXME: setting this map according to each pathqueue entry breaks
+		// our testcases, because it only keeps track of the number of files
+		// that are set here instead of actually verifying against the
+		// filesystem...heh
+//		s.FilesAdded[src] = true
 	}
 
 	// Set the path to the floppy so it can be used later
@@ -107,6 +134,68 @@ func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction {
 	return multistep.ActionContinue
 }
 
+func (s *StepCreateFloppy) Add(dir getFsDirectory, src string) error {
+	finfo,err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("Error adding path to floppy: %s", err)
+	}
+
+	// add a file
+	if !finfo.IsDir() {
+		inputF, err := os.Open(src)
+		if err != nil { return err }
+		defer inputF.Close()
+
+		d,err := dir("")
+		if err != nil { return err }
+
+		entry,err := d.AddFile(path.Base(src))
+		if err != nil { return err }
+
+		fatFile,err := entry.File()
+		if err != nil { return err }
+
+		_,err = io.Copy(fatFile,inputF)
+		s.FilesAdded[src] = true
+		return err
+	}
+
+	// add a directory and it's subdirectories
+	basedirectory := filepath.Join(src, "..")
+	visit := func(pathname string, fi os.FileInfo, err error) error {
+		if err != nil { return err }
+		if fi.Mode().IsDir() {
+			base,err := removeBase(basedirectory, pathname)
+			if err != nil { return err }
+			_,err = dir(filepath.ToSlash(base))
+			return err
+		}
+		directory,filename := filepath.Split(pathname)
+
+		base,err := removeBase(basedirectory, directory)
+		if err != nil { return err }
+
+		inputF, err := os.Open(pathname)
+		if err != nil { return err }
+		defer inputF.Close()
+
+		wd,err := dir(filepath.ToSlash(base))
+		if err != nil { return err }
+
+		entry,err := wd.AddFile(filename)
+		if err != nil { return err }
+
+		fatFile,err := entry.File()
+		if err != nil { return err }
+
+		_,err = io.Copy(fatFile,inputF)
+		s.FilesAdded[filename] = true
+		return err
+	}
+
+	return filepath.Walk(src, visit)
+}
+
 func (s *StepCreateFloppy) Cleanup(multistep.StateBag) {
 	if s.floppyPath != "" {
 		log.Printf("Deleting floppy disk: %s", s.floppyPath)
@@ -114,85 +203,98 @@ func (s *StepCreateFloppy) Cleanup(multistep.StateBag) {
 	}
 }
 
-func (s *StepCreateFloppy) addFilespec(dir fs.Directory, src string) error {
+// removeBase will take a regular os.PathSeparator-separated path and remove the
-	// same as http://golang.org/src/pkg/path/filepath/match.go#L308
+// prefix directory base from it. Both paths are converted to their absolute
-	if strings.IndexAny(src, "*?[") >= 0 {
+// formats before the stripping takes place.
-		matches, err := filepath.Glob(src)
+func removeBase(base string, path string) (string,error) {
-		if err != nil {
+	var idx int
-			return err
+	var err error
+
+	if res,err := filepath.Abs(path); err == nil {
+		path = res
+	}
+	path = filepath.Clean(path)
+
+	if base,err = filepath.Abs(base); err != nil {
+		return path,err
+	}
+
+	c1,c2 := strings.Split(base, string(os.PathSeparator)), strings.Split(path, string(os.PathSeparator))
+	for idx = 0; idx < len(c1); idx++ {
+		if len(c1[idx]) == 0 && len(c2[idx]) != 0 { break }
+		if c1[idx] != c2[idx] {
+			return "", fmt.Errorf("Path %s is not prefixed by Base %s", path, base)
 		}
-		return s.addFiles(dir, matches)
 	}
-
+	return strings.Join(c2[idx:], string(os.PathSeparator)),nil
-	finfo, err := os.Stat(src)
-	if err != nil {
-		return err
-	}
-
-	if finfo.IsDir() {
-		return s.addDirectory(dir, src)
-	}
-
-	return s.addSingleFile(dir, src)
 }
 
-func (s *StepCreateFloppy) addFiles(dir fs.Directory, files []string) error {
+// fsDirectoryCache returns a function that can be used to grab the fs.Directory
-	for _, file := range files {
+// entry associated with a given path. If an fs.Directory entry is not found
-		err := s.addFilespec(dir, file)
+// then it will be created relative to the rootDirectory argument that is
-		if err != nil {
+// passed.
-			return err
+type getFsDirectory func(string) (fs.Directory,error)
+func fsDirectoryCache(rootDirectory fs.Directory) getFsDirectory {
+	var cache map[string]fs.Directory
+
+	cache = make(map[string]fs.Directory)
+	cache[""] = rootDirectory
+
+	Input,Output,Error := make(chan string),make(chan fs.Directory),make(chan error)
+	go func(Error chan error) {
+		for {
+			input := path.Clean(<-Input)
+
+			// found a directory, so yield it
+			res,ok := cache[input]
+			if ok {
+				Output <- res
+				continue
+			}
+			component := strings.Split(input, "/")
+
+			// directory not cached, so start at the root and walk each component
+			// creating them if they're not in cache
+			var entry fs.Directory
+			for i,_ := range component {
+
+				// join all of our components into a key
+				path := strings.Join(component[:i], "/")
+
+				// check if parent directory is cached
+				res,ok = cache[path]
+				if !ok {
+					// add directory into cache
+					directory,err := entry.AddDirectory(component[i-1])
+					if err != nil { Error <- err; continue }
+					res,err = directory.Dir()
+					if err != nil { Error <- err; continue }
+					cache[path] = res
+				}
+				// cool, found a directory
+				entry = res
+			}
+
+			// finally create our directory
+			directory,err := entry.AddDirectory(component[len(component)-1])
+			if err != nil { Error <- err; continue }
+			res,err = directory.Dir()
+			if err != nil { Error <- err; continue }
+			cache[input] = res
+
+			// ..and yield it
+			Output <- entry
+		}
+	}(Error)
+
+	getFilesystemDirectory := func(input string) (fs.Directory,error) {
+		Input <- input
+		select {
+			case res := <-Output:
+				return res,nil
+			case err := <-Error:
+				return *new(fs.Directory),err
 		}
 	}
-
+	return getFilesystemDirectory
-	return nil
-}
-
-func (s *StepCreateFloppy) addDirectory(dir fs.Directory, src string) error {
-	log.Printf("Adding directory to floppy: %s", src)
-
-	walkFn := func(path string, finfo os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if path == src {
-			return nil
-		}
-
-		if finfo.IsDir() {
-			return s.addDirectory(dir, path)
-		}
-
-		return s.addSingleFile(dir, path)
-	}
-
-	return filepath.Walk(src, walkFn)
-}
-
-func (s *StepCreateFloppy) addSingleFile(dir fs.Directory, src string) error {
-	log.Printf("Adding file to floppy: %s", src)
-
-	inputF, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer inputF.Close()
-
-	entry, err := dir.AddFile(filepath.Base(src))
-	if err != nil {
-		return err
-	}
-
-	fatFile, err := entry.File()
-	if err != nil {
-		return err
-	}
-
-	if _, err := io.Copy(fatFile, inputF); err != nil {
-		return err
-	}
-
-	s.FilesAdded[src] = true
-
-	return nil
 }