From 52269b66b98e119bb4600a470007997c6476dafc Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 10 Jun 2015 12:30:18 -0700 Subject: [PATCH 01/28] Added new compress post-processor, contributed by Vasiliy Tolstov --- post-processor/compress/LICENSE | 21 ++ post-processor/compress/artifact.go | 32 +- post-processor/compress/benchmark.go | 197 ++++++++++ post-processor/compress/post-processor.go | 419 +++++++++++++++++++--- 4 files changed, 600 insertions(+), 69 deletions(-) create mode 100644 post-processor/compress/LICENSE create mode 100644 post-processor/compress/benchmark.go diff --git a/post-processor/compress/LICENSE b/post-processor/compress/LICENSE new file mode 100644 index 000000000..38bbf26f3 --- /dev/null +++ b/post-processor/compress/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Vasiliy Tolstov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/post-processor/compress/artifact.go b/post-processor/compress/artifact.go index 34a7ce8d6..f428a3b55 100644 --- a/post-processor/compress/artifact.go +++ b/post-processor/compress/artifact.go @@ -5,40 +5,34 @@ import ( "os" ) -const BuilderId = "packer.post-processor.compress" +const BuilderId = "vtolstov.compress" type Artifact struct { - Path string - Provider string + builderId string + dir string + f []string } -func NewArtifact(provider, path string) *Artifact { - return &Artifact{ - Path: path, - Provider: provider, - } -} - -func (*Artifact) BuilderId() string { +func (a *Artifact) BuilderId() string { return BuilderId } -func (self *Artifact) Id() string { - return "" +func (a *Artifact) Files() []string { + return a.f } -func (self *Artifact) Files() []string { - return []string{self.Path} +func (*Artifact) Id() string { + return "COMPRESS" } -func (self *Artifact) String() string { - return fmt.Sprintf("'%s' compressing: %s", self.Provider, self.Path) +func (a *Artifact) String() string { + return fmt.Sprintf("VM compressed files in directory: %s", a.dir) } func (*Artifact) State(name string) interface{} { return nil } -func (self *Artifact) Destroy() error { - return os.Remove(self.Path) +func (a *Artifact) Destroy() error { + return os.RemoveAll(a.dir) } diff --git a/post-processor/compress/benchmark.go b/post-processor/compress/benchmark.go new file mode 100644 index 000000000..a2585bc89 --- /dev/null +++ b/post-processor/compress/benchmark.go @@ -0,0 +1,197 @@ +// +build ignore + +package main + +import ( + "compress/flate" + gzip "compress/gzip" + "io" + "io/ioutil" + "fmt" + "os" + "runtime" + "testing" + + bgzf "github.com/biogo/hts/bgzf" + pgzip "github.com/klauspost/pgzip" + lz4 "github.com/pierrec/lz4" +) + +type Compressor struct { + r *os.File + w *os.File + sr int64 + sw int64 +} + +func (c *Compressor) Close() error { + var err error + + fi, _ := c.w.Stat() + c.sw = fi.Size() + if err = c.w.Close(); err != nil { + return err + } + + fi, _ = c.r.Stat() + c.sr = fi.Size() + if err = c.r.Close(); err != nil { + return err + } + + return nil +} + +func NewCompressor(src, dst string) (*Compressor, error) { + r, err := os.Open(src) + if err != nil { + return nil, err + } + + w, err := os.Create(dst) + if err != nil { + r.Close() + return nil, err + } + + c := &Compressor{r: r, w: w} + return c, nil +} + +func main() { + + runtime.GOMAXPROCS(runtime.NumCPU()) + + var resw testing.BenchmarkResult + var resr testing.BenchmarkResult + + c, err := NewCompressor("/tmp/image.r", "/tmp/image.w") + if err != nil { + panic(err) + } + resw = testing.Benchmark(c.BenchmarkGZIPWriter) + c.w.Seek(0, 0) + resr = testing.Benchmark(c.BenchmarkGZIPReader) + c.Close() + fmt.Printf("gzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) + + c, err = NewCompressor("/tmp/image.r", "/tmp/image.w") + if err != nil { + panic(err) + } + resw = testing.Benchmark(c.BenchmarkBGZFWriter) + c.w.Seek(0, 0) + resr = testing.Benchmark(c.BenchmarkBGZFReader) + c.Close() + fmt.Printf("bgzf:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) + + c, err = NewCompressor("/tmp/image.r", "/tmp/image.w") + if err != nil { + panic(err) + } + resw = testing.Benchmark(c.BenchmarkPGZIPWriter) + c.w.Seek(0, 0) + resr = testing.Benchmark(c.BenchmarkPGZIPReader) + c.Close() + fmt.Printf("pgzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) + + c, err = NewCompressor("/tmp/image.r", "/tmp/image.w") + if err != nil { + panic(err) + } + resw = testing.Benchmark(c.BenchmarkLZ4Writer) + c.w.Seek(0, 0) + resr = testing.Benchmark(c.BenchmarkLZ4Reader) + c.Close() + fmt.Printf("lz4:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) + +} + +func (c *Compressor) BenchmarkGZIPWriter(b *testing.B) { + cw, _ := gzip.NewWriterLevel(c.w, flate.BestSpeed) + b.ResetTimer() + + _, err := io.Copy(cw, c.r) + if err != nil { + b.Fatal(err) + } + cw.Close() + c.w.Sync() +} + +func (c *Compressor) BenchmarkGZIPReader(b *testing.B) { + cr, _ := gzip.NewReader(c.w) + b.ResetTimer() + + _, err := io.Copy(ioutil.Discard, cr) + if err != nil { + b.Fatal(err) + } +} + +func (c *Compressor) BenchmarkBGZFWriter(b *testing.B) { + cw, _ := bgzf.NewWriterLevel(c.w, flate.BestSpeed, runtime.NumCPU()) + b.ResetTimer() + + _, err := io.Copy(cw, c.r) + if err != nil { + b.Fatal(err) + } + c.w.Sync() +} + +func (c *Compressor) BenchmarkBGZFReader(b *testing.B) { + cr, _ := bgzf.NewReader(c.w, 0) + b.ResetTimer() + + _, err := io.Copy(ioutil.Discard, cr) + if err != nil { + b.Fatal(err) + } +} + +func (c *Compressor) BenchmarkPGZIPWriter(b *testing.B) { + cw, _ := pgzip.NewWriterLevel(c.w, flate.BestSpeed) + b.ResetTimer() + + _, err := io.Copy(cw, c.r) + if err != nil { + b.Fatal(err) + } + cw.Close() + c.w.Sync() +} + +func (c *Compressor) BenchmarkPGZIPReader(b *testing.B) { + cr, _ := pgzip.NewReader(c.w) + b.ResetTimer() + + _, err := io.Copy(ioutil.Discard, cr) + if err != nil { + b.Fatal(err) + } +} + +func (c *Compressor) BenchmarkLZ4Writer(b *testing.B) { + cw := lz4.NewWriter(c.w) +// cw.Header.HighCompression = true + cw.Header.NoChecksum = true + b.ResetTimer() + + _, err := io.Copy(cw, c.r) + if err != nil { + b.Fatal(err) + } + cw.Close() + c.w.Sync() +} + +func (c *Compressor) BenchmarkLZ4Reader(b *testing.B) { + cr := lz4.NewReader(c.w) + b.ResetTimer() + + _, err := io.Copy(ioutil.Discard, cr) + if err != nil { + b.Fatal(err) + } +} diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index ccf300946..f62bea858 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -1,98 +1,417 @@ package compress import ( - "archive/tar" - "compress/gzip" + tar "archive/tar" + zip "archive/zip" + "compress/flate" + gzip "compress/gzip" "fmt" "io" "os" + "path/filepath" + "runtime" + "strings" + "time" + bgzf "github.com/biogo/hts/bgzf" + pgzip "github.com/klauspost/pgzip" "github.com/mitchellh/packer/common" - "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" - "github.com/mitchellh/packer/template/interpolate" + lz4 "github.com/pierrec/lz4" + "gopkg.in/yaml.v2" ) +type Metadata map[string]Metaitem + +type Metaitem struct { + CompSize int64 `yaml:"compsize"` + OrigSize int64 `yaml:"origsize"` + CompType string `yaml:"comptype"` + CompDate string `yaml:"compdate"` +} + type Config struct { common.PackerConfig `mapstructure:",squash"` - OutputPath string `mapstructure:"output"` - - ctx interpolate.Context + OutputPath string `mapstructure:"output"` + OutputFile string `mapstructure:"file"` + Compression int `mapstructure:"compression"` + Metadata bool `mapstructure:"metadata"` + NumCPU int `mapstructure:"numcpu"` + Format string `mapstructure:"format"` + KeepInputArtifact bool `mapstructure:"keep_input_artifact"` + tpl *packer.ConfigTemplate } -type PostProcessor struct { - config Config +type CompressPostProcessor struct { + cfg Config } -func (self *PostProcessor) Configure(raws ...interface{}) error { - err := config.Decode(&self.config, &config.DecodeOpts{ - Interpolate: true, - InterpolateFilter: &interpolate.RenderFilter{ - Exclude: []string{}, - }, - }, raws...) +func (p *CompressPostProcessor) Configure(raws ...interface{}) error { + p.cfg.Compression = -1 + _, err := common.DecodeConfig(&p.cfg, raws...) if err != nil { return err } + errs := new(packer.MultiError) + + p.cfg.tpl, err = packer.NewConfigTemplate() + if err != nil { + return err + } + p.cfg.tpl.UserVars = p.cfg.PackerUserVars + + if p.cfg.OutputPath == "" { + p.cfg.OutputPath = "packer_{{.BuildName}}_{{.Provider}}" + } + + if err = p.cfg.tpl.Validate(p.cfg.OutputPath); err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Error parsing target template: %s", err)) + } + + templates := map[string]*string{ + "output": &p.cfg.OutputPath, + } + + if p.cfg.Compression > flate.BestCompression { + p.cfg.Compression = flate.BestCompression + } + if p.cfg.Compression == -1 { + p.cfg.Compression = flate.DefaultCompression + } + + if p.cfg.NumCPU < 1 { + p.cfg.NumCPU = runtime.NumCPU() + } + + runtime.GOMAXPROCS(p.cfg.NumCPU) + + for key, ptr := range templates { + if *ptr == "" { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("%s must be set", key)) + } + + *ptr, err = p.cfg.tpl.Process(*ptr, nil) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Error processing %s: %s", key, err)) + } + } + + if len(errs.Errors) > 0 { + return errs + } + return nil } -func (self *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - ui.Say(fmt.Sprintf("Creating archive for '%s'", artifact.BuilderId())) +func (p *CompressPostProcessor) fillMetadata(metadata Metadata, files []string) Metadata { + // layout shows by example how the reference time should be represented. + const layout = "2006-01-02_15-04-05" + t := time.Now() - // Create the compressed archive file at the appropriate OutputPath. - fw, err := os.Create(self.config.OutputPath) + if !p.cfg.Metadata { + return metadata + } + for _, f := range files { + if fi, err := os.Stat(f); err != nil { + continue + } else { + if i, ok := metadata[filepath.Base(f)]; !ok { + metadata[filepath.Base(f)] = Metaitem{CompType: p.cfg.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)} + } else { + i.CompSize = fi.Size() + i.CompDate = t.Format(layout) + metadata[filepath.Base(f)] = i + } + } + } + return metadata +} + +func (p *CompressPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { + newartifact := &Artifact{builderId: artifact.BuilderId(), dir: p.cfg.OutputPath} + var metafile string = filepath.Join(p.cfg.OutputPath, "metadata") + + _, err := os.Stat(newartifact.dir) + if err == nil { + return nil, false, fmt.Errorf("output dir must not exists: %s", err) + } + err = os.MkdirAll(newartifact.dir, 0755) if err != nil { - return nil, false, fmt.Errorf( - "Failed creating file for compressed archive: %s", self.config.OutputPath) + return nil, false, fmt.Errorf("failed to create output: %s", err) + } + + formats := strings.Split(p.cfg.Format, ".") + files := artifact.Files() + + metadata := make(Metadata, 0) + metadata = p.fillMetadata(metadata, files) + + for _, compress := range formats { + switch compress { + case "tar": + files, err = p.cmpTAR(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) + metadata = p.fillMetadata(metadata, files) + case "zip": + files, err = p.cmpZIP(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) + metadata = p.fillMetadata(metadata, files) + case "pgzip": + files, err = p.cmpPGZIP(files, p.cfg.OutputPath) + metadata = p.fillMetadata(metadata, files) + case "gzip": + files, err = p.cmpGZIP(files, p.cfg.OutputPath) + metadata = p.fillMetadata(metadata, files) + case "bgzf": + files, err = p.cmpBGZF(files, p.cfg.OutputPath) + metadata = p.fillMetadata(metadata, files) + case "lz4": + files, err = p.cmpLZ4(files, p.cfg.OutputPath) + metadata = p.fillMetadata(metadata, files) + case "e2fs": + files, err = p.cmpE2FS(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) + metadata = p.fillMetadata(metadata, files) + } + if err != nil { + return nil, false, fmt.Errorf("Failed to compress: %s", err) + } + } + + if p.cfg.Metadata { + fp, err := os.Create(metafile) + if err != nil { + return nil, false, err + } + if buf, err := yaml.Marshal(metadata); err != nil { + fp.Close() + return nil, false, err + } else { + if _, err = fp.Write(buf); err != nil { + fp.Close() + return nil, false, err + } + fp.Close() + } + } + + newartifact.f = append(newartifact.f, files...) + if p.cfg.Metadata { + newartifact.f = append(newartifact.f, metafile) + } + + return newartifact, p.cfg.KeepInputArtifact, nil +} + +func (p *CompressPostProcessor) cmpTAR(src []string, dst string) ([]string, error) { + fw, err := os.Create(dst) + if err != nil { + return nil, fmt.Errorf("tar error: %s", err) } defer fw.Close() - gw := gzip.NewWriter(fw) - defer gw.Close() + tw := tar.NewWriter(fw) + defer tw.Close() - // Iterate through all of the artifact's files and put them into the - // compressed archive using the tar/gzip writers. - for _, path := range artifact.Files() { - fi, err := os.Stat(path) + for _, name := range src { + fi, err := os.Stat(name) if err != nil { - return nil, false, fmt.Errorf( - "Failed stating file: %s", path) + return nil, fmt.Errorf("tar error: %s", err) } - target, _ := os.Readlink(path) + target, _ := os.Readlink(name) header, err := tar.FileInfoHeader(fi, target) if err != nil { - return nil, false, fmt.Errorf( - "Failed creating archive header: %s", path) + return nil, fmt.Errorf("tar erorr: %s", err) } - tw := tar.NewWriter(gw) - defer tw.Close() - - // Write the header first to the archive. This takes partial data - // from the FileInfo that is grabbed by running the stat command. - if err := tw.WriteHeader(header); err != nil { - return nil, false, fmt.Errorf( - "Failed writing archive header: %s", path) + if err = tw.WriteHeader(header); err != nil { + return nil, fmt.Errorf("tar error: %s", err) } - // Open the target file for archiving and compressing. - fr, err := os.Open(path) + fr, err := os.Open(name) if err != nil { - return nil, false, fmt.Errorf( - "Failed opening file '%s' to write compressed archive.", path) + return nil, fmt.Errorf("tar error: %s", err) } - defer fr.Close() if _, err = io.Copy(tw, fr); err != nil { - return nil, false, fmt.Errorf( - "Failed copying file to archive: %s", path) + fr.Close() + return nil, fmt.Errorf("tar error: %s", err) } + fr.Close() } - - return NewArtifact(artifact.BuilderId(), self.config.OutputPath), false, nil + return []string{dst}, nil +} + +func (p *CompressPostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { + var res []string + for _, name := range src { + filename := filepath.Join(dst, filepath.Base(name)) + fw, err := os.Create(filename) + if err != nil { + return nil, fmt.Errorf("gzip error: %s", err) + } + cw, err := gzip.NewWriterLevel(fw, p.cfg.Compression) + if err != nil { + fw.Close() + return nil, fmt.Errorf("gzip error: %s", err) + } + fr, err := os.Open(name) + if err != nil { + cw.Close() + fw.Close() + return nil, fmt.Errorf("gzip error: %s", err) + } + if _, err = io.Copy(cw, fr); err != nil { + cw.Close() + fr.Close() + fw.Close() + return nil, fmt.Errorf("gzip error: %s", err) + } + cw.Close() + fr.Close() + fw.Close() + res = append(res, filename) + } + return res, nil +} + +func (p *CompressPostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) { + var res []string + for _, name := range src { + filename := filepath.Join(dst, filepath.Base(name)) + fw, err := os.Create(filename) + if err != nil { + return nil, fmt.Errorf("pgzip error: %s", err) + } + cw, err := pgzip.NewWriterLevel(fw, p.cfg.Compression) + if err != nil { + fw.Close() + return nil, fmt.Errorf("pgzip error: %s", err) + } + fr, err := os.Open(name) + if err != nil { + cw.Close() + fw.Close() + return nil, fmt.Errorf("pgzip error: %s", err) + } + if _, err = io.Copy(cw, fr); err != nil { + cw.Close() + fr.Close() + fw.Close() + return nil, fmt.Errorf("pgzip error: %s", err) + } + cw.Close() + fr.Close() + fw.Close() + res = append(res, filename) + } + return res, nil +} + +func (p *CompressPostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { + var res []string + for _, name := range src { + filename := filepath.Join(dst, filepath.Base(name)) + fw, err := os.Create(filename) + if err != nil { + return nil, fmt.Errorf("lz4 error: %s", err) + } + cw := lz4.NewWriter(fw) + if err != nil { + fw.Close() + return nil, fmt.Errorf("lz4 error: %s", err) + } + if p.cfg.Compression > flate.DefaultCompression { + cw.Header.HighCompression = true + } + fr, err := os.Open(name) + if err != nil { + cw.Close() + fw.Close() + return nil, fmt.Errorf("lz4 error: %s", err) + } + if _, err = io.Copy(cw, fr); err != nil { + cw.Close() + fr.Close() + fw.Close() + return nil, fmt.Errorf("lz4 error: %s", err) + } + cw.Close() + fr.Close() + fw.Close() + res = append(res, filename) + } + return res, nil +} + +func (p *CompressPostProcessor) cmpBGZF(src []string, dst string) ([]string, error) { + var res []string + for _, name := range src { + filename := filepath.Join(dst, filepath.Base(name)) + fw, err := os.Create(filename) + if err != nil { + return nil, fmt.Errorf("bgzf error: %s", err) + } + + cw, err := bgzf.NewWriterLevel(fw, p.cfg.Compression, runtime.NumCPU()) + if err != nil { + return nil, fmt.Errorf("bgzf error: %s", err) + } + fr, err := os.Open(name) + if err != nil { + cw.Close() + fw.Close() + return nil, fmt.Errorf("bgzf error: %s", err) + } + if _, err = io.Copy(cw, fr); err != nil { + cw.Close() + fr.Close() + fw.Close() + return nil, fmt.Errorf("bgzf error: %s", err) + } + cw.Close() + fr.Close() + fw.Close() + res = append(res, filename) + } + return res, nil +} + +func (p *CompressPostProcessor) cmpE2FS(src []string, dst string) ([]string, error) { + panic("not implemented") +} + +func (p *CompressPostProcessor) cmpZIP(src []string, dst string) ([]string, error) { + fw, err := os.Create(dst) + if err != nil { + return nil, fmt.Errorf("zip error: %s", err) + } + defer fw.Close() + + zw := zip.NewWriter(fw) + defer zw.Close() + + for _, name := range src { + header, err := zw.Create(name) + if err != nil { + return nil, fmt.Errorf("zip erorr: %s", err) + } + + fr, err := os.Open(name) + if err != nil { + return nil, fmt.Errorf("zip error: %s", err) + } + + if _, err = io.Copy(header, fr); err != nil { + fr.Close() + return nil, fmt.Errorf("zip error: %s", err) + } + fr.Close() + } + return []string{dst}, nil + } From c4fc365c657038e8929f17ce49c8db2434d92a12 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 10 Jun 2015 13:33:50 -0700 Subject: [PATCH 02/28] Updated to reflect changes to template code --- plugin/post-processor-compress/main.go | 2 +- post-processor/compress/post-processor.go | 26 +++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugin/post-processor-compress/main.go b/plugin/post-processor-compress/main.go index 3acc85228..15bf6f223 100644 --- a/plugin/post-processor-compress/main.go +++ b/plugin/post-processor-compress/main.go @@ -10,6 +10,6 @@ func main() { if err != nil { panic(err) } - server.RegisterPostProcessor(new(compress.PostProcessor)) + server.RegisterPostProcessor(new(compress.CompressPostProcessor)) server.Serve() } diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index f62bea858..9751a24a8 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -16,7 +16,9 @@ import ( bgzf "github.com/biogo/hts/bgzf" pgzip "github.com/klauspost/pgzip" "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/template/interpolate" lz4 "github.com/pierrec/lz4" "gopkg.in/yaml.v2" ) @@ -40,7 +42,7 @@ type Config struct { NumCPU int `mapstructure:"numcpu"` Format string `mapstructure:"format"` KeepInputArtifact bool `mapstructure:"keep_input_artifact"` - tpl *packer.ConfigTemplate + ctx *interpolate.Context } type CompressPostProcessor struct { @@ -49,24 +51,22 @@ type CompressPostProcessor struct { func (p *CompressPostProcessor) Configure(raws ...interface{}) error { p.cfg.Compression = -1 - _, err := common.DecodeConfig(&p.cfg, raws...) - if err != nil { - return err - } + err := config.Decode(&p.cfg, &config.DecodeOpts{ + Interpolate: true, + InterpolateFilter: &interpolate.RenderFilter{ + Exclude: []string{ + // TODO figure out if something needs to go here. + }, + }, + }, raws...) errs := new(packer.MultiError) - p.cfg.tpl, err = packer.NewConfigTemplate() - if err != nil { - return err - } - p.cfg.tpl.UserVars = p.cfg.PackerUserVars - if p.cfg.OutputPath == "" { p.cfg.OutputPath = "packer_{{.BuildName}}_{{.Provider}}" } - if err = p.cfg.tpl.Validate(p.cfg.OutputPath); err != nil { + if err = interpolate.Validate(p.cfg.OutputPath, p.cfg.ctx); err != nil { errs = packer.MultiErrorAppend( errs, fmt.Errorf("Error parsing target template: %s", err)) } @@ -94,7 +94,7 @@ func (p *CompressPostProcessor) Configure(raws ...interface{}) error { errs, fmt.Errorf("%s must be set", key)) } - *ptr, err = p.cfg.tpl.Process(*ptr, nil) + *ptr, err = interpolate.Render(p.cfg.OutputPath, p.cfg.ctx) if err != nil { errs = packer.MultiErrorAppend( errs, fmt.Errorf("Error processing %s: %s", key, err)) From 42d749ab5f38d01b8e89b781d7056bc949c32f91 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 10 Jun 2015 13:46:21 -0700 Subject: [PATCH 03/28] Light style and typo cleanup --- plugin/post-processor-compress/main.go | 2 +- post-processor/compress/post-processor.go | 28 +++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/plugin/post-processor-compress/main.go b/plugin/post-processor-compress/main.go index 15bf6f223..3acc85228 100644 --- a/plugin/post-processor-compress/main.go +++ b/plugin/post-processor-compress/main.go @@ -10,6 +10,6 @@ func main() { if err != nil { panic(err) } - server.RegisterPostProcessor(new(compress.CompressPostProcessor)) + server.RegisterPostProcessor(new(compress.PostProcessor)) server.Serve() } diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index 9751a24a8..74228d909 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -45,11 +45,11 @@ type Config struct { ctx *interpolate.Context } -type CompressPostProcessor struct { +type PostProcessor struct { cfg Config } -func (p *CompressPostProcessor) Configure(raws ...interface{}) error { +func (p *PostProcessor) Configure(raws ...interface{}) error { p.cfg.Compression = -1 err := config.Decode(&p.cfg, &config.DecodeOpts{ Interpolate: true, @@ -109,7 +109,7 @@ func (p *CompressPostProcessor) Configure(raws ...interface{}) error { } -func (p *CompressPostProcessor) fillMetadata(metadata Metadata, files []string) Metadata { +func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata { // layout shows by example how the reference time should be represented. const layout = "2006-01-02_15-04-05" t := time.Now() @@ -133,9 +133,9 @@ func (p *CompressPostProcessor) fillMetadata(metadata Metadata, files []string) return metadata } -func (p *CompressPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { +func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { newartifact := &Artifact{builderId: artifact.BuilderId(), dir: p.cfg.OutputPath} - var metafile string = filepath.Join(p.cfg.OutputPath, "metadata") + metafile := filepath.Join(p.cfg.OutputPath, "metadata") _, err := os.Stat(newartifact.dir) if err == nil { @@ -206,7 +206,7 @@ func (p *CompressPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifa return newartifact, p.cfg.KeepInputArtifact, nil } -func (p *CompressPostProcessor) cmpTAR(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { fw, err := os.Create(dst) if err != nil { return nil, fmt.Errorf("tar error: %s", err) @@ -225,7 +225,7 @@ func (p *CompressPostProcessor) cmpTAR(src []string, dst string) ([]string, erro target, _ := os.Readlink(name) header, err := tar.FileInfoHeader(fi, target) if err != nil { - return nil, fmt.Errorf("tar erorr: %s", err) + return nil, fmt.Errorf("tar error: %s", err) } if err = tw.WriteHeader(header); err != nil { @@ -246,7 +246,7 @@ func (p *CompressPostProcessor) cmpTAR(src []string, dst string) ([]string, erro return []string{dst}, nil } -func (p *CompressPostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { var res []string for _, name := range src { filename := filepath.Join(dst, filepath.Base(name)) @@ -279,7 +279,7 @@ func (p *CompressPostProcessor) cmpGZIP(src []string, dst string) ([]string, err return res, nil } -func (p *CompressPostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) { var res []string for _, name := range src { filename := filepath.Join(dst, filepath.Base(name)) @@ -312,7 +312,7 @@ func (p *CompressPostProcessor) cmpPGZIP(src []string, dst string) ([]string, er return res, nil } -func (p *CompressPostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { var res []string for _, name := range src { filename := filepath.Join(dst, filepath.Base(name)) @@ -348,7 +348,7 @@ func (p *CompressPostProcessor) cmpLZ4(src []string, dst string) ([]string, erro return res, nil } -func (p *CompressPostProcessor) cmpBGZF(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpBGZF(src []string, dst string) ([]string, error) { var res []string for _, name := range src { filename := filepath.Join(dst, filepath.Base(name)) @@ -381,11 +381,11 @@ func (p *CompressPostProcessor) cmpBGZF(src []string, dst string) ([]string, err return res, nil } -func (p *CompressPostProcessor) cmpE2FS(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpE2FS(src []string, dst string) ([]string, error) { panic("not implemented") } -func (p *CompressPostProcessor) cmpZIP(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpZIP(src []string, dst string) ([]string, error) { fw, err := os.Create(dst) if err != nil { return nil, fmt.Errorf("zip error: %s", err) @@ -398,7 +398,7 @@ func (p *CompressPostProcessor) cmpZIP(src []string, dst string) ([]string, erro for _, name := range src { header, err := zw.Create(name) if err != nil { - return nil, fmt.Errorf("zip erorr: %s", err) + return nil, fmt.Errorf("zip error: %s", err) } fr, err := os.Open(name) From e294db8ede0e9813c77192478159bc124412b811 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 10 Jun 2015 14:04:24 -0700 Subject: [PATCH 04/28] Revert to original BuilderId --- post-processor/compress/artifact.go | 2 +- post-processor/compress/post-processor.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/post-processor/compress/artifact.go b/post-processor/compress/artifact.go index f428a3b55..054d501d1 100644 --- a/post-processor/compress/artifact.go +++ b/post-processor/compress/artifact.go @@ -5,7 +5,7 @@ import ( "os" ) -const BuilderId = "vtolstov.compress" +const BuilderId = "packer.post-processor.compress" type Artifact struct { builderId string diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index 74228d909..6d28e7c0e 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -1,10 +1,10 @@ package compress import ( - tar "archive/tar" - zip "archive/zip" + "archive/tar" + "archive/zip" "compress/flate" - gzip "compress/gzip" + "compress/gzip" "fmt" "io" "os" @@ -13,13 +13,13 @@ import ( "strings" "time" - bgzf "github.com/biogo/hts/bgzf" - pgzip "github.com/klauspost/pgzip" + "github.com/biogo/hts/bgzf" + "github.com/klauspost/pgzip" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" - lz4 "github.com/pierrec/lz4" + "github.com/pierrec/lz4" "gopkg.in/yaml.v2" ) From 3ac74bbae8cc2b1b3bb3bcd65454b2e29391418d Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 10 Jun 2015 14:07:13 -0700 Subject: [PATCH 05/28] Remove redundant aliases --- post-processor/compress/benchmark.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/post-processor/compress/benchmark.go b/post-processor/compress/benchmark.go index a2585bc89..ed4d68168 100644 --- a/post-processor/compress/benchmark.go +++ b/post-processor/compress/benchmark.go @@ -4,17 +4,17 @@ package main import ( "compress/flate" - gzip "compress/gzip" + "compress/gzip" + "fmt" "io" "io/ioutil" - "fmt" "os" "runtime" "testing" - bgzf "github.com/biogo/hts/bgzf" - pgzip "github.com/klauspost/pgzip" - lz4 "github.com/pierrec/lz4" + "github.com/biogo/hts/bgzf" + "github.com/klauspost/pgzip" + "github.com/pierrec/lz4" ) type Compressor struct { @@ -60,7 +60,7 @@ func NewCompressor(src, dst string) (*Compressor, error) { func main() { - runtime.GOMAXPROCS(runtime.NumCPU()) + runtime.GOMAXPROCS(runtime.NumCPU()) var resw testing.BenchmarkResult var resr testing.BenchmarkResult @@ -174,7 +174,7 @@ func (c *Compressor) BenchmarkPGZIPReader(b *testing.B) { func (c *Compressor) BenchmarkLZ4Writer(b *testing.B) { cw := lz4.NewWriter(c.w) -// cw.Header.HighCompression = true + // cw.Header.HighCompression = true cw.Header.NoChecksum = true b.ResetTimer() From 64fd3a3302c04868065edd5a874010a53e28f7a2 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Fri, 12 Jun 2015 17:24:03 -0700 Subject: [PATCH 06/28] Added file builder as a cheap, fast way to build something with output for testing post-processors --- builder/file/artifact.go | 36 ++++++++++++++++++++++++ builder/file/artifact_test.go | 11 ++++++++ builder/file/builder.go | 53 +++++++++++++++++++++++++++++++++++ builder/file/builder_test.go | 11 ++++++++ builder/file/config.go | 48 +++++++++++++++++++++++++++++++ builder/file/config_test.go | 35 +++++++++++++++++++++++ 6 files changed, 194 insertions(+) create mode 100644 builder/file/artifact.go create mode 100644 builder/file/artifact_test.go create mode 100644 builder/file/builder.go create mode 100644 builder/file/builder_test.go create mode 100644 builder/file/config.go create mode 100644 builder/file/config_test.go diff --git a/builder/file/artifact.go b/builder/file/artifact.go new file mode 100644 index 000000000..35bf06e6c --- /dev/null +++ b/builder/file/artifact.go @@ -0,0 +1,36 @@ +package file + +import ( + "fmt" + "log" + "os" +) + +type FileArtifact struct { + filename string +} + +func (*FileArtifact) BuilderId() string { + return BuilderId +} + +func (a *FileArtifact) Files() []string { + return []string{a.filename} +} + +func (a *FileArtifact) Id() string { + return "File" +} + +func (a *FileArtifact) String() string { + return fmt.Sprintf("Stored file: %s", a.filename) +} + +func (a *FileArtifact) State(name string) interface{} { + return nil +} + +func (a *FileArtifact) Destroy() error { + log.Printf("Deleting %s", a.filename) + return os.Remove(a.filename) +} diff --git a/builder/file/artifact_test.go b/builder/file/artifact_test.go new file mode 100644 index 000000000..0aa77894b --- /dev/null +++ b/builder/file/artifact_test.go @@ -0,0 +1,11 @@ +package file + +import ( + "testing" + + "github.com/mitchellh/packer/packer" +) + +func TestNullArtifact(t *testing.T) { + var _ packer.Artifact = new(FileArtifact) +} diff --git a/builder/file/builder.go b/builder/file/builder.go new file mode 100644 index 000000000..89047ab75 --- /dev/null +++ b/builder/file/builder.go @@ -0,0 +1,53 @@ +package file + +/* +The File builder creates an artifact from a file. Because it does not require +any virutalization or network resources, it's very fast and useful for testing. +*/ + +import ( + "io/ioutil" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" +) + +const BuilderId = "cbednarski.file" + +type Builder struct { + config *Config + runner multistep.Runner +} + +// Prepare is responsible for configuring the builder and validating +// that configuration. Any setup should be done in this method. Note that +// NO side effects should take place in prepare, it is meant as a state +// setup only. Calling Prepare is not necessarilly followed by a Run. +// +// The parameters to Prepare are a set of interface{} values of the +// configuration. These are almost always `map[string]interface{}` +// parsed from a template, but no guarantee is made. +// +// Each of the configuration values should merge into the final +// configuration. +// +// Prepare should return a list of warnings along with any errors +// that occured while preparing. +func (b *Builder) Prepare(...interface{}) ([]string, error) { + return nil, nil +} + +// Run is where the actual build should take place. It takes a Build and a Ui. +func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { + artifact := new(FileArtifact) + + ioutil.WriteFile(b.config.Filename, []byte(b.config.Contents), 0600) + + return artifact, nil +} + +// Cancel cancels a possibly running Builder. This should block until +// the builder actually cancels and cleans up after itself. +func (b *Builder) Cancel() { + b.runner.Cancel() +} diff --git a/builder/file/builder_test.go b/builder/file/builder_test.go new file mode 100644 index 000000000..63d36a0a5 --- /dev/null +++ b/builder/file/builder_test.go @@ -0,0 +1,11 @@ +package file + +import ( + "testing" + + "github.com/mitchellh/packer/packer" +) + +func TestBuilder_implBuilder(t *testing.T) { + var _ packer.Builder = new(Builder) +} diff --git a/builder/file/config.go b/builder/file/config.go new file mode 100644 index 000000000..534428ca4 --- /dev/null +++ b/builder/file/config.go @@ -0,0 +1,48 @@ +package file + +import ( + "fmt" + + "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/helper/config" + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/template/interpolate" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + Filename string `mapstructure:"filename"` + Contents string `mapstructure:"contents"` +} + +func NewConfig(raws ...interface{}) (*Config, []string, error) { + c := new(Config) + warnings := []string{} + + err := config.Decode(c, &config.DecodeOpts{ + Interpolate: true, + InterpolateFilter: &interpolate.RenderFilter{ + Exclude: []string{}, + }, + }, raws...) + if err != nil { + return nil, warnings, err + } + + var errs *packer.MultiError + + if c.Filename == "" { + errs = packer.MultiErrorAppend(errs, fmt.Errorf("filename is required")) + } + + if c.Contents == "" { + warnings = append(warnings, "contents is empty") + } + + if errs != nil && len(errs.Errors) > 0 { + return nil, warnings, errs + } + + return c, warnings, nil +} diff --git a/builder/file/config_test.go b/builder/file/config_test.go new file mode 100644 index 000000000..061bb97e5 --- /dev/null +++ b/builder/file/config_test.go @@ -0,0 +1,35 @@ +package file + +import ( + "fmt" + "testing" +) + +func testConfig() map[string]interface{} { + return map[string]interface{}{ + "filename": "test.txt", + "contents": "Hello, world!", + } +} + +func TestNoFilename(t *testing.T) { + raw := testConfig() + + delete(raw, "filename") + _, _, errs := NewConfig(raw) + if errs == nil { + t.Error("Expected config to error without a filename") + } +} + +func TestNoContent(t *testing.T) { + raw := testConfig() + + delete(raw, "contents") + _, warns, _ := NewConfig(raw) + fmt.Println(len(warns)) + fmt.Printf("%#v\n", warns) + if len(warns) == 0 { + t.Error("Expected config to warn without any content") + } +} From 53e4688529c75c632cafe8c43a5c0783b2c2dd1e Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Fri, 12 Jun 2015 17:25:09 -0700 Subject: [PATCH 07/28] Renamed some things to be more consistent with existing modules --- post-processor/compress/artifact.go | 25 ++++-- post-processor/compress/post-processor.go | 103 ++++++++++++---------- 2 files changed, 71 insertions(+), 57 deletions(-) diff --git a/post-processor/compress/artifact.go b/post-processor/compress/artifact.go index 054d501d1..cfc914a55 100644 --- a/post-processor/compress/artifact.go +++ b/post-processor/compress/artifact.go @@ -8,25 +8,32 @@ import ( const BuilderId = "packer.post-processor.compress" type Artifact struct { - builderId string - dir string - f []string + Path string + Provider string + files []string +} + +func NewArtifact(provider, path string) *Artifact { + return &Artifact{ + Path: path, + Provider: provider, + } } func (a *Artifact) BuilderId() string { return BuilderId } -func (a *Artifact) Files() []string { - return a.f +func (*Artifact) Id() string { + return "" } -func (*Artifact) Id() string { - return "COMPRESS" +func (a *Artifact) Files() []string { + return a.files } func (a *Artifact) String() string { - return fmt.Sprintf("VM compressed files in directory: %s", a.dir) + return fmt.Sprintf("'%s' compressing: %s", a.Provider, a.Path) } func (*Artifact) State(name string) interface{} { @@ -34,5 +41,5 @@ func (*Artifact) State(name string) interface{} { } func (a *Artifact) Destroy() error { - return os.RemoveAll(a.dir) + return os.Remove(a.Path) } diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index 6d28e7c0e..340a75dd4 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -46,47 +46,45 @@ type Config struct { } type PostProcessor struct { - cfg Config + config Config } func (p *PostProcessor) Configure(raws ...interface{}) error { - p.cfg.Compression = -1 - err := config.Decode(&p.cfg, &config.DecodeOpts{ + p.config.Compression = -1 + err := config.Decode(&p.config, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ - Exclude: []string{ - // TODO figure out if something needs to go here. - }, + Exclude: []string{}, }, }, raws...) errs := new(packer.MultiError) - if p.cfg.OutputPath == "" { - p.cfg.OutputPath = "packer_{{.BuildName}}_{{.Provider}}" + if p.config.OutputPath == "" { + p.config.OutputPath = "packer_{{.BuildName}}_{{.Provider}}" } - if err = interpolate.Validate(p.cfg.OutputPath, p.cfg.ctx); err != nil { + if err = interpolate.Validate(p.config.OutputPath, p.config.ctx); err != nil { errs = packer.MultiErrorAppend( errs, fmt.Errorf("Error parsing target template: %s", err)) } templates := map[string]*string{ - "output": &p.cfg.OutputPath, + "output": &p.config.OutputPath, } - if p.cfg.Compression > flate.BestCompression { - p.cfg.Compression = flate.BestCompression + if p.config.Compression > flate.BestCompression { + p.config.Compression = flate.BestCompression } - if p.cfg.Compression == -1 { - p.cfg.Compression = flate.DefaultCompression + if p.config.Compression == -1 { + p.config.Compression = flate.DefaultCompression } - if p.cfg.NumCPU < 1 { - p.cfg.NumCPU = runtime.NumCPU() + if p.config.NumCPU < 1 { + p.config.NumCPU = runtime.NumCPU() } - runtime.GOMAXPROCS(p.cfg.NumCPU) + runtime.GOMAXPROCS(p.config.NumCPU) for key, ptr := range templates { if *ptr == "" { @@ -94,7 +92,7 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { errs, fmt.Errorf("%s must be set", key)) } - *ptr, err = interpolate.Render(p.cfg.OutputPath, p.cfg.ctx) + *ptr, err = interpolate.Render(p.config.OutputPath, p.config.ctx) if err != nil { errs = packer.MultiErrorAppend( errs, fmt.Errorf("Error processing %s: %s", key, err)) @@ -114,7 +112,7 @@ func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata const layout = "2006-01-02_15-04-05" t := time.Now() - if !p.cfg.Metadata { + if !p.config.Metadata { return metadata } for _, f := range files { @@ -122,7 +120,7 @@ func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata continue } else { if i, ok := metadata[filepath.Base(f)]; !ok { - metadata[filepath.Base(f)] = Metaitem{CompType: p.cfg.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)} + metadata[filepath.Base(f)] = Metaitem{CompType: p.config.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)} } else { i.CompSize = fi.Size() i.CompDate = t.Format(layout) @@ -134,46 +132,55 @@ func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata } func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - newartifact := &Artifact{builderId: artifact.BuilderId(), dir: p.cfg.OutputPath} - metafile := filepath.Join(p.cfg.OutputPath, "metadata") + newartifact := &Artifact{Path: p.config.OutputPath} + metafile := filepath.Join(p.config.OutputPath, "metadata") - _, err := os.Stat(newartifact.dir) + ui.Say(fmt.Sprintf("[CBEDNARSKI] Creating archive at %s", newartifact.Path)) + _, err := os.Stat(newartifact.Path) if err == nil { - return nil, false, fmt.Errorf("output dir must not exists: %s", err) + return nil, false, fmt.Errorf("output dir %s must not exists", newartifact.Path) } - err = os.MkdirAll(newartifact.dir, 0755) + err = os.MkdirAll(newartifact.Path, 0755) if err != nil { return nil, false, fmt.Errorf("failed to create output: %s", err) } - formats := strings.Split(p.cfg.Format, ".") + p.config.Format += "tar.gzip" + formats := strings.Split(p.config.Format, ".") + ui.Say(fmt.Sprintf("[CBEDNARSKI] Formats length %d", len(formats))) + if len(p.config.Format) == 0 { + ui.Say("[CBEDNARSKI] Formats is empty") + formats[0] = "tar.gzip" + } files := artifact.Files() metadata := make(Metadata, 0) metadata = p.fillMetadata(metadata, files) + ui.Say(fmt.Sprintf("[CBEDNARSKI] Formats %#v", formats)) + for _, compress := range formats { switch compress { case "tar": - files, err = p.cmpTAR(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) + files, err = p.cmpTAR(files, filepath.Join(p.config.OutputPath, p.config.OutputFile)) metadata = p.fillMetadata(metadata, files) case "zip": - files, err = p.cmpZIP(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) + files, err = p.cmpZIP(files, filepath.Join(p.config.OutputPath, p.config.OutputFile)) metadata = p.fillMetadata(metadata, files) case "pgzip": - files, err = p.cmpPGZIP(files, p.cfg.OutputPath) + files, err = p.cmpPGZIP(files, p.config.OutputPath) metadata = p.fillMetadata(metadata, files) case "gzip": - files, err = p.cmpGZIP(files, p.cfg.OutputPath) + files, err = p.cmpGZIP(files, p.config.OutputPath) metadata = p.fillMetadata(metadata, files) case "bgzf": - files, err = p.cmpBGZF(files, p.cfg.OutputPath) + files, err = p.cmpBGZF(files, p.config.OutputPath) metadata = p.fillMetadata(metadata, files) case "lz4": - files, err = p.cmpLZ4(files, p.cfg.OutputPath) + files, err = p.cmpLZ4(files, p.config.OutputPath) metadata = p.fillMetadata(metadata, files) case "e2fs": - files, err = p.cmpE2FS(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) + files, err = p.cmpE2FS(files, filepath.Join(p.config.OutputPath, p.config.OutputFile)) metadata = p.fillMetadata(metadata, files) } if err != nil { @@ -181,7 +188,7 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac } } - if p.cfg.Metadata { + if p.config.Metadata { fp, err := os.Create(metafile) if err != nil { return nil, false, err @@ -198,18 +205,18 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac } } - newartifact.f = append(newartifact.f, files...) - if p.cfg.Metadata { - newartifact.f = append(newartifact.f, metafile) + newartifact.files = append(newartifact.files, files...) + if p.config.Metadata { + newartifact.files = append(newartifact.files, metafile) } - return newartifact, p.cfg.KeepInputArtifact, nil + return newartifact, p.config.KeepInputArtifact, nil } func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { fw, err := os.Create(dst) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, fmt.Errorf("tar error creating tar %s: %s", dst, err) } defer fw.Close() @@ -219,27 +226,27 @@ func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { for _, name := range src { fi, err := os.Stat(name) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, fmt.Errorf("tar error on stat of %s: %s", name, err) } target, _ := os.Readlink(name) header, err := tar.FileInfoHeader(fi, target) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, fmt.Errorf("tar error reading info for %s: %s", name, err) } if err = tw.WriteHeader(header); err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, fmt.Errorf("tar error writing header for %s: %s", name, err) } fr, err := os.Open(name) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, fmt.Errorf("tar error opening file %s: %s", name, err) } if _, err = io.Copy(tw, fr); err != nil { fr.Close() - return nil, fmt.Errorf("tar error: %s", err) + return nil, fmt.Errorf("tar error copying contents of %s: %s", name, err) } fr.Close() } @@ -254,7 +261,7 @@ func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { if err != nil { return nil, fmt.Errorf("gzip error: %s", err) } - cw, err := gzip.NewWriterLevel(fw, p.cfg.Compression) + cw, err := gzip.NewWriterLevel(fw, p.config.Compression) if err != nil { fw.Close() return nil, fmt.Errorf("gzip error: %s", err) @@ -287,7 +294,7 @@ func (p *PostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) { if err != nil { return nil, fmt.Errorf("pgzip error: %s", err) } - cw, err := pgzip.NewWriterLevel(fw, p.cfg.Compression) + cw, err := pgzip.NewWriterLevel(fw, p.config.Compression) if err != nil { fw.Close() return nil, fmt.Errorf("pgzip error: %s", err) @@ -325,7 +332,7 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { fw.Close() return nil, fmt.Errorf("lz4 error: %s", err) } - if p.cfg.Compression > flate.DefaultCompression { + if p.config.Compression > flate.DefaultCompression { cw.Header.HighCompression = true } fr, err := os.Open(name) @@ -357,7 +364,7 @@ func (p *PostProcessor) cmpBGZF(src []string, dst string) ([]string, error) { return nil, fmt.Errorf("bgzf error: %s", err) } - cw, err := bgzf.NewWriterLevel(fw, p.cfg.Compression, runtime.NumCPU()) + cw, err := bgzf.NewWriterLevel(fw, p.config.Compression, runtime.NumCPU()) if err != nil { return nil, fmt.Errorf("bgzf error: %s", err) } From 766d217ed71f511b12a30e13efa829bcf3b05b23 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Fri, 12 Jun 2015 17:34:46 -0700 Subject: [PATCH 08/28] Pull config into the builder --- builder/file/builder.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/builder/file/builder.go b/builder/file/builder.go index 89047ab75..3b00aae60 100644 --- a/builder/file/builder.go +++ b/builder/file/builder.go @@ -33,8 +33,14 @@ type Builder struct { // // Prepare should return a list of warnings along with any errors // that occured while preparing. -func (b *Builder) Prepare(...interface{}) ([]string, error) { - return nil, nil +func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { + c, warnings, errs := NewConfig(raws...) + if errs != nil { + return warnings, errs + } + b.config = c + + return warnings, nil } // Run is where the actual build should take place. It takes a Build and a Ui. From f7d85eb49cf169f1dfdf66d20320032ff370fff6 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Fri, 12 Jun 2015 17:35:17 -0700 Subject: [PATCH 09/28] Add main() for file builder --- plugin/builder-file/main.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 plugin/builder-file/main.go diff --git a/plugin/builder-file/main.go b/plugin/builder-file/main.go new file mode 100644 index 000000000..54bc4f437 --- /dev/null +++ b/plugin/builder-file/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/mitchellh/packer/builder/file" + "github.com/mitchellh/packer/packer/plugin" +) + +func main() { + server, err := plugin.Server() + if err != nil { + panic(err) + } + server.RegisterBuilder(new(file.Builder)) + server.Serve() +} From e60b22d48f8b557c620ff083a55263acedc19d55 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Fri, 12 Jun 2015 18:18:38 -0700 Subject: [PATCH 10/28] Changed file builder to support content or source file operation --- builder/file/builder.go | 33 ++++++++++++++++++++++++++++++++- builder/file/config.go | 20 ++++++++++++++------ builder/file/config_test.go | 22 +++++++++++++++++----- 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/builder/file/builder.go b/builder/file/builder.go index 3b00aae60..ea3206dad 100644 --- a/builder/file/builder.go +++ b/builder/file/builder.go @@ -6,7 +6,10 @@ any virutalization or network resources, it's very fast and useful for testing. */ import ( + "fmt" + "io" "io/ioutil" + "os" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" @@ -47,7 +50,35 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { artifact := new(FileArtifact) - ioutil.WriteFile(b.config.Filename, []byte(b.config.Contents), 0600) + if b.config.Source != "" { + source, err := os.Open(b.config.Source) + defer source.Close() + if err != nil { + return nil, err + } + + target, err := os.OpenFile(b.config.Target, os.O_WRONLY, 0600) + defer target.Close() + if err != nil { + return nil, err + } + + ui.Say(fmt.Sprintf("Copying %s to %s", source.Name(), target.Name())) + bytes, err := io.Copy(source, target) + if err != nil { + return nil, err + } + ui.Say(fmt.Sprintf("Copied %d bytes", bytes)) + artifact.filename = target.Name() + } else { + // We're going to write Contents; if it's empty we'll just create an + // empty file. + err := ioutil.WriteFile(b.config.Target, []byte(b.config.Content), 0600) + if err != nil { + return nil, err + } + artifact.filename = b.config.Target + } return artifact, nil } diff --git a/builder/file/config.go b/builder/file/config.go index 534428ca4..6702e6894 100644 --- a/builder/file/config.go +++ b/builder/file/config.go @@ -9,11 +9,15 @@ import ( "github.com/mitchellh/packer/template/interpolate" ) +var ErrTargetRequired = fmt.Errorf("target required") +var ErrContentSourceConflict = fmt.Errorf("Cannot specify source file AND content") + type Config struct { common.PackerConfig `mapstructure:",squash"` - Filename string `mapstructure:"filename"` - Contents string `mapstructure:"contents"` + Source string `mapstructure:"source"` + Target string `mapstructure:"target"` + Content string `mapstructure:"content"` } func NewConfig(raws ...interface{}) (*Config, []string, error) { @@ -32,12 +36,16 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { var errs *packer.MultiError - if c.Filename == "" { - errs = packer.MultiErrorAppend(errs, fmt.Errorf("filename is required")) + if c.Target == "" { + errs = packer.MultiErrorAppend(errs, ErrTargetRequired) } - if c.Contents == "" { - warnings = append(warnings, "contents is empty") + if c.Content == "" && c.Source == "" { + warnings = append(warnings, "Both source file and contents are blank; target will have no content") + } + + if c.Content != "" && c.Source != "" { + errs = packer.MultiErrorAppend(errs, ErrContentSourceConflict) } if errs != nil && len(errs.Errors) > 0 { diff --git a/builder/file/config_test.go b/builder/file/config_test.go index 061bb97e5..6d8039558 100644 --- a/builder/file/config_test.go +++ b/builder/file/config_test.go @@ -2,13 +2,24 @@ package file import ( "fmt" + "strings" "testing" ) func testConfig() map[string]interface{} { return map[string]interface{}{ - "filename": "test.txt", - "contents": "Hello, world!", + "source": "src.txt", + "target": "dst.txt", + "content": "Hello, world!", + } +} + +func TestContentSourceConflict(t *testing.T) { + raw := testConfig() + + _, _, errs := NewConfig(raw) + if !strings.Contains(errs.Error(), ErrContentSourceConflict.Error()) { + t.Errorf("Expected config error: %s", ErrContentSourceConflict.Error()) } } @@ -18,18 +29,19 @@ func TestNoFilename(t *testing.T) { delete(raw, "filename") _, _, errs := NewConfig(raw) if errs == nil { - t.Error("Expected config to error without a filename") + t.Errorf("Expected config error: %s", ErrTargetRequired.Error()) } } func TestNoContent(t *testing.T) { raw := testConfig() - delete(raw, "contents") + delete(raw, "content") + delete(raw, "source") _, warns, _ := NewConfig(raw) fmt.Println(len(warns)) fmt.Printf("%#v\n", warns) if len(warns) == 0 { - t.Error("Expected config to warn without any content") + t.Error("Expected config warning without any content") } } From 29f02d243f2c16be24be9d3f6d1c13d9343aa7ac Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Mon, 15 Jun 2015 18:56:09 -0700 Subject: [PATCH 11/28] Had io.Copy args swapped; also use os.Create instead of os.OpenFile for MAGIC --- builder/file/builder.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builder/file/builder.go b/builder/file/builder.go index ea3206dad..9297b456d 100644 --- a/builder/file/builder.go +++ b/builder/file/builder.go @@ -57,14 +57,15 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe return nil, err } - target, err := os.OpenFile(b.config.Target, os.O_WRONLY, 0600) + // Create will truncate an existing file + target, err := os.Create(b.config.Target) defer target.Close() if err != nil { return nil, err } ui.Say(fmt.Sprintf("Copying %s to %s", source.Name(), target.Name())) - bytes, err := io.Copy(source, target) + bytes, err := io.Copy(target, source) if err != nil { return nil, err } From fe0c548619b2288c3a196369ed0b27ff21eb27e0 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 16 Jun 2015 11:30:49 -0700 Subject: [PATCH 12/28] Added acceptance test for file builder --- builder/file/builder.go | 16 +---- builder/file/builder_test.go | 67 +++++++++++++++++++ builder/file/config_test.go | 4 +- builder/file/test-fixtures/artifact.txt | 1 + .../compress/post-processor_test.go | 21 +++++- 5 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 builder/file/test-fixtures/artifact.txt diff --git a/builder/file/builder.go b/builder/file/builder.go index 9297b456d..9a2c2cc7f 100644 --- a/builder/file/builder.go +++ b/builder/file/builder.go @@ -15,27 +15,13 @@ import ( "github.com/mitchellh/packer/packer" ) -const BuilderId = "cbednarski.file" +const BuilderId = "packer.file" type Builder struct { config *Config runner multistep.Runner } -// Prepare is responsible for configuring the builder and validating -// that configuration. Any setup should be done in this method. Note that -// NO side effects should take place in prepare, it is meant as a state -// setup only. Calling Prepare is not necessarilly followed by a Run. -// -// The parameters to Prepare are a set of interface{} values of the -// configuration. These are almost always `map[string]interface{}` -// parsed from a template, but no guarantee is made. -// -// Each of the configuration values should merge into the final -// configuration. -// -// Prepare should return a list of warnings along with any errors -// that occured while preparing. func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { c, warnings, errs := NewConfig(raws...) if errs != nil { diff --git a/builder/file/builder_test.go b/builder/file/builder_test.go index 63d36a0a5..3ce9e77ae 100644 --- a/builder/file/builder_test.go +++ b/builder/file/builder_test.go @@ -1,11 +1,78 @@ package file import ( + "fmt" + "io/ioutil" "testing" + builderT "github.com/mitchellh/packer/helper/builder/testing" "github.com/mitchellh/packer/packer" ) func TestBuilder_implBuilder(t *testing.T) { var _ packer.Builder = new(Builder) } + +func TestBuilderFileAcc_content(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + Builder: &Builder{}, + Template: fileContentTest, + Check: checkContent, + }) +} + +func TestBuilderFileAcc_copy(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + Builder: &Builder{}, + Template: fileCopyTest, + Check: checkCopy, + }) +} + +func checkContent(artifacts []packer.Artifact) error { + content, err := ioutil.ReadFile("contentTest.txt") + if err != nil { + return err + } + contentString := string(content) + if contentString != "hello world!" { + return fmt.Errorf("Unexpected file contents: %s", contentString) + } + return nil +} + +func checkCopy(artifacts []packer.Artifact) error { + content, err := ioutil.ReadFile("copyTest.txt") + if err != nil { + return err + } + contentString := string(content) + if contentString != "Hello world.\n" { + return fmt.Errorf("Unexpected file contents: %s", contentString) + } + return nil +} + +const fileContentTest = ` +{ + "builders": [ + { + "type":"test", + "target":"contentTest.txt", + "content":"hello world!" + } + ] +} +` + +const fileCopyTest = ` +{ + "builders": [ + { + "type":"test", + "target":"copyTest.txt", + "source":"test-fixtures/artifact.txt" + } + ] +} +` diff --git a/builder/file/config_test.go b/builder/file/config_test.go index 6d8039558..9d8f346fc 100644 --- a/builder/file/config_test.go +++ b/builder/file/config_test.go @@ -1,7 +1,6 @@ package file import ( - "fmt" "strings" "testing" ) @@ -39,8 +38,7 @@ func TestNoContent(t *testing.T) { delete(raw, "content") delete(raw, "source") _, warns, _ := NewConfig(raw) - fmt.Println(len(warns)) - fmt.Printf("%#v\n", warns) + if len(warns) == 0 { t.Error("Expected config warning without any content") } diff --git a/builder/file/test-fixtures/artifact.txt b/builder/file/test-fixtures/artifact.txt new file mode 100644 index 000000000..18249f335 --- /dev/null +++ b/builder/file/test-fixtures/artifact.txt @@ -0,0 +1 @@ +Hello world. diff --git a/post-processor/compress/post-processor_test.go b/post-processor/compress/post-processor_test.go index 92cbfc4b3..5f4d6b9ca 100644 --- a/post-processor/compress/post-processor_test.go +++ b/post-processor/compress/post-processor_test.go @@ -1,3 +1,22 @@ package compress -import () +// import ( +// "testing" +// +// builderT "github.com/mitchellh/packer/helper/builder/testing" +// ) +// +// func TestBuilderTagsAcc_basic(t *testing.T) { +// builderT.Test(t, builderT.TestCase{ +// Builder: &Builder{}, +// Template: simpleTestCase, +// Check: checkTags(), +// }) +// } + +const simpleTestCase = ` +{ + "type": "compress", + "output": "foo.tar.gz" +} +` From aea70d5a720b9cc415bd57f46707a0cb4e4193dd Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 16 Jun 2015 11:31:53 -0700 Subject: [PATCH 13/28] Added acceptance test for file builder --- builder/file/builder.go | 16 +----- builder/file/builder_test.go | 67 +++++++++++++++++++++++++ builder/file/config_test.go | 4 +- builder/file/test-fixtures/artifact.txt | 1 + 4 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 builder/file/test-fixtures/artifact.txt diff --git a/builder/file/builder.go b/builder/file/builder.go index 9297b456d..9a2c2cc7f 100644 --- a/builder/file/builder.go +++ b/builder/file/builder.go @@ -15,27 +15,13 @@ import ( "github.com/mitchellh/packer/packer" ) -const BuilderId = "cbednarski.file" +const BuilderId = "packer.file" type Builder struct { config *Config runner multistep.Runner } -// Prepare is responsible for configuring the builder and validating -// that configuration. Any setup should be done in this method. Note that -// NO side effects should take place in prepare, it is meant as a state -// setup only. Calling Prepare is not necessarilly followed by a Run. -// -// The parameters to Prepare are a set of interface{} values of the -// configuration. These are almost always `map[string]interface{}` -// parsed from a template, but no guarantee is made. -// -// Each of the configuration values should merge into the final -// configuration. -// -// Prepare should return a list of warnings along with any errors -// that occured while preparing. func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { c, warnings, errs := NewConfig(raws...) if errs != nil { diff --git a/builder/file/builder_test.go b/builder/file/builder_test.go index 63d36a0a5..3ce9e77ae 100644 --- a/builder/file/builder_test.go +++ b/builder/file/builder_test.go @@ -1,11 +1,78 @@ package file import ( + "fmt" + "io/ioutil" "testing" + builderT "github.com/mitchellh/packer/helper/builder/testing" "github.com/mitchellh/packer/packer" ) func TestBuilder_implBuilder(t *testing.T) { var _ packer.Builder = new(Builder) } + +func TestBuilderFileAcc_content(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + Builder: &Builder{}, + Template: fileContentTest, + Check: checkContent, + }) +} + +func TestBuilderFileAcc_copy(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + Builder: &Builder{}, + Template: fileCopyTest, + Check: checkCopy, + }) +} + +func checkContent(artifacts []packer.Artifact) error { + content, err := ioutil.ReadFile("contentTest.txt") + if err != nil { + return err + } + contentString := string(content) + if contentString != "hello world!" { + return fmt.Errorf("Unexpected file contents: %s", contentString) + } + return nil +} + +func checkCopy(artifacts []packer.Artifact) error { + content, err := ioutil.ReadFile("copyTest.txt") + if err != nil { + return err + } + contentString := string(content) + if contentString != "Hello world.\n" { + return fmt.Errorf("Unexpected file contents: %s", contentString) + } + return nil +} + +const fileContentTest = ` +{ + "builders": [ + { + "type":"test", + "target":"contentTest.txt", + "content":"hello world!" + } + ] +} +` + +const fileCopyTest = ` +{ + "builders": [ + { + "type":"test", + "target":"copyTest.txt", + "source":"test-fixtures/artifact.txt" + } + ] +} +` diff --git a/builder/file/config_test.go b/builder/file/config_test.go index 6d8039558..9d8f346fc 100644 --- a/builder/file/config_test.go +++ b/builder/file/config_test.go @@ -1,7 +1,6 @@ package file import ( - "fmt" "strings" "testing" ) @@ -39,8 +38,7 @@ func TestNoContent(t *testing.T) { delete(raw, "content") delete(raw, "source") _, warns, _ := NewConfig(raw) - fmt.Println(len(warns)) - fmt.Printf("%#v\n", warns) + if len(warns) == 0 { t.Error("Expected config warning without any content") } diff --git a/builder/file/test-fixtures/artifact.txt b/builder/file/test-fixtures/artifact.txt new file mode 100644 index 000000000..18249f335 --- /dev/null +++ b/builder/file/test-fixtures/artifact.txt @@ -0,0 +1 @@ +Hello world. From 12cf6650a075feab94e5f6695734c2ab4700d7b0 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 16 Jun 2015 12:10:28 -0700 Subject: [PATCH 14/28] Revert compress post-processor to master to get baseline test --- post-processor/compress/artifact.go | 30 +- post-processor/compress/post-processor.go | 405 +++------------------- 2 files changed, 61 insertions(+), 374 deletions(-) diff --git a/post-processor/compress/artifact.go b/post-processor/compress/artifact.go index 054d501d1..34a7ce8d6 100644 --- a/post-processor/compress/artifact.go +++ b/post-processor/compress/artifact.go @@ -8,31 +8,37 @@ import ( const BuilderId = "packer.post-processor.compress" type Artifact struct { - builderId string - dir string - f []string + Path string + Provider string } -func (a *Artifact) BuilderId() string { +func NewArtifact(provider, path string) *Artifact { + return &Artifact{ + Path: path, + Provider: provider, + } +} + +func (*Artifact) BuilderId() string { return BuilderId } -func (a *Artifact) Files() []string { - return a.f +func (self *Artifact) Id() string { + return "" } -func (*Artifact) Id() string { - return "COMPRESS" +func (self *Artifact) Files() []string { + return []string{self.Path} } -func (a *Artifact) String() string { - return fmt.Sprintf("VM compressed files in directory: %s", a.dir) +func (self *Artifact) String() string { + return fmt.Sprintf("'%s' compressing: %s", self.Provider, self.Path) } func (*Artifact) State(name string) interface{} { return nil } -func (a *Artifact) Destroy() error { - return os.RemoveAll(a.dir) +func (self *Artifact) Destroy() error { + return os.Remove(self.Path) } diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index 6d28e7c0e..ccf300946 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -2,416 +2,97 @@ package compress import ( "archive/tar" - "archive/zip" - "compress/flate" "compress/gzip" "fmt" "io" "os" - "path/filepath" - "runtime" - "strings" - "time" - "github.com/biogo/hts/bgzf" - "github.com/klauspost/pgzip" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" - "github.com/pierrec/lz4" - "gopkg.in/yaml.v2" ) -type Metadata map[string]Metaitem - -type Metaitem struct { - CompSize int64 `yaml:"compsize"` - OrigSize int64 `yaml:"origsize"` - CompType string `yaml:"comptype"` - CompDate string `yaml:"compdate"` -} - type Config struct { common.PackerConfig `mapstructure:",squash"` - OutputPath string `mapstructure:"output"` - OutputFile string `mapstructure:"file"` - Compression int `mapstructure:"compression"` - Metadata bool `mapstructure:"metadata"` - NumCPU int `mapstructure:"numcpu"` - Format string `mapstructure:"format"` - KeepInputArtifact bool `mapstructure:"keep_input_artifact"` - ctx *interpolate.Context + OutputPath string `mapstructure:"output"` + + ctx interpolate.Context } type PostProcessor struct { - cfg Config + config Config } -func (p *PostProcessor) Configure(raws ...interface{}) error { - p.cfg.Compression = -1 - err := config.Decode(&p.cfg, &config.DecodeOpts{ +func (self *PostProcessor) Configure(raws ...interface{}) error { + err := config.Decode(&self.config, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ - Exclude: []string{ - // TODO figure out if something needs to go here. - }, + Exclude: []string{}, }, }, raws...) - - errs := new(packer.MultiError) - - if p.cfg.OutputPath == "" { - p.cfg.OutputPath = "packer_{{.BuildName}}_{{.Provider}}" - } - - if err = interpolate.Validate(p.cfg.OutputPath, p.cfg.ctx); err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Error parsing target template: %s", err)) - } - - templates := map[string]*string{ - "output": &p.cfg.OutputPath, - } - - if p.cfg.Compression > flate.BestCompression { - p.cfg.Compression = flate.BestCompression - } - if p.cfg.Compression == -1 { - p.cfg.Compression = flate.DefaultCompression - } - - if p.cfg.NumCPU < 1 { - p.cfg.NumCPU = runtime.NumCPU() - } - - runtime.GOMAXPROCS(p.cfg.NumCPU) - - for key, ptr := range templates { - if *ptr == "" { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("%s must be set", key)) - } - - *ptr, err = interpolate.Render(p.cfg.OutputPath, p.cfg.ctx) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Error processing %s: %s", key, err)) - } - } - - if len(errs.Errors) > 0 { - return errs + if err != nil { + return err } return nil } -func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata { - // layout shows by example how the reference time should be represented. - const layout = "2006-01-02_15-04-05" - t := time.Now() +func (self *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { + ui.Say(fmt.Sprintf("Creating archive for '%s'", artifact.BuilderId())) - if !p.cfg.Metadata { - return metadata - } - for _, f := range files { - if fi, err := os.Stat(f); err != nil { - continue - } else { - if i, ok := metadata[filepath.Base(f)]; !ok { - metadata[filepath.Base(f)] = Metaitem{CompType: p.cfg.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)} - } else { - i.CompSize = fi.Size() - i.CompDate = t.Format(layout) - metadata[filepath.Base(f)] = i - } - } - } - return metadata -} - -func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - newartifact := &Artifact{builderId: artifact.BuilderId(), dir: p.cfg.OutputPath} - metafile := filepath.Join(p.cfg.OutputPath, "metadata") - - _, err := os.Stat(newartifact.dir) - if err == nil { - return nil, false, fmt.Errorf("output dir must not exists: %s", err) - } - err = os.MkdirAll(newartifact.dir, 0755) + // Create the compressed archive file at the appropriate OutputPath. + fw, err := os.Create(self.config.OutputPath) if err != nil { - return nil, false, fmt.Errorf("failed to create output: %s", err) - } - - formats := strings.Split(p.cfg.Format, ".") - files := artifact.Files() - - metadata := make(Metadata, 0) - metadata = p.fillMetadata(metadata, files) - - for _, compress := range formats { - switch compress { - case "tar": - files, err = p.cmpTAR(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) - metadata = p.fillMetadata(metadata, files) - case "zip": - files, err = p.cmpZIP(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) - metadata = p.fillMetadata(metadata, files) - case "pgzip": - files, err = p.cmpPGZIP(files, p.cfg.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "gzip": - files, err = p.cmpGZIP(files, p.cfg.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "bgzf": - files, err = p.cmpBGZF(files, p.cfg.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "lz4": - files, err = p.cmpLZ4(files, p.cfg.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "e2fs": - files, err = p.cmpE2FS(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) - metadata = p.fillMetadata(metadata, files) - } - if err != nil { - return nil, false, fmt.Errorf("Failed to compress: %s", err) - } - } - - if p.cfg.Metadata { - fp, err := os.Create(metafile) - if err != nil { - return nil, false, err - } - if buf, err := yaml.Marshal(metadata); err != nil { - fp.Close() - return nil, false, err - } else { - if _, err = fp.Write(buf); err != nil { - fp.Close() - return nil, false, err - } - fp.Close() - } - } - - newartifact.f = append(newartifact.f, files...) - if p.cfg.Metadata { - newartifact.f = append(newartifact.f, metafile) - } - - return newartifact, p.cfg.KeepInputArtifact, nil -} - -func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { - fw, err := os.Create(dst) - if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, false, fmt.Errorf( + "Failed creating file for compressed archive: %s", self.config.OutputPath) } defer fw.Close() - tw := tar.NewWriter(fw) - defer tw.Close() + gw := gzip.NewWriter(fw) + defer gw.Close() - for _, name := range src { - fi, err := os.Stat(name) + // Iterate through all of the artifact's files and put them into the + // compressed archive using the tar/gzip writers. + for _, path := range artifact.Files() { + fi, err := os.Stat(path) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, false, fmt.Errorf( + "Failed stating file: %s", path) } - target, _ := os.Readlink(name) + target, _ := os.Readlink(path) header, err := tar.FileInfoHeader(fi, target) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, false, fmt.Errorf( + "Failed creating archive header: %s", path) } - if err = tw.WriteHeader(header); err != nil { - return nil, fmt.Errorf("tar error: %s", err) + tw := tar.NewWriter(gw) + defer tw.Close() + + // Write the header first to the archive. This takes partial data + // from the FileInfo that is grabbed by running the stat command. + if err := tw.WriteHeader(header); err != nil { + return nil, false, fmt.Errorf( + "Failed writing archive header: %s", path) } - fr, err := os.Open(name) + // Open the target file for archiving and compressing. + fr, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, false, fmt.Errorf( + "Failed opening file '%s' to write compressed archive.", path) } + defer fr.Close() if _, err = io.Copy(tw, fr); err != nil { - fr.Close() - return nil, fmt.Errorf("tar error: %s", err) + return nil, false, fmt.Errorf( + "Failed copying file to archive: %s", path) } - fr.Close() } - return []string{dst}, nil -} - -func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("gzip error: %s", err) - } - cw, err := gzip.NewWriterLevel(fw, p.cfg.Compression) - if err != nil { - fw.Close() - return nil, fmt.Errorf("gzip error: %s", err) - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("gzip error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("gzip error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("pgzip error: %s", err) - } - cw, err := pgzip.NewWriterLevel(fw, p.cfg.Compression) - if err != nil { - fw.Close() - return nil, fmt.Errorf("pgzip error: %s", err) - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("pgzip error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("pgzip error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("lz4 error: %s", err) - } - cw := lz4.NewWriter(fw) - if err != nil { - fw.Close() - return nil, fmt.Errorf("lz4 error: %s", err) - } - if p.cfg.Compression > flate.DefaultCompression { - cw.Header.HighCompression = true - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("lz4 error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("lz4 error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpBGZF(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("bgzf error: %s", err) - } - - cw, err := bgzf.NewWriterLevel(fw, p.cfg.Compression, runtime.NumCPU()) - if err != nil { - return nil, fmt.Errorf("bgzf error: %s", err) - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("bgzf error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("bgzf error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpE2FS(src []string, dst string) ([]string, error) { - panic("not implemented") -} - -func (p *PostProcessor) cmpZIP(src []string, dst string) ([]string, error) { - fw, err := os.Create(dst) - if err != nil { - return nil, fmt.Errorf("zip error: %s", err) - } - defer fw.Close() - - zw := zip.NewWriter(fw) - defer zw.Close() - - for _, name := range src { - header, err := zw.Create(name) - if err != nil { - return nil, fmt.Errorf("zip error: %s", err) - } - - fr, err := os.Open(name) - if err != nil { - return nil, fmt.Errorf("zip error: %s", err) - } - - if _, err = io.Copy(header, fr); err != nil { - fr.Close() - return nil, fmt.Errorf("zip error: %s", err) - } - fr.Close() - } - return []string{dst}, nil - + + return NewArtifact(artifact.BuilderId(), self.config.OutputPath), false, nil } From fe105107d25d909d07344c2a58dd524983b652d6 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 16 Jun 2015 12:11:11 -0700 Subject: [PATCH 15/28] Removed extra files -- will re-add later --- post-processor/compress/LICENSE | 21 --- post-processor/compress/benchmark.go | 197 --------------------------- 2 files changed, 218 deletions(-) delete mode 100644 post-processor/compress/LICENSE delete mode 100644 post-processor/compress/benchmark.go diff --git a/post-processor/compress/LICENSE b/post-processor/compress/LICENSE deleted file mode 100644 index 38bbf26f3..000000000 --- a/post-processor/compress/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Vasiliy Tolstov - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/post-processor/compress/benchmark.go b/post-processor/compress/benchmark.go deleted file mode 100644 index ed4d68168..000000000 --- a/post-processor/compress/benchmark.go +++ /dev/null @@ -1,197 +0,0 @@ -// +build ignore - -package main - -import ( - "compress/flate" - "compress/gzip" - "fmt" - "io" - "io/ioutil" - "os" - "runtime" - "testing" - - "github.com/biogo/hts/bgzf" - "github.com/klauspost/pgzip" - "github.com/pierrec/lz4" -) - -type Compressor struct { - r *os.File - w *os.File - sr int64 - sw int64 -} - -func (c *Compressor) Close() error { - var err error - - fi, _ := c.w.Stat() - c.sw = fi.Size() - if err = c.w.Close(); err != nil { - return err - } - - fi, _ = c.r.Stat() - c.sr = fi.Size() - if err = c.r.Close(); err != nil { - return err - } - - return nil -} - -func NewCompressor(src, dst string) (*Compressor, error) { - r, err := os.Open(src) - if err != nil { - return nil, err - } - - w, err := os.Create(dst) - if err != nil { - r.Close() - return nil, err - } - - c := &Compressor{r: r, w: w} - return c, nil -} - -func main() { - - runtime.GOMAXPROCS(runtime.NumCPU()) - - var resw testing.BenchmarkResult - var resr testing.BenchmarkResult - - c, err := NewCompressor("/tmp/image.r", "/tmp/image.w") - if err != nil { - panic(err) - } - resw = testing.Benchmark(c.BenchmarkGZIPWriter) - c.w.Seek(0, 0) - resr = testing.Benchmark(c.BenchmarkGZIPReader) - c.Close() - fmt.Printf("gzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) - - c, err = NewCompressor("/tmp/image.r", "/tmp/image.w") - if err != nil { - panic(err) - } - resw = testing.Benchmark(c.BenchmarkBGZFWriter) - c.w.Seek(0, 0) - resr = testing.Benchmark(c.BenchmarkBGZFReader) - c.Close() - fmt.Printf("bgzf:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) - - c, err = NewCompressor("/tmp/image.r", "/tmp/image.w") - if err != nil { - panic(err) - } - resw = testing.Benchmark(c.BenchmarkPGZIPWriter) - c.w.Seek(0, 0) - resr = testing.Benchmark(c.BenchmarkPGZIPReader) - c.Close() - fmt.Printf("pgzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) - - c, err = NewCompressor("/tmp/image.r", "/tmp/image.w") - if err != nil { - panic(err) - } - resw = testing.Benchmark(c.BenchmarkLZ4Writer) - c.w.Seek(0, 0) - resr = testing.Benchmark(c.BenchmarkLZ4Reader) - c.Close() - fmt.Printf("lz4:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) - -} - -func (c *Compressor) BenchmarkGZIPWriter(b *testing.B) { - cw, _ := gzip.NewWriterLevel(c.w, flate.BestSpeed) - b.ResetTimer() - - _, err := io.Copy(cw, c.r) - if err != nil { - b.Fatal(err) - } - cw.Close() - c.w.Sync() -} - -func (c *Compressor) BenchmarkGZIPReader(b *testing.B) { - cr, _ := gzip.NewReader(c.w) - b.ResetTimer() - - _, err := io.Copy(ioutil.Discard, cr) - if err != nil { - b.Fatal(err) - } -} - -func (c *Compressor) BenchmarkBGZFWriter(b *testing.B) { - cw, _ := bgzf.NewWriterLevel(c.w, flate.BestSpeed, runtime.NumCPU()) - b.ResetTimer() - - _, err := io.Copy(cw, c.r) - if err != nil { - b.Fatal(err) - } - c.w.Sync() -} - -func (c *Compressor) BenchmarkBGZFReader(b *testing.B) { - cr, _ := bgzf.NewReader(c.w, 0) - b.ResetTimer() - - _, err := io.Copy(ioutil.Discard, cr) - if err != nil { - b.Fatal(err) - } -} - -func (c *Compressor) BenchmarkPGZIPWriter(b *testing.B) { - cw, _ := pgzip.NewWriterLevel(c.w, flate.BestSpeed) - b.ResetTimer() - - _, err := io.Copy(cw, c.r) - if err != nil { - b.Fatal(err) - } - cw.Close() - c.w.Sync() -} - -func (c *Compressor) BenchmarkPGZIPReader(b *testing.B) { - cr, _ := pgzip.NewReader(c.w) - b.ResetTimer() - - _, err := io.Copy(ioutil.Discard, cr) - if err != nil { - b.Fatal(err) - } -} - -func (c *Compressor) BenchmarkLZ4Writer(b *testing.B) { - cw := lz4.NewWriter(c.w) - // cw.Header.HighCompression = true - cw.Header.NoChecksum = true - b.ResetTimer() - - _, err := io.Copy(cw, c.r) - if err != nil { - b.Fatal(err) - } - cw.Close() - c.w.Sync() -} - -func (c *Compressor) BenchmarkLZ4Reader(b *testing.B) { - cr := lz4.NewReader(c.w) - b.ResetTimer() - - _, err := io.Copy(ioutil.Discard, cr) - if err != nil { - b.Fatal(err) - } -} From ddbc145d29f8e312057cb1221e49ff70172ff77c Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 16 Jun 2015 16:31:09 -0700 Subject: [PATCH 16/28] Implemented acceptance test for compress --- .../compress/post-processor_test.go | 103 +++++++++++++++--- 1 file changed, 88 insertions(+), 15 deletions(-) diff --git a/post-processor/compress/post-processor_test.go b/post-processor/compress/post-processor_test.go index 5f4d6b9ca..12faeabed 100644 --- a/post-processor/compress/post-processor_test.go +++ b/post-processor/compress/post-processor_test.go @@ -1,22 +1,95 @@ package compress -// import ( -// "testing" -// -// builderT "github.com/mitchellh/packer/helper/builder/testing" -// ) -// -// func TestBuilderTagsAcc_basic(t *testing.T) { -// builderT.Test(t, builderT.TestCase{ -// Builder: &Builder{}, -// Template: simpleTestCase, -// Check: checkTags(), -// }) -// } +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/mitchellh/packer/builder/file" + env "github.com/mitchellh/packer/helper/builder/testing" + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/template" +) + +func setup(t *testing.T) (packer.Ui, packer.Artifact, error) { + // Create fake UI and Cache + ui := packer.TestUi(t) + cache := &packer.FileCache{CacheDir: os.TempDir()} + + // Create config for file builder + const fileConfig = `{"builders":[{"type":"file","target":"package.txt","content":"Hello world!"}]}` + tpl, err := template.Parse(strings.NewReader(fileConfig)) + if err != nil { + return nil, nil, fmt.Errorf("Unable to parse setup configuration: %s", err) + } + + // Prepare the file builder + builder := file.Builder{} + warnings, err := builder.Prepare(tpl.Builders["file"].Config) + if len(warnings) > 0 { + for _, warn := range warnings { + return nil, nil, fmt.Errorf("Configuration warning: %s", warn) + } + } + if err != nil { + return nil, nil, fmt.Errorf("Invalid configuration: %s", err) + } + + // Run the file builder + artifact, err := builder.Run(ui, nil, cache) + if err != nil { + return nil, nil, fmt.Errorf("Failed to build artifact: %s", err) + } + + return ui, artifact, err +} + +func TestSimpleCompress(t *testing.T) { + if os.Getenv(env.TestEnvVar) == "" { + t.Skip(fmt.Sprintf( + "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) + } + + ui, artifact, err := setup(t) + if err != nil { + t.Fatalf("Error bootstrapping test: %s", err) + } + if artifact != nil { + defer artifact.Destroy() + } + + tpl, err := template.Parse(strings.NewReader(simpleTestCase)) + if err != nil { + t.Fatalf("Unable to parse test config: %s", err) + } + + compressor := PostProcessor{} + compressor.Configure(tpl.PostProcessors[0][0].Config) + artifactOut, _, err := compressor.PostProcess(ui, artifact) + if err != nil { + t.Fatalf("Failed to compress artifact: %s", err) + } + // Cleanup after the test completes + defer artifactOut.Destroy() + + // Verify things look good + fi, err := os.Stat("package.tar.gz") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } + if fi.IsDir() { + t.Error("Archive should not be a directory") + } +} const simpleTestCase = ` { - "type": "compress", - "output": "foo.tar.gz" + "post-processors": [ + { + "type": "compress", + "output": "package.tar.gz" + } + ] } ` From 2d6f8279e6a807f7b65c14cb99b08e038aac18c8 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 16 Jun 2015 19:08:22 -0700 Subject: [PATCH 17/28] Restore configuration structure from vtolstov's branch --- post-processor/compress/post-processor.go | 34 +++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index c2f608685..b08465a66 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -2,23 +2,47 @@ package compress import ( "archive/tar" + "archive/zip" + "compress/flate" "compress/gzip" "fmt" "io" "os" + "path/filepath" + "runtime" + "strings" + "time" + "gopkg.in/yaml.v2" + + "github.com/biogo/hts/bgzf" + "github.com/klauspost/pgzip" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" + "github.com/pierrec/lz4" ) +type Metadata map[string]Metaitem + +type Metaitem struct { + CompSize int64 `yaml:"compsize"` + OrigSize int64 `yaml:"origsize"` + CompType string `yaml:"comptype"` + CompDate string `yaml:"compdate"` +} + type Config struct { common.PackerConfig `mapstructure:",squash"` - - OutputPath string `mapstructure:"output"` - - ctx interpolate.Context + OutputPath string `mapstructure:"output"` + OutputFile string `mapstructure:"file"` + Compression int `mapstructure:"compression"` + Metadata bool `mapstructure:"metadata"` + NumCPU int `mapstructure:"numcpu"` + Format string `mapstructure:"format"` + KeepInputArtifact bool `mapstructure:"keep_input_artifact"` + ctx *interpolate.Context } type PostProcessor struct { @@ -205,7 +229,7 @@ func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { return nil, fmt.Errorf("tar error on stat of %s: %s", name, err) } - target, _ := os.Readlink(path) + target, _ := os.Readlink(name) header, err := tar.FileInfoHeader(fi, target) if err != nil { return nil, fmt.Errorf("tar error reading info for %s: %s", name, err) From 47bb5ae89908716683f11fe402440656f6582b5d Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 16 Jun 2015 20:23:40 -0700 Subject: [PATCH 18/28] Re-added benchmark and license --- post-processor/compress/LICENSE | 21 +++ post-processor/compress/benchmark.go | 197 +++++++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 post-processor/compress/LICENSE create mode 100644 post-processor/compress/benchmark.go diff --git a/post-processor/compress/LICENSE b/post-processor/compress/LICENSE new file mode 100644 index 000000000..38bbf26f3 --- /dev/null +++ b/post-processor/compress/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Vasiliy Tolstov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/post-processor/compress/benchmark.go b/post-processor/compress/benchmark.go new file mode 100644 index 000000000..ed4d68168 --- /dev/null +++ b/post-processor/compress/benchmark.go @@ -0,0 +1,197 @@ +// +build ignore + +package main + +import ( + "compress/flate" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "runtime" + "testing" + + "github.com/biogo/hts/bgzf" + "github.com/klauspost/pgzip" + "github.com/pierrec/lz4" +) + +type Compressor struct { + r *os.File + w *os.File + sr int64 + sw int64 +} + +func (c *Compressor) Close() error { + var err error + + fi, _ := c.w.Stat() + c.sw = fi.Size() + if err = c.w.Close(); err != nil { + return err + } + + fi, _ = c.r.Stat() + c.sr = fi.Size() + if err = c.r.Close(); err != nil { + return err + } + + return nil +} + +func NewCompressor(src, dst string) (*Compressor, error) { + r, err := os.Open(src) + if err != nil { + return nil, err + } + + w, err := os.Create(dst) + if err != nil { + r.Close() + return nil, err + } + + c := &Compressor{r: r, w: w} + return c, nil +} + +func main() { + + runtime.GOMAXPROCS(runtime.NumCPU()) + + var resw testing.BenchmarkResult + var resr testing.BenchmarkResult + + c, err := NewCompressor("/tmp/image.r", "/tmp/image.w") + if err != nil { + panic(err) + } + resw = testing.Benchmark(c.BenchmarkGZIPWriter) + c.w.Seek(0, 0) + resr = testing.Benchmark(c.BenchmarkGZIPReader) + c.Close() + fmt.Printf("gzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) + + c, err = NewCompressor("/tmp/image.r", "/tmp/image.w") + if err != nil { + panic(err) + } + resw = testing.Benchmark(c.BenchmarkBGZFWriter) + c.w.Seek(0, 0) + resr = testing.Benchmark(c.BenchmarkBGZFReader) + c.Close() + fmt.Printf("bgzf:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) + + c, err = NewCompressor("/tmp/image.r", "/tmp/image.w") + if err != nil { + panic(err) + } + resw = testing.Benchmark(c.BenchmarkPGZIPWriter) + c.w.Seek(0, 0) + resr = testing.Benchmark(c.BenchmarkPGZIPReader) + c.Close() + fmt.Printf("pgzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) + + c, err = NewCompressor("/tmp/image.r", "/tmp/image.w") + if err != nil { + panic(err) + } + resw = testing.Benchmark(c.BenchmarkLZ4Writer) + c.w.Seek(0, 0) + resr = testing.Benchmark(c.BenchmarkLZ4Reader) + c.Close() + fmt.Printf("lz4:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw) + +} + +func (c *Compressor) BenchmarkGZIPWriter(b *testing.B) { + cw, _ := gzip.NewWriterLevel(c.w, flate.BestSpeed) + b.ResetTimer() + + _, err := io.Copy(cw, c.r) + if err != nil { + b.Fatal(err) + } + cw.Close() + c.w.Sync() +} + +func (c *Compressor) BenchmarkGZIPReader(b *testing.B) { + cr, _ := gzip.NewReader(c.w) + b.ResetTimer() + + _, err := io.Copy(ioutil.Discard, cr) + if err != nil { + b.Fatal(err) + } +} + +func (c *Compressor) BenchmarkBGZFWriter(b *testing.B) { + cw, _ := bgzf.NewWriterLevel(c.w, flate.BestSpeed, runtime.NumCPU()) + b.ResetTimer() + + _, err := io.Copy(cw, c.r) + if err != nil { + b.Fatal(err) + } + c.w.Sync() +} + +func (c *Compressor) BenchmarkBGZFReader(b *testing.B) { + cr, _ := bgzf.NewReader(c.w, 0) + b.ResetTimer() + + _, err := io.Copy(ioutil.Discard, cr) + if err != nil { + b.Fatal(err) + } +} + +func (c *Compressor) BenchmarkPGZIPWriter(b *testing.B) { + cw, _ := pgzip.NewWriterLevel(c.w, flate.BestSpeed) + b.ResetTimer() + + _, err := io.Copy(cw, c.r) + if err != nil { + b.Fatal(err) + } + cw.Close() + c.w.Sync() +} + +func (c *Compressor) BenchmarkPGZIPReader(b *testing.B) { + cr, _ := pgzip.NewReader(c.w) + b.ResetTimer() + + _, err := io.Copy(ioutil.Discard, cr) + if err != nil { + b.Fatal(err) + } +} + +func (c *Compressor) BenchmarkLZ4Writer(b *testing.B) { + cw := lz4.NewWriter(c.w) + // cw.Header.HighCompression = true + cw.Header.NoChecksum = true + b.ResetTimer() + + _, err := io.Copy(cw, c.r) + if err != nil { + b.Fatal(err) + } + cw.Close() + c.w.Sync() +} + +func (c *Compressor) BenchmarkLZ4Reader(b *testing.B) { + cr := lz4.NewReader(c.w) + b.ResetTimer() + + _, err := io.Copy(ioutil.Discard, cr) + if err != nil { + b.Fatal(err) + } +} From 8fdb4f77e09eba301713f70d132ca3b7c8c0b3c2 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 00:47:33 -0700 Subject: [PATCH 19/28] WIP 2/4 tests passing, still need to re-implement ZIP and bare compression files and do some cleanup --- post-processor/compress/post-processor.go | 322 ++++++++---------- .../compress/post-processor_test.go | 140 ++++++++ .../post-processors/compress.html.markdown | 39 ++- 3 files changed, 323 insertions(+), 178 deletions(-) diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index b08465a66..42cea2d35 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -3,19 +3,14 @@ package compress import ( "archive/tar" "archive/zip" - "compress/flate" "compress/gzip" "fmt" "io" "os" "path/filepath" + "regexp" "runtime" - "strings" - "time" - "gopkg.in/yaml.v2" - - "github.com/biogo/hts/bgzf" "github.com/klauspost/pgzip" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/helper/config" @@ -24,24 +19,13 @@ import ( "github.com/pierrec/lz4" ) -type Metadata map[string]Metaitem - -type Metaitem struct { - CompSize int64 `yaml:"compsize"` - OrigSize int64 `yaml:"origsize"` - CompType string `yaml:"comptype"` - CompDate string `yaml:"compdate"` -} - type Config struct { common.PackerConfig `mapstructure:",squash"` OutputPath string `mapstructure:"output"` - OutputFile string `mapstructure:"file"` - Compression int `mapstructure:"compression"` - Metadata bool `mapstructure:"metadata"` - NumCPU int `mapstructure:"numcpu"` - Format string `mapstructure:"format"` + Level int `mapstructure:"level"` KeepInputArtifact bool `mapstructure:"keep_input_artifact"` + Archive string + Algorithm string ctx *interpolate.Context } @@ -49,8 +33,52 @@ type PostProcessor struct { config Config } +// ErrInvalidCompressionLevel is returned when the compression level passed to +// gzip is not in the expected range. See compress/flate for details. +var ErrInvalidCompressionLevel = fmt.Errorf( + "Invalid compression level. Expected an integer from -1 to 9.") + +var ErrWrongInputCount = fmt.Errorf( + "Can only have 1 input file when not using tar/zip") + +func detectFromFilename(config *Config) error { + re := regexp.MustCompile("^.+?(?:\\.([a-z0-9]+))?\\.([a-z0-9]+)$") + + extensions := map[string]string{ + "tar": "tar", + "zip": "zip", + "gz": "pgzip", + "lz4": "lz4", + } + + result := re.FindAllString(config.OutputPath, -1) + + // Should we make an archive? E.g. tar or zip? + if result[0] == "tar" { + config.Archive = "tar" + } + if result[1] == "zip" || result[1] == "tar" { + config.Archive = result[1] + // Tar or zip is our final artifact. Bail out. + return nil + } + + // Should we compress the artifact? + algorithm, ok := extensions[result[1]] + if ok { + config.Algorithm = algorithm + // We found our compression algorithm something. Bail out. + return nil + } + + // We didn't find anything. Default to tar + pgzip + config.Algorithm = "pgzip" + config.Archive = "tar" + return fmt.Errorf("Unable to detect compression algorithm") +} + func (p *PostProcessor) Configure(raws ...interface{}) error { - p.config.Compression = -1 + p.config.Level = -1 err := config.Decode(&p.config, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ @@ -73,19 +101,13 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { "output": &p.config.OutputPath, } - if p.config.Compression > flate.BestCompression { - p.config.Compression = flate.BestCompression + if p.config.Level > gzip.BestCompression { + p.config.Level = gzip.BestCompression } - if p.config.Compression == -1 { - p.config.Compression = flate.DefaultCompression + if p.config.Level == -1 { + p.config.Level = gzip.DefaultCompression } - if p.config.NumCPU < 1 { - p.config.NumCPU = runtime.NumCPU() - } - - runtime.GOMAXPROCS(p.config.NumCPU) - for key, ptr := range templates { if *ptr == "" { errs = packer.MultiErrorAppend( @@ -107,123 +129,113 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } -func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata { - // layout shows by example how the reference time should be represented. - const layout = "2006-01-02_15-04-05" - t := time.Now() - - if !p.config.Metadata { - return metadata - } - for _, f := range files { - if fi, err := os.Stat(f); err != nil { - continue - } else { - if i, ok := metadata[filepath.Base(f)]; !ok { - metadata[filepath.Base(f)] = Metaitem{CompType: p.config.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)} - } else { - i.CompSize = fi.Size() - i.CompDate = t.Format(layout) - metadata[filepath.Base(f)] = i - } - } - } - return metadata -} - func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - newartifact := &Artifact{Path: p.config.OutputPath} - metafile := filepath.Join(p.config.OutputPath, "metadata") - ui.Say(fmt.Sprintf("[CBEDNARSKI] Creating archive at %s", newartifact.Path)) - _, err := os.Stat(newartifact.Path) - if err == nil { - return nil, false, fmt.Errorf("output dir %s must not exists", newartifact.Path) - } - err = os.MkdirAll(newartifact.Path, 0755) + newArtifact := &Artifact{Path: p.config.OutputPath} + + outputFile, err := os.Create(p.config.OutputPath) if err != nil { - return nil, false, fmt.Errorf("failed to create output: %s", err) + return nil, false, fmt.Errorf( + "Unable to create archive %s: %s", p.config.OutputPath, err) } + defer outputFile.Close() - p.config.Format += "tar.gzip" - formats := strings.Split(p.config.Format, ".") - ui.Say(fmt.Sprintf("[CBEDNARSKI] Formats length %d", len(formats))) - if len(p.config.Format) == 0 { - ui.Say("[CBEDNARSKI] Formats is empty") - formats[0] = "tar.gzip" - } - files := artifact.Files() - - metadata := make(Metadata, 0) - metadata = p.fillMetadata(metadata, files) - - ui.Say(fmt.Sprintf("[CBEDNARSKI] Formats %#v", formats)) - - for _, compress := range formats { - switch compress { - case "tar": - files, err = p.cmpTAR(files, filepath.Join(p.config.OutputPath, p.config.OutputFile)) - metadata = p.fillMetadata(metadata, files) - case "zip": - files, err = p.cmpZIP(files, filepath.Join(p.config.OutputPath, p.config.OutputFile)) - metadata = p.fillMetadata(metadata, files) - case "pgzip": - files, err = p.cmpPGZIP(files, p.config.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "gzip": - files, err = p.cmpGZIP(files, p.config.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "bgzf": - files, err = p.cmpBGZF(files, p.config.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "lz4": - files, err = p.cmpLZ4(files, p.config.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "e2fs": - files, err = p.cmpE2FS(files, filepath.Join(p.config.OutputPath, p.config.OutputFile)) - metadata = p.fillMetadata(metadata, files) + // Setup output interface. If we're using compression, output is a + // compression writer. Otherwise it's just a file. + var output io.WriteCloser + switch p.config.Algorithm { + case "lz4": + lzwriter := lz4.NewWriter(outputFile) + if p.config.Level > gzip.DefaultCompression { + lzwriter.Header.HighCompression = true } + defer lzwriter.Close() + output = lzwriter + case "pgzip": + output, err = pgzip.NewWriterLevel(outputFile, p.config.Level) if err != nil { - return nil, false, fmt.Errorf("Failed to compress: %s", err) + return nil, false, ErrInvalidCompressionLevel } + defer output.Close() + default: + output = outputFile } - if p.config.Metadata { - fp, err := os.Create(metafile) + //Archive + switch p.config.Archive { + case "tar": + archiveTar(artifact.Files(), output) + case "zip": + archive := zip.NewWriter(output) + defer archive.Close() + default: + // We have a regular file, so we'll just do an io.Copy + if len(artifact.Files()) != 1 { + return nil, false, fmt.Errorf( + "Can only have 1 input file when not using tar/zip. Found %d "+ + "files: %v", len(artifact.Files()), artifact.Files()) + } + source, err := os.Open(artifact.Files()[0]) if err != nil { - return nil, false, err - } - if buf, err := yaml.Marshal(metadata); err != nil { - fp.Close() - return nil, false, err - } else { - if _, err = fp.Write(buf); err != nil { - fp.Close() - return nil, false, err - } - fp.Close() + return nil, false, fmt.Errorf( + "Failed to open source file %s for reading: %s", + artifact.Files()[0], err) } + defer source.Close() + io.Copy(output, source) } - newartifact.files = append(newartifact.files, files...) - if p.config.Metadata { - newartifact.files = append(newartifact.files, metafile) - } - - return newartifact, p.config.KeepInputArtifact, nil + return newArtifact, p.config.KeepInputArtifact, nil } -func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { - fw, err := os.Create(dst) +func archiveTar(files []string, output io.WriteCloser) error { + archive := tar.NewWriter(output) + defer archive.Close() + + for _, path := range files { + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("Unable to read file %s: %s", path, err) + } + defer file.Close() + + fi, err := file.Stat() + if err != nil { + return fmt.Errorf("Unable to get fileinfo for %s: %s", path, err) + } + + target, err := os.Readlink(path) + if err != nil { + return fmt.Errorf("Failed to readlink for %s: %s", path, err) + } + + header, err := tar.FileInfoHeader(fi, target) + if err != nil { + return fmt.Errorf("Failed to create tar header for %s: %s", path, err) + } + + if err := archive.WriteHeader(header); err != nil { + return fmt.Errorf("Failed to write tar header for %s: %s", path, err) + } + + if _, err := io.Copy(archive, file); err != nil { + return fmt.Errorf("Failed to copy %s data to archive: %s", path, err) + } + } + return nil +} + +func (p *PostProcessor) cmpTAR(files []string, target string) ([]string, error) { + fw, err := os.Create(target) if err != nil { - return nil, fmt.Errorf("tar error creating tar %s: %s", dst, err) + return nil, fmt.Errorf("tar error creating tar %s: %s", target, err) } defer fw.Close() tw := tar.NewWriter(fw) defer tw.Close() - for _, name := range src { + for _, name := range files { fi, err := os.Stat(name) if err != nil { return nil, fmt.Errorf("tar error on stat of %s: %s", name, err) @@ -250,18 +262,18 @@ func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { } fr.Close() } - return []string{dst}, nil + return []string{target}, nil } -func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpGZIP(files []string, target string) ([]string, error) { var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) + for _, name := range files { + filename := filepath.Join(target, filepath.Base(name)) fw, err := os.Create(filename) if err != nil { - return nil, fmt.Errorf("gzip error: %s", err) + return nil, fmt.Errorf("gzip error creating archive: %s", err) } - cw, err := gzip.NewWriterLevel(fw, p.config.Compression) + cw, err := gzip.NewWriterLevel(fw, p.config.Level) if err != nil { fw.Close() return nil, fmt.Errorf("gzip error: %s", err) @@ -286,15 +298,16 @@ func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { return res, nil } -func (p *PostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpPGZIP(files []string, target string) ([]string, error) { var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) + for _, name := range files { + filename := filepath.Join(target, filepath.Base(name)) fw, err := os.Create(filename) if err != nil { return nil, fmt.Errorf("pgzip error: %s", err) } - cw, err := pgzip.NewWriterLevel(fw, p.config.Compression) + cw, err := pgzip.NewWriterLevel(fw, p.config.Level) + cw.SetConcurrency(500000, runtime.GOMAXPROCS(-1)) if err != nil { fw.Close() return nil, fmt.Errorf("pgzip error: %s", err) @@ -332,7 +345,7 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { fw.Close() return nil, fmt.Errorf("lz4 error: %s", err) } - if p.config.Compression > flate.DefaultCompression { + if p.config.Level > gzip.DefaultCompression { cw.Header.HighCompression = true } fr, err := os.Open(name) @@ -355,43 +368,6 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { return res, nil } -func (p *PostProcessor) cmpBGZF(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("bgzf error: %s", err) - } - - cw, err := bgzf.NewWriterLevel(fw, p.config.Compression, runtime.NumCPU()) - if err != nil { - return nil, fmt.Errorf("bgzf error: %s", err) - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("bgzf error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("bgzf error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpE2FS(src []string, dst string) ([]string, error) { - panic("not implemented") -} - func (p *PostProcessor) cmpZIP(src []string, dst string) ([]string, error) { fw, err := os.Create(dst) if err != nil { diff --git a/post-processor/compress/post-processor_test.go b/post-processor/compress/post-processor_test.go index 12faeabed..6d28a6698 100644 --- a/post-processor/compress/post-processor_test.go +++ b/post-processor/compress/post-processor_test.go @@ -83,6 +83,111 @@ func TestSimpleCompress(t *testing.T) { } } +func TestZipArchive(t *testing.T) { + if os.Getenv(env.TestEnvVar) == "" { + t.Skip(fmt.Sprintf( + "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) + } + + ui, artifact, err := setup(t) + if err != nil { + t.Fatalf("Error bootstrapping test: %s", err) + } + if artifact != nil { + defer artifact.Destroy() + } + + tpl, err := template.Parse(strings.NewReader(tarTestCase)) + if err != nil { + t.Fatalf("Unable to parse test config: %s", err) + } + + compressor := PostProcessor{} + compressor.Configure(tpl.PostProcessors[0][0].Config) + artifactOut, _, err := compressor.PostProcess(ui, artifact) + if err != nil { + t.Fatalf("Failed to archive artifact: %s", err) + } + // Cleanup after the test completes + defer artifactOut.Destroy() + + // Verify things look good + _, err = os.Stat("package.zip") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } +} + +func TestTarArchive(t *testing.T) { + if os.Getenv(env.TestEnvVar) == "" { + t.Skip(fmt.Sprintf( + "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) + } + + ui, artifact, err := setup(t) + if err != nil { + t.Fatalf("Error bootstrapping test: %s", err) + } + if artifact != nil { + defer artifact.Destroy() + } + + tpl, err := template.Parse(strings.NewReader(tarTestCase)) + if err != nil { + t.Fatalf("Unable to parse test config: %s", err) + } + + compressor := PostProcessor{} + compressor.Configure(tpl.PostProcessors[0][0].Config) + artifactOut, _, err := compressor.PostProcess(ui, artifact) + if err != nil { + t.Fatalf("Failed to archive artifact: %s", err) + } + // Cleanup after the test completes + defer artifactOut.Destroy() + + // Verify things look good + _, err = os.Stat("package.tar") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } +} + +func TestCompressOptions(t *testing.T) { + if os.Getenv(env.TestEnvVar) == "" { + t.Skip(fmt.Sprintf( + "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) + } + + ui, artifact, err := setup(t) + if err != nil { + t.Fatalf("Error bootstrapping test: %s", err) + } + if artifact != nil { + defer artifact.Destroy() + } + + tpl, err := template.Parse(strings.NewReader(zipTestCase)) + if err != nil { + t.Fatalf("Unable to parse test config: %s", err) + } + + compressor := PostProcessor{} + compressor.Configure(tpl.PostProcessors[0][0].Config) + artifactOut, _, err := compressor.PostProcess(ui, artifact) + if err != nil { + t.Fatalf("Failed to archive artifact: %s", err) + } + // Cleanup after the test completes + defer artifactOut.Destroy() + + // Verify things look good + _, err = os.Stat("package.gz") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } +} + const simpleTestCase = ` { "post-processors": [ @@ -93,3 +198,38 @@ const simpleTestCase = ` ] } ` + +const zipTestCase = ` +{ + "post-processors": [ + { + "type": "compress", + "output": "package.zip" + } + ] +} +` + +const tarTestCase = ` +{ + "post-processors": [ + { + "type": "compress", + "output": "package.tar" + } + ] +} +` + +const optionsTestCase = ` +{ + "post-processors": [ + { + "type": "compress", + "output": "package.gz", + "level": 9, + "parallel": false + } + ] +} +` diff --git a/website/source/docs/post-processors/compress.html.markdown b/website/source/docs/post-processors/compress.html.markdown index ea3b9c7ac..6f1430e2e 100644 --- a/website/source/docs/post-processors/compress.html.markdown +++ b/website/source/docs/post-processors/compress.html.markdown @@ -15,17 +15,46 @@ archive. ## Configuration -The configuration for this post-processor is extremely simple. +The minimal required configuration is to specify the output file. This will create a gzipped tarball. -* `output` (string) - The path to save the compressed archive. +* `output` (required, string) - The path to save the compressed archive. The archive format is inferred from the filename. E.g. `.tar.gz` will be a gzipped tarball. `.zip` will be a zip file. + + If the extension can't be detected tar+gzip will be used as a fallback. + +If you want more control over how the archive is created you can specify the following settings: + +* `level` (optional, integer) - Specify the compression level, for algorithms that support it. Value from -1 through 9 inclusive. 9 offers the smallest file size, but takes longer +* `keep_input_artifact` (optional, bool) - Keep source files; defaults to false + +## Supported Formats + +Supported file extensions include `.zip`, `.tar`, `.gz`, `.tar.gz`, `.lz4` and `.tar.lz4`. ## Example -An example is shown below, showing only the post-processor configuration: +Some minimal examples are shown below, showing only the post-processor configuration: -```javascript +```json { "type": "compress", - "output": "foo.tar.gz" + "output": "archive.tar.gz" +} +``` + +```json +{ + "type": "compress", + "output": "archive.zip" +} +``` + +A more complex example, again showing only the post-processor configuration: + +```json +{ + "type": "compress", + "output": "archive.gz", + "compression": 9, + "parallel": false } ``` From b767aa7f9945b8b7e311d1d367b4200fba2be5c2 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 03:55:51 -0700 Subject: [PATCH 20/28] Change to compression_level, fix and add tests for format detection --- post-processor/compress/post-processor.go | 155 +++++++++--------- .../compress/post-processor_test.go | 52 +++++- .../post-processors/compress.html.markdown | 20 +-- 3 files changed, 137 insertions(+), 90 deletions(-) diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index 42cea2d35..5bceae8c8 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -22,27 +22,31 @@ import ( type Config struct { common.PackerConfig `mapstructure:",squash"` OutputPath string `mapstructure:"output"` - Level int `mapstructure:"level"` + CompressionLevel int `mapstructure:"compression_level"` KeepInputArtifact bool `mapstructure:"keep_input_artifact"` Archive string Algorithm string + UsingDefault bool ctx *interpolate.Context } type PostProcessor struct { - config Config + config *Config } -// ErrInvalidCompressionLevel is returned when the compression level passed to -// gzip is not in the expected range. See compress/flate for details. -var ErrInvalidCompressionLevel = fmt.Errorf( - "Invalid compression level. Expected an integer from -1 to 9.") +var ( + // ErrInvalidCompressionLevel is returned when the compression level passed + // to gzip is not in the expected range. See compress/flate for details. + ErrInvalidCompressionLevel = fmt.Errorf( + "Invalid compression level. Expected an integer from -1 to 9.") -var ErrWrongInputCount = fmt.Errorf( - "Can only have 1 input file when not using tar/zip") + ErrWrongInputCount = fmt.Errorf( + "Can only have 1 input file when not using tar/zip") -func detectFromFilename(config *Config) error { - re := regexp.MustCompile("^.+?(?:\\.([a-z0-9]+))?\\.([a-z0-9]+)$") + filenamePattern = regexp.MustCompile(`(?:\.([a-z0-9]+))`) +) + +func (config *Config) detectFromFilename() { extensions := map[string]string{ "tar": "tar", @@ -51,34 +55,47 @@ func detectFromFilename(config *Config) error { "lz4": "lz4", } - result := re.FindAllString(config.OutputPath, -1) + result := filenamePattern.FindAllStringSubmatch(config.OutputPath, -1) + + if len(result) == 0 { + config.Algorithm = "pgzip" + config.Archive = "tar" + return + } // Should we make an archive? E.g. tar or zip? - if result[0] == "tar" { + var nextToLastItem string + if len(result) == 1 { + nextToLastItem = "" + } else { + nextToLastItem = result[len(result)-2][1] + } + + lastItem := result[len(result)-1][1] + if nextToLastItem == "tar" { config.Archive = "tar" } - if result[1] == "zip" || result[1] == "tar" { - config.Archive = result[1] + if lastItem == "zip" || lastItem == "tar" { + config.Archive = lastItem // Tar or zip is our final artifact. Bail out. - return nil + return } // Should we compress the artifact? - algorithm, ok := extensions[result[1]] + algorithm, ok := extensions[lastItem] if ok { config.Algorithm = algorithm - // We found our compression algorithm something. Bail out. - return nil + // We found our compression algorithm. Bail out. + return } // We didn't find anything. Default to tar + pgzip config.Algorithm = "pgzip" config.Archive = "tar" - return fmt.Errorf("Unable to detect compression algorithm") + return } func (p *PostProcessor) Configure(raws ...interface{}) error { - p.config.Level = -1 err := config.Decode(&p.config, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ @@ -86,6 +103,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { }, }, raws...) + fmt.Printf("CompressionLevel: %d\n", p.config.CompressionLevel) + errs := new(packer.MultiError) if p.config.OutputPath == "" { @@ -101,13 +120,17 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { "output": &p.config.OutputPath, } - if p.config.Level > gzip.BestCompression { - p.config.Level = gzip.BestCompression + if p.config.CompressionLevel > pgzip.BestCompression { + p.config.CompressionLevel = pgzip.BestCompression } - if p.config.Level == -1 { - p.config.Level = gzip.DefaultCompression + // Technically 0 means "don't compress" but I don't know how to + // differentiate between "user entered zero" and "user entered nothing". + // Also, why bother creating a compressed file with zero compression? + if p.config.CompressionLevel == -1 || p.config.CompressionLevel == 0 { + p.config.CompressionLevel = pgzip.DefaultCompression } + fmt.Printf("CompressionLevel: %d\n", p.config.CompressionLevel) for key, ptr := range templates { if *ptr == "" { errs = packer.MultiErrorAppend( @@ -121,6 +144,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } } + p.config.detectFromFilename() + if len(errs.Errors) > 0 { return errs } @@ -131,12 +156,13 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - newArtifact := &Artifact{Path: p.config.OutputPath} + target := p.config.OutputPath + newArtifact := &Artifact{Path: target} - outputFile, err := os.Create(p.config.OutputPath) + outputFile, err := os.Create(target) if err != nil { return nil, false, fmt.Errorf( - "Unable to create archive %s: %s", p.config.OutputPath, err) + "Unable to create archive %s: %s", target, err) } defer outputFile.Close() @@ -145,31 +171,44 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac var output io.WriteCloser switch p.config.Algorithm { case "lz4": + ui.Say(fmt.Sprintf("Preparing lz4 compression for %s", target)) lzwriter := lz4.NewWriter(outputFile) - if p.config.Level > gzip.DefaultCompression { + if p.config.CompressionLevel > gzip.DefaultCompression { lzwriter.Header.HighCompression = true } defer lzwriter.Close() output = lzwriter case "pgzip": - output, err = pgzip.NewWriterLevel(outputFile, p.config.Level) + ui.Say(fmt.Sprintf("Preparing gzip compression for %s", target)) + gzipWriter, err := pgzip.NewWriterLevel(outputFile, p.config.CompressionLevel) if err != nil { return nil, false, ErrInvalidCompressionLevel } + gzipWriter.SetConcurrency(500000, runtime.GOMAXPROCS(-1)) + output = gzipWriter defer output.Close() default: output = outputFile } - //Archive + compression := p.config.Algorithm + if compression == "" { + compression = "no" + } + + // Build an archive, if we're supposed to do that. switch p.config.Archive { case "tar": - archiveTar(artifact.Files(), output) + ui.Say(fmt.Sprintf("Taring %s with %s compression", target, compression)) + createTarArchive(artifact.Files(), output) case "zip": + ui.Say(fmt.Sprintf("Zipping %s", target)) archive := zip.NewWriter(output) defer archive.Close() default: - // We have a regular file, so we'll just do an io.Copy + ui.Say(fmt.Sprintf("Copying %s with %s compression", target, compression)) + // Filename indicates no tarball (just compress) so we'll do an io.Copy + // into our compressor. if len(artifact.Files()) != 1 { return nil, false, fmt.Errorf( "Can only have 1 input file when not using tar/zip. Found %d "+ @@ -185,10 +224,12 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac io.Copy(output, source) } + ui.Say(fmt.Sprintf("Archive %s completed", target)) + return newArtifact, p.config.KeepInputArtifact, nil } -func archiveTar(files []string, output io.WriteCloser) error { +func createTarArchive(files []string, output io.WriteCloser) error { archive := tar.NewWriter(output) defer archive.Close() @@ -225,44 +266,8 @@ func archiveTar(files []string, output io.WriteCloser) error { return nil } -func (p *PostProcessor) cmpTAR(files []string, target string) ([]string, error) { - fw, err := os.Create(target) - if err != nil { - return nil, fmt.Errorf("tar error creating tar %s: %s", target, err) - } - defer fw.Close() - - tw := tar.NewWriter(fw) - defer tw.Close() - - for _, name := range files { - fi, err := os.Stat(name) - if err != nil { - return nil, fmt.Errorf("tar error on stat of %s: %s", name, err) - } - - target, _ := os.Readlink(name) - header, err := tar.FileInfoHeader(fi, target) - if err != nil { - return nil, fmt.Errorf("tar error reading info for %s: %s", name, err) - } - - if err = tw.WriteHeader(header); err != nil { - return nil, fmt.Errorf("tar error writing header for %s: %s", name, err) - } - - fr, err := os.Open(name) - if err != nil { - return nil, fmt.Errorf("tar error opening file %s: %s", name, err) - } - - if _, err = io.Copy(tw, fr); err != nil { - fr.Close() - return nil, fmt.Errorf("tar error copying contents of %s: %s", name, err) - } - fr.Close() - } - return []string{target}, nil +func createZipArchive(files []string, output io.WriteCloser) error { + return fmt.Errorf("Not implemented") } func (p *PostProcessor) cmpGZIP(files []string, target string) ([]string, error) { @@ -273,7 +278,7 @@ func (p *PostProcessor) cmpGZIP(files []string, target string) ([]string, error) if err != nil { return nil, fmt.Errorf("gzip error creating archive: %s", err) } - cw, err := gzip.NewWriterLevel(fw, p.config.Level) + cw, err := gzip.NewWriterLevel(fw, p.config.CompressionLevel) if err != nil { fw.Close() return nil, fmt.Errorf("gzip error: %s", err) @@ -306,8 +311,8 @@ func (p *PostProcessor) cmpPGZIP(files []string, target string) ([]string, error if err != nil { return nil, fmt.Errorf("pgzip error: %s", err) } - cw, err := pgzip.NewWriterLevel(fw, p.config.Level) - cw.SetConcurrency(500000, runtime.GOMAXPROCS(-1)) + cw, err := pgzip.NewWriterLevel(fw, p.config.CompressionLevel) + if err != nil { fw.Close() return nil, fmt.Errorf("pgzip error: %s", err) @@ -345,7 +350,7 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { fw.Close() return nil, fmt.Errorf("lz4 error: %s", err) } - if p.config.Level > gzip.DefaultCompression { + if p.config.CompressionLevel > gzip.DefaultCompression { cw.Header.HighCompression = true } fr, err := os.Open(name) diff --git a/post-processor/compress/post-processor_test.go b/post-processor/compress/post-processor_test.go index 6d28a6698..f60a7846a 100644 --- a/post-processor/compress/post-processor_test.go +++ b/post-processor/compress/post-processor_test.go @@ -45,6 +45,48 @@ func setup(t *testing.T) (packer.Ui, packer.Artifact, error) { return ui, artifact, err } +func TestDetectFilename(t *testing.T) { + // Test default / fallback with no file extension + nakedFilename := Config{OutputPath: "test"} + nakedFilename.detectFromFilename() + if nakedFilename.Archive != "tar" { + t.Error("Expected to find tar archive setting") + } + if nakedFilename.Algorithm != "pgzip" { + t.Error("Expected to find pgzip algorithm setting") + } + + // Test .archive + zipFilename := Config{OutputPath: "test.zip"} + zipFilename.detectFromFilename() + if zipFilename.Archive != "zip" { + t.Error("Expected to find zip archive setting") + } + if zipFilename.Algorithm != "" { + t.Error("Expected to find empty algorithm setting") + } + + // Test .compress + lz4Filename := Config{OutputPath: "test.lz4"} + lz4Filename.detectFromFilename() + if lz4Filename.Archive != "" { + t.Error("Expected to find empty archive setting") + } + if lz4Filename.Algorithm != "lz4" { + t.Error("Expected to find lz4 algorithm setting") + } + + // Test .archive.compress with some.extra.dots... + lotsOfDots := Config{OutputPath: "test.blah.bloo.blee.tar.lz4"} + lotsOfDots.detectFromFilename() + if lotsOfDots.Archive != "tar" { + t.Error("Expected to find tar archive setting") + } + if lotsOfDots.Algorithm != "lz4" { + t.Error("Expected to find lz4 algorithm setting") + } +} + func TestSimpleCompress(t *testing.T) { if os.Getenv(env.TestEnvVar) == "" { t.Skip(fmt.Sprintf( @@ -167,13 +209,18 @@ func TestCompressOptions(t *testing.T) { defer artifact.Destroy() } - tpl, err := template.Parse(strings.NewReader(zipTestCase)) + tpl, err := template.Parse(strings.NewReader(optionsTestCase)) if err != nil { t.Fatalf("Unable to parse test config: %s", err) } compressor := PostProcessor{} compressor.Configure(tpl.PostProcessors[0][0].Config) + + if compressor.config.CompressionLevel != 9 { + t.Errorf("Expected compression_level 9, got %d", compressor.config.CompressionLevel) + } + artifactOut, _, err := compressor.PostProcess(ui, artifact) if err != nil { t.Fatalf("Failed to archive artifact: %s", err) @@ -227,8 +274,7 @@ const optionsTestCase = ` { "type": "compress", "output": "package.gz", - "level": 9, - "parallel": false + "compression_level": 9 } ] } diff --git a/website/source/docs/post-processors/compress.html.markdown b/website/source/docs/post-processors/compress.html.markdown index 6f1430e2e..c5a05a937 100644 --- a/website/source/docs/post-processors/compress.html.markdown +++ b/website/source/docs/post-processors/compress.html.markdown @@ -10,25 +10,24 @@ description: |- Type: `compress` The Packer compress post-processor takes an artifact with files (such as from -VMware or VirtualBox) and gzip compresses the artifact into a single -archive. +VMware or VirtualBox) and compresses the artifact into a single archive. ## Configuration -The minimal required configuration is to specify the output file. This will create a gzipped tarball. +You must specify the output filename. The archive format is derived from the filename. -* `output` (required, string) - The path to save the compressed archive. The archive format is inferred from the filename. E.g. `.tar.gz` will be a gzipped tarball. `.zip` will be a zip file. +* `output` (required, string) - The path to save the compressed archive. The archive format is inferred from the filename. E.g. `.tar.gz` will be a gzipped tarball. `.zip` will be a zip file. If the extension can't be detected packer defaults to `.tar.gz` behavior but will not change the filename. - If the extension can't be detected tar+gzip will be used as a fallback. + If you are executing multiple builders in parallel you should make sure `output` is unique for each one. For example `packer_{{.BuildName}}_{{.Provider}}.zip`. If you want more control over how the archive is created you can specify the following settings: -* `level` (optional, integer) - Specify the compression level, for algorithms that support it. Value from -1 through 9 inclusive. 9 offers the smallest file size, but takes longer +* `compression_level` (optional, integer) - Specify the compression level, for algorithms that support it, from 1 through 9 inclusive. Typically higher compression levels take longer but produce smaller files. Default if omitted is 6 * `keep_input_artifact` (optional, bool) - Keep source files; defaults to false ## Supported Formats -Supported file extensions include `.zip`, `.tar`, `.gz`, `.tar.gz`, `.lz4` and `.tar.lz4`. +Supported file extensions include `.zip`, `.tar`, `.gz`, `.tar.gz`, `.lz4` and `.tar.lz4`. Note that `.gz` and `.lz4` will fail if you have multiple files to compress. ## Example @@ -37,7 +36,7 @@ Some minimal examples are shown below, showing only the post-processor configura ```json { "type": "compress", - "output": "archive.tar.gz" + "output": "archive.tar.lz4" } ``` @@ -48,13 +47,10 @@ Some minimal examples are shown below, showing only the post-processor configura } ``` -A more complex example, again showing only the post-processor configuration: - ```json { "type": "compress", "output": "archive.gz", - "compression": 9, - "parallel": false + "compression": 9 } ``` From d8f78d9174bad365d4cd8aead5de23e748b9274a Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 04:41:05 -0700 Subject: [PATCH 21/28] Cleanup --- post-processor/compress/post-processor.go | 308 ++++++++-------------- 1 file changed, 106 insertions(+), 202 deletions(-) diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index 5bceae8c8..0ecc7db86 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -46,55 +46,6 @@ var ( filenamePattern = regexp.MustCompile(`(?:\.([a-z0-9]+))`) ) -func (config *Config) detectFromFilename() { - - extensions := map[string]string{ - "tar": "tar", - "zip": "zip", - "gz": "pgzip", - "lz4": "lz4", - } - - result := filenamePattern.FindAllStringSubmatch(config.OutputPath, -1) - - if len(result) == 0 { - config.Algorithm = "pgzip" - config.Archive = "tar" - return - } - - // Should we make an archive? E.g. tar or zip? - var nextToLastItem string - if len(result) == 1 { - nextToLastItem = "" - } else { - nextToLastItem = result[len(result)-2][1] - } - - lastItem := result[len(result)-1][1] - if nextToLastItem == "tar" { - config.Archive = "tar" - } - if lastItem == "zip" || lastItem == "tar" { - config.Archive = lastItem - // Tar or zip is our final artifact. Bail out. - return - } - - // Should we compress the artifact? - algorithm, ok := extensions[lastItem] - if ok { - config.Algorithm = algorithm - // We found our compression algorithm. Bail out. - return - } - - // We didn't find anything. Default to tar + pgzip - config.Algorithm = "pgzip" - config.Archive = "tar" - return -} - func (p *PostProcessor) Configure(raws ...interface{}) error { err := config.Decode(&p.config, &config.DecodeOpts{ Interpolate: true, @@ -157,6 +108,7 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { target := p.config.OutputPath + keep := p.config.KeepInputArtifact newArtifact := &Artifact{Path: target} outputFile, err := os.Create(target) @@ -172,20 +124,11 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac switch p.config.Algorithm { case "lz4": ui.Say(fmt.Sprintf("Preparing lz4 compression for %s", target)) - lzwriter := lz4.NewWriter(outputFile) - if p.config.CompressionLevel > gzip.DefaultCompression { - lzwriter.Header.HighCompression = true - } - defer lzwriter.Close() - output = lzwriter + output, err = makeLZ4Writer(outputFile, p.config.CompressionLevel) + defer output.Close() case "pgzip": ui.Say(fmt.Sprintf("Preparing gzip compression for %s", target)) - gzipWriter, err := pgzip.NewWriterLevel(outputFile, p.config.CompressionLevel) - if err != nil { - return nil, false, ErrInvalidCompressionLevel - } - gzipWriter.SetConcurrency(500000, runtime.GOMAXPROCS(-1)) - output = gzipWriter + output, err = makePgzipWriter(outputFile, p.config.CompressionLevel) defer output.Close() default: output = outputFile @@ -199,34 +142,112 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac // Build an archive, if we're supposed to do that. switch p.config.Archive { case "tar": - ui.Say(fmt.Sprintf("Taring %s with %s compression", target, compression)) - createTarArchive(artifact.Files(), output) + ui.Say(fmt.Sprintf("Tarring %s with %s compression", target, compression)) + err = createTarArchive(artifact.Files(), output) + if err != nil { + return nil, keep, fmt.Errorf("Error creating tar: %s", err) + } case "zip": ui.Say(fmt.Sprintf("Zipping %s", target)) - archive := zip.NewWriter(output) - defer archive.Close() + err = createZipArchive(artifact.Files(), output) + if err != nil { + return nil, keep, fmt.Errorf("Error creating zip: %s", err) + } default: ui.Say(fmt.Sprintf("Copying %s with %s compression", target, compression)) // Filename indicates no tarball (just compress) so we'll do an io.Copy // into our compressor. if len(artifact.Files()) != 1 { - return nil, false, fmt.Errorf( + return nil, keep, fmt.Errorf( "Can only have 1 input file when not using tar/zip. Found %d "+ "files: %v", len(artifact.Files()), artifact.Files()) } + source, err := os.Open(artifact.Files()[0]) if err != nil { - return nil, false, fmt.Errorf( + return nil, keep, fmt.Errorf( "Failed to open source file %s for reading: %s", artifact.Files()[0], err) } defer source.Close() - io.Copy(output, source) + + if _, err = io.Copy(output, source); err != nil { + return nil, keep, fmt.Errorf("Failed to compress %s: %s", + artifact.Files()[0], err) + } } ui.Say(fmt.Sprintf("Archive %s completed", target)) - return newArtifact, p.config.KeepInputArtifact, nil + return newArtifact, keep, nil +} + +func (config *Config) detectFromFilename() { + + extensions := map[string]string{ + "tar": "tar", + "zip": "zip", + "gz": "pgzip", + "lz4": "lz4", + } + + result := filenamePattern.FindAllStringSubmatch(config.OutputPath, -1) + + // No dots. Bail out with defaults. + if len(result) == 0 { + config.Algorithm = "pgzip" + config.Archive = "tar" + return + } + + // Parse the last two .groups, if they're there + lastItem := result[len(result)-1][1] + var nextToLastItem string + if len(result) == 1 { + nextToLastItem = "" + } else { + nextToLastItem = result[len(result)-2][1] + } + + // Should we make an archive? E.g. tar or zip? + if nextToLastItem == "tar" { + config.Archive = "tar" + } + if lastItem == "zip" || lastItem == "tar" { + config.Archive = lastItem + // Tar or zip is our final artifact. Bail out. + return + } + + // Should we compress the artifact? + algorithm, ok := extensions[lastItem] + if ok { + config.Algorithm = algorithm + // We found our compression algorithm. Bail out. + return + } + + // We didn't match a known compression format. Default to tar + pgzip + config.Algorithm = "pgzip" + config.Archive = "tar" + return +} + +func makeLZ4Writer(output io.WriteCloser, compressionLevel int) (io.WriteCloser, error) { + lzwriter := lz4.NewWriter(output) + if compressionLevel > gzip.DefaultCompression { + lzwriter.Header.HighCompression = true + } + return lzwriter, nil +} + +func makePgzipWriter(output io.WriteCloser, compressionLevel int) (io.WriteCloser, error) { + gzipWriter, err := pgzip.NewWriterLevel(output, compressionLevel) + if err != nil { + return nil, ErrInvalidCompressionLevel + } + gzipWriter.SetConcurrency(500000, runtime.GOMAXPROCS(-1)) + return gzipWriter, nil } func createTarArchive(files []string, output io.WriteCloser) error { @@ -245,12 +266,7 @@ func createTarArchive(files []string, output io.WriteCloser) error { return fmt.Errorf("Unable to get fileinfo for %s: %s", path, err) } - target, err := os.Readlink(path) - if err != nil { - return fmt.Errorf("Failed to readlink for %s: %s", path, err) - } - - header, err := tar.FileInfoHeader(fi, target) + header, err := tar.FileInfoHeader(fi, path) if err != nil { return fmt.Errorf("Failed to create tar header for %s: %s", path, err) } @@ -267,139 +283,27 @@ func createTarArchive(files []string, output io.WriteCloser) error { } func createZipArchive(files []string, output io.WriteCloser) error { - return fmt.Errorf("Not implemented") -} + archive := zip.NewWriter(output) + defer archive.Close() -func (p *PostProcessor) cmpGZIP(files []string, target string) ([]string, error) { - var res []string - for _, name := range files { - filename := filepath.Join(target, filepath.Base(name)) - fw, err := os.Create(filename) + for _, path := range files { + path = filepath.ToSlash(path) + + source, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("gzip error creating archive: %s", err) + return fmt.Errorf("Unable to read file %s: %s", path, err) } - cw, err := gzip.NewWriterLevel(fw, p.config.CompressionLevel) + defer source.Close() + + target, err := archive.Create(path) if err != nil { - fw.Close() - return nil, fmt.Errorf("gzip error: %s", err) + return fmt.Errorf("Failed to add zip header for %s: %s", path, err) } - fr, err := os.Open(name) + + _, err = io.Copy(target, source) if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("gzip error: %s", err) + return fmt.Errorf("Failed to copy %s data to archive: %s", path, err) } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("gzip error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) } - return res, nil -} - -func (p *PostProcessor) cmpPGZIP(files []string, target string) ([]string, error) { - var res []string - for _, name := range files { - filename := filepath.Join(target, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("pgzip error: %s", err) - } - cw, err := pgzip.NewWriterLevel(fw, p.config.CompressionLevel) - - if err != nil { - fw.Close() - return nil, fmt.Errorf("pgzip error: %s", err) - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("pgzip error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("pgzip error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("lz4 error: %s", err) - } - cw := lz4.NewWriter(fw) - if err != nil { - fw.Close() - return nil, fmt.Errorf("lz4 error: %s", err) - } - if p.config.CompressionLevel > gzip.DefaultCompression { - cw.Header.HighCompression = true - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("lz4 error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("lz4 error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpZIP(src []string, dst string) ([]string, error) { - fw, err := os.Create(dst) - if err != nil { - return nil, fmt.Errorf("zip error: %s", err) - } - defer fw.Close() - - zw := zip.NewWriter(fw) - defer zw.Close() - - for _, name := range src { - header, err := zw.Create(name) - if err != nil { - return nil, fmt.Errorf("zip error: %s", err) - } - - fr, err := os.Open(name) - if err != nil { - return nil, fmt.Errorf("zip error: %s", err) - } - - if _, err = io.Copy(header, fr); err != nil { - fr.Close() - return nil, fmt.Errorf("zip error: %s", err) - } - fr.Close() - } - return []string{dst}, nil - + return nil } From e81378ac393af8e08316d6d6446357eed37248b5 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 04:46:39 -0700 Subject: [PATCH 22/28] Fix test case --- post-processor/compress/post-processor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/post-processor/compress/post-processor_test.go b/post-processor/compress/post-processor_test.go index f60a7846a..f6e82b1c9 100644 --- a/post-processor/compress/post-processor_test.go +++ b/post-processor/compress/post-processor_test.go @@ -139,7 +139,7 @@ func TestZipArchive(t *testing.T) { defer artifact.Destroy() } - tpl, err := template.Parse(strings.NewReader(tarTestCase)) + tpl, err := template.Parse(strings.NewReader(zipTestCase)) if err != nil { t.Fatalf("Unable to parse test config: %s", err) } From 9cd572461d1ed7ca8f2bc3d80b10f18b221b1965 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 05:13:48 -0700 Subject: [PATCH 23/28] Updated docs, fix artifact bug --- post-processor/compress/artifact.go | 2 +- .../post-processors/compress.html.markdown | 25 +++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/post-processor/compress/artifact.go b/post-processor/compress/artifact.go index cfc914a55..56a5ce402 100644 --- a/post-processor/compress/artifact.go +++ b/post-processor/compress/artifact.go @@ -29,7 +29,7 @@ func (*Artifact) Id() string { } func (a *Artifact) Files() []string { - return a.files + return []string{a.Path} } func (a *Artifact) String() string { diff --git a/website/source/docs/post-processors/compress.html.markdown b/website/source/docs/post-processors/compress.html.markdown index c5a05a937..8fcd81ee3 100644 --- a/website/source/docs/post-processors/compress.html.markdown +++ b/website/source/docs/post-processors/compress.html.markdown @@ -2,7 +2,7 @@ layout: "docs" page_title: "compress Post-Processor" description: |- - The Packer compress post-processor takes an artifact with files (such as from VMware or VirtualBox) and gzip compresses the artifact into a single archive. + The Packer compress post-processor takes an artifact with files (such as from VMware or VirtualBox) and compresses the artifact into a single archive. --- # Compress Post-Processor @@ -14,22 +14,33 @@ VMware or VirtualBox) and compresses the artifact into a single archive. ## Configuration +### Required: + You must specify the output filename. The archive format is derived from the filename. -* `output` (required, string) - The path to save the compressed archive. The archive format is inferred from the filename. E.g. `.tar.gz` will be a gzipped tarball. `.zip` will be a zip file. If the extension can't be detected packer defaults to `.tar.gz` behavior but will not change the filename. +* `output` (string) - The path to save the compressed archive. The archive + format is inferred from the filename. E.g. `.tar.gz` will be a gzipped + tarball. `.zip` will be a zip file. If the extension can't be detected packer + defaults to `.tar.gz` behavior but will not change the filename. - If you are executing multiple builders in parallel you should make sure `output` is unique for each one. For example `packer_{{.BuildName}}_{{.Provider}}.zip`. + If you are executing multiple builders in parallel you should make sure + `output` is unique for each one. For example `packer_{{.BuildName}}_{{.Provider}}.zip`. + +### Optional: If you want more control over how the archive is created you can specify the following settings: -* `compression_level` (optional, integer) - Specify the compression level, for algorithms that support it, from 1 through 9 inclusive. Typically higher compression levels take longer but produce smaller files. Default if omitted is 6 -* `keep_input_artifact` (optional, bool) - Keep source files; defaults to false +* `compression_level` (integer) - Specify the compression level, for algorithms + that support it, from 1 through 9 inclusive. Typically higher compression + levels take longer but produce smaller files. Defaults to `6` -## Supported Formats +* `keep_input_artifact` (bool) - Keep source files; defaults to `false` + +### Supported Formats Supported file extensions include `.zip`, `.tar`, `.gz`, `.tar.gz`, `.lz4` and `.tar.lz4`. Note that `.gz` and `.lz4` will fail if you have multiple files to compress. -## Example +## Examples Some minimal examples are shown below, showing only the post-processor configuration: From 0880d448f06fca9338d26b35a1c183bf1fe13d76 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 14:01:00 -0700 Subject: [PATCH 24/28] Cleanup some debug code and reorganize config struct --- post-processor/compress/post-processor.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index 0ecc7db86..769f21fb2 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -21,13 +21,17 @@ import ( type Config struct { common.PackerConfig `mapstructure:",squash"` - OutputPath string `mapstructure:"output"` - CompressionLevel int `mapstructure:"compression_level"` - KeepInputArtifact bool `mapstructure:"keep_input_artifact"` - Archive string - Algorithm string - UsingDefault bool - ctx *interpolate.Context + + // Fields from config file + OutputPath string `mapstructure:"output"` + CompressionLevel int `mapstructure:"compression_level"` + KeepInputArtifact bool `mapstructure:"keep_input_artifact"` + + // Derived fields + Archive string + Algorithm string + + ctx *interpolate.Context } type PostProcessor struct { @@ -54,8 +58,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { }, }, raws...) - fmt.Printf("CompressionLevel: %d\n", p.config.CompressionLevel) - errs := new(packer.MultiError) if p.config.OutputPath == "" { @@ -81,7 +83,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { p.config.CompressionLevel = pgzip.DefaultCompression } - fmt.Printf("CompressionLevel: %d\n", p.config.CompressionLevel) for key, ptr := range templates { if *ptr == "" { errs = packer.MultiErrorAppend( From 7497db67b4d2c652759024e5133501c3cbd0c6b0 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 14:22:26 -0700 Subject: [PATCH 25/28] Tweaked some of the UI/UX around GOMAXPROCS --- post-processor/compress/post-processor.go | 24 +++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index 769f21fb2..1a4b97595 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -60,6 +60,11 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { errs := new(packer.MultiError) + // If there is no explicit number of Go threads to use, then set it + if os.Getenv("GOMAXPROCS") == "" { + runtime.GOMAXPROCS(runtime.NumCPU()) + } + if p.config.OutputPath == "" { p.config.OutputPath = "packer_{{.BuildName}}_{{.Provider}}" } @@ -124,11 +129,13 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac var output io.WriteCloser switch p.config.Algorithm { case "lz4": - ui.Say(fmt.Sprintf("Preparing lz4 compression for %s", target)) + ui.Say(fmt.Sprintf("Using lz4 compression with %d cores for %s", + runtime.GOMAXPROCS(-1), target)) output, err = makeLZ4Writer(outputFile, p.config.CompressionLevel) defer output.Close() case "pgzip": - ui.Say(fmt.Sprintf("Preparing gzip compression for %s", target)) + ui.Say(fmt.Sprintf("Using pgzip compression with %d cores for %s", + runtime.GOMAXPROCS(-1), target)) output, err = makePgzipWriter(outputFile, p.config.CompressionLevel) defer output.Close() default: @@ -137,13 +144,13 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac compression := p.config.Algorithm if compression == "" { - compression = "no" + compression = "no compression" } // Build an archive, if we're supposed to do that. switch p.config.Archive { case "tar": - ui.Say(fmt.Sprintf("Tarring %s with %s compression", target, compression)) + ui.Say(fmt.Sprintf("Tarring %s with %s", target, compression)) err = createTarArchive(artifact.Files(), output) if err != nil { return nil, keep, fmt.Errorf("Error creating tar: %s", err) @@ -155,7 +162,6 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac return nil, keep, fmt.Errorf("Error creating zip: %s", err) } default: - ui.Say(fmt.Sprintf("Copying %s with %s compression", target, compression)) // Filename indicates no tarball (just compress) so we'll do an io.Copy // into our compressor. if len(artifact.Files()) != 1 { @@ -163,18 +169,20 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac "Can only have 1 input file when not using tar/zip. Found %d "+ "files: %v", len(artifact.Files()), artifact.Files()) } + archiveFile := artifact.Files()[0] + ui.Say(fmt.Sprintf("Archiving %s with %s", archiveFile, compression)) - source, err := os.Open(artifact.Files()[0]) + source, err := os.Open(archiveFile) if err != nil { return nil, keep, fmt.Errorf( "Failed to open source file %s for reading: %s", - artifact.Files()[0], err) + archiveFile, err) } defer source.Close() if _, err = io.Copy(output, source); err != nil { return nil, keep, fmt.Errorf("Failed to compress %s: %s", - artifact.Files()[0], err) + archiveFile, err) } } From d02f6644d2407a64c9a6cb3b519922363c45bc96 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 18:21:03 -0700 Subject: [PATCH 26/28] Refactored test so it's more DRY and also rearranged things so the test cases and configs are at the top of the file --- .../compress/post-processor_test.go | 291 +++++++----------- 1 file changed, 110 insertions(+), 181 deletions(-) diff --git a/post-processor/compress/post-processor_test.go b/post-processor/compress/post-processor_test.go index f6e82b1c9..31525adf7 100644 --- a/post-processor/compress/post-processor_test.go +++ b/post-processor/compress/post-processor_test.go @@ -12,39 +12,6 @@ import ( "github.com/mitchellh/packer/template" ) -func setup(t *testing.T) (packer.Ui, packer.Artifact, error) { - // Create fake UI and Cache - ui := packer.TestUi(t) - cache := &packer.FileCache{CacheDir: os.TempDir()} - - // Create config for file builder - const fileConfig = `{"builders":[{"type":"file","target":"package.txt","content":"Hello world!"}]}` - tpl, err := template.Parse(strings.NewReader(fileConfig)) - if err != nil { - return nil, nil, fmt.Errorf("Unable to parse setup configuration: %s", err) - } - - // Prepare the file builder - builder := file.Builder{} - warnings, err := builder.Prepare(tpl.Builders["file"].Config) - if len(warnings) > 0 { - for _, warn := range warnings { - return nil, nil, fmt.Errorf("Configuration warning: %s", warn) - } - } - if err != nil { - return nil, nil, fmt.Errorf("Invalid configuration: %s", err) - } - - // Run the file builder - artifact, err := builder.Run(ui, nil, cache) - if err != nil { - return nil, nil, fmt.Errorf("Failed to build artifact: %s", err) - } - - return ui, artifact, err -} - func TestDetectFilename(t *testing.T) { // Test default / fallback with no file extension nakedFilename := Config{OutputPath: "test"} @@ -87,154 +54,6 @@ func TestDetectFilename(t *testing.T) { } } -func TestSimpleCompress(t *testing.T) { - if os.Getenv(env.TestEnvVar) == "" { - t.Skip(fmt.Sprintf( - "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) - } - - ui, artifact, err := setup(t) - if err != nil { - t.Fatalf("Error bootstrapping test: %s", err) - } - if artifact != nil { - defer artifact.Destroy() - } - - tpl, err := template.Parse(strings.NewReader(simpleTestCase)) - if err != nil { - t.Fatalf("Unable to parse test config: %s", err) - } - - compressor := PostProcessor{} - compressor.Configure(tpl.PostProcessors[0][0].Config) - artifactOut, _, err := compressor.PostProcess(ui, artifact) - if err != nil { - t.Fatalf("Failed to compress artifact: %s", err) - } - // Cleanup after the test completes - defer artifactOut.Destroy() - - // Verify things look good - fi, err := os.Stat("package.tar.gz") - if err != nil { - t.Errorf("Unable to read archive: %s", err) - } - if fi.IsDir() { - t.Error("Archive should not be a directory") - } -} - -func TestZipArchive(t *testing.T) { - if os.Getenv(env.TestEnvVar) == "" { - t.Skip(fmt.Sprintf( - "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) - } - - ui, artifact, err := setup(t) - if err != nil { - t.Fatalf("Error bootstrapping test: %s", err) - } - if artifact != nil { - defer artifact.Destroy() - } - - tpl, err := template.Parse(strings.NewReader(zipTestCase)) - if err != nil { - t.Fatalf("Unable to parse test config: %s", err) - } - - compressor := PostProcessor{} - compressor.Configure(tpl.PostProcessors[0][0].Config) - artifactOut, _, err := compressor.PostProcess(ui, artifact) - if err != nil { - t.Fatalf("Failed to archive artifact: %s", err) - } - // Cleanup after the test completes - defer artifactOut.Destroy() - - // Verify things look good - _, err = os.Stat("package.zip") - if err != nil { - t.Errorf("Unable to read archive: %s", err) - } -} - -func TestTarArchive(t *testing.T) { - if os.Getenv(env.TestEnvVar) == "" { - t.Skip(fmt.Sprintf( - "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) - } - - ui, artifact, err := setup(t) - if err != nil { - t.Fatalf("Error bootstrapping test: %s", err) - } - if artifact != nil { - defer artifact.Destroy() - } - - tpl, err := template.Parse(strings.NewReader(tarTestCase)) - if err != nil { - t.Fatalf("Unable to parse test config: %s", err) - } - - compressor := PostProcessor{} - compressor.Configure(tpl.PostProcessors[0][0].Config) - artifactOut, _, err := compressor.PostProcess(ui, artifact) - if err != nil { - t.Fatalf("Failed to archive artifact: %s", err) - } - // Cleanup after the test completes - defer artifactOut.Destroy() - - // Verify things look good - _, err = os.Stat("package.tar") - if err != nil { - t.Errorf("Unable to read archive: %s", err) - } -} - -func TestCompressOptions(t *testing.T) { - if os.Getenv(env.TestEnvVar) == "" { - t.Skip(fmt.Sprintf( - "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) - } - - ui, artifact, err := setup(t) - if err != nil { - t.Fatalf("Error bootstrapping test: %s", err) - } - if artifact != nil { - defer artifact.Destroy() - } - - tpl, err := template.Parse(strings.NewReader(optionsTestCase)) - if err != nil { - t.Fatalf("Unable to parse test config: %s", err) - } - - compressor := PostProcessor{} - compressor.Configure(tpl.PostProcessors[0][0].Config) - - if compressor.config.CompressionLevel != 9 { - t.Errorf("Expected compression_level 9, got %d", compressor.config.CompressionLevel) - } - - artifactOut, _, err := compressor.PostProcess(ui, artifact) - if err != nil { - t.Fatalf("Failed to archive artifact: %s", err) - } - // Cleanup after the test completes - defer artifactOut.Destroy() - - // Verify things look good - _, err = os.Stat("package.gz") - if err != nil { - t.Errorf("Unable to read archive: %s", err) - } -} - const simpleTestCase = ` { "post-processors": [ @@ -246,6 +65,19 @@ const simpleTestCase = ` } ` +func TestSimpleCompress(t *testing.T) { + artifact := testArchive(t, simpleTestCase) + defer artifact.Destroy() + + fi, err := os.Stat("package.tar.gz") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } + if fi.IsDir() { + t.Error("Archive should not be a directory") + } +} + const zipTestCase = ` { "post-processors": [ @@ -257,6 +89,17 @@ const zipTestCase = ` } ` +func TestZipArchive(t *testing.T) { + artifact := testArchive(t, zipTestCase) + defer artifact.Destroy() + + // Verify things look good + _, err := os.Stat("package.zip") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } +} + const tarTestCase = ` { "post-processors": [ @@ -268,6 +111,17 @@ const tarTestCase = ` } ` +func TestTarArchive(t *testing.T) { + artifact := testArchive(t, tarTestCase) + defer artifact.Destroy() + + // Verify things look good + _, err := os.Stat("package.tar") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } +} + const optionsTestCase = ` { "post-processors": [ @@ -279,3 +133,78 @@ const optionsTestCase = ` ] } ` + +func TestCompressOptions(t *testing.T) { + artifact := testArchive(t, optionsTestCase) + defer artifact.Destroy() + + // Verify things look good + _, err := os.Stat("package.gz") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } +} + +// Test Helpers + +func setup(t *testing.T) (packer.Ui, packer.Artifact, error) { + // Create fake UI and Cache + ui := packer.TestUi(t) + cache := &packer.FileCache{CacheDir: os.TempDir()} + + // Create config for file builder + const fileConfig = `{"builders":[{"type":"file","target":"package.txt","content":"Hello world!"}]}` + tpl, err := template.Parse(strings.NewReader(fileConfig)) + if err != nil { + return nil, nil, fmt.Errorf("Unable to parse setup configuration: %s", err) + } + + // Prepare the file builder + builder := file.Builder{} + warnings, err := builder.Prepare(tpl.Builders["file"].Config) + if len(warnings) > 0 { + for _, warn := range warnings { + return nil, nil, fmt.Errorf("Configuration warning: %s", warn) + } + } + if err != nil { + return nil, nil, fmt.Errorf("Invalid configuration: %s", err) + } + + // Run the file builder + artifact, err := builder.Run(ui, nil, cache) + if err != nil { + return nil, nil, fmt.Errorf("Failed to build artifact: %s", err) + } + + return ui, artifact, err +} + +func testArchive(t *testing.T, config string) packer.Artifact { + if os.Getenv(env.TestEnvVar) == "" { + t.Skip(fmt.Sprintf( + "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) + } + + ui, artifact, err := setup(t) + if err != nil { + t.Fatalf("Error bootstrapping test: %s", err) + } + if artifact != nil { + defer artifact.Destroy() + } + + tpl, err := template.Parse(strings.NewReader(config)) + if err != nil { + t.Fatalf("Unable to parse test config: %s", err) + } + + compressor := PostProcessor{} + compressor.Configure(tpl.PostProcessors[0][0].Config) + artifactOut, _, err := compressor.PostProcess(ui, artifact) + if err != nil { + t.Fatalf("Failed to compress artifact: %s", err) + } + + return artifactOut +} From 801e5aaa30ccb73fcf4ebee49b4c519aba610e16 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 18:25:46 -0700 Subject: [PATCH 27/28] Move test configs into the test func --- .../compress/post-processor_test.go | 97 +++++++++---------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/post-processor/compress/post-processor_test.go b/post-processor/compress/post-processor_test.go index 31525adf7..a4812723f 100644 --- a/post-processor/compress/post-processor_test.go +++ b/post-processor/compress/post-processor_test.go @@ -54,19 +54,18 @@ func TestDetectFilename(t *testing.T) { } } -const simpleTestCase = ` -{ - "post-processors": [ - { - "type": "compress", - "output": "package.tar.gz" - } - ] -} -` - func TestSimpleCompress(t *testing.T) { - artifact := testArchive(t, simpleTestCase) + const config = ` + { + "post-processors": [ + { + "type": "compress", + "output": "package.tar.gz" + } + ] + } + ` + artifact := testArchive(t, config) defer artifact.Destroy() fi, err := os.Stat("package.tar.gz") @@ -78,19 +77,19 @@ func TestSimpleCompress(t *testing.T) { } } -const zipTestCase = ` -{ - "post-processors": [ - { - "type": "compress", - "output": "package.zip" - } - ] -} -` - func TestZipArchive(t *testing.T) { - artifact := testArchive(t, zipTestCase) + const config = ` + { + "post-processors": [ + { + "type": "compress", + "output": "package.zip" + } + ] + } + ` + + artifact := testArchive(t, config) defer artifact.Destroy() // Verify things look good @@ -100,19 +99,19 @@ func TestZipArchive(t *testing.T) { } } -const tarTestCase = ` -{ - "post-processors": [ - { - "type": "compress", - "output": "package.tar" - } - ] -} -` - func TestTarArchive(t *testing.T) { - artifact := testArchive(t, tarTestCase) + const config = ` + { + "post-processors": [ + { + "type": "compress", + "output": "package.tar" + } + ] + } + ` + + artifact := testArchive(t, config) defer artifact.Destroy() // Verify things look good @@ -122,20 +121,20 @@ func TestTarArchive(t *testing.T) { } } -const optionsTestCase = ` -{ - "post-processors": [ - { - "type": "compress", - "output": "package.gz", - "compression_level": 9 - } - ] -} -` - func TestCompressOptions(t *testing.T) { - artifact := testArchive(t, optionsTestCase) + const config = ` + { + "post-processors": [ + { + "type": "compress", + "output": "package.gz", + "compression_level": 9 + } + ] + } + ` + + artifact := testArchive(t, config) defer artifact.Destroy() // Verify things look good From 2d92fd8733ca73487ed5fe9c08d1016d3f25cc9b Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 19:08:13 -0700 Subject: [PATCH 28/28] Added test case for gzip that expands the data and compares to what we put in --- post-processor/compress/post-processor_test.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/post-processor/compress/post-processor_test.go b/post-processor/compress/post-processor_test.go index a4812723f..d7bca6c7a 100644 --- a/post-processor/compress/post-processor_test.go +++ b/post-processor/compress/post-processor_test.go @@ -1,7 +1,9 @@ package compress import ( + "compress/gzip" "fmt" + "io/ioutil" "os" "strings" "testing" @@ -54,6 +56,8 @@ func TestDetectFilename(t *testing.T) { } } +const expectedFileContents = "Hello world!" + func TestSimpleCompress(t *testing.T) { const config = ` { @@ -137,10 +141,13 @@ func TestCompressOptions(t *testing.T) { artifact := testArchive(t, config) defer artifact.Destroy() - // Verify things look good - _, err := os.Stat("package.gz") - if err != nil { - t.Errorf("Unable to read archive: %s", err) + filename := "package.gz" + archive, _ := os.Open(filename) + gzipReader, _ := gzip.NewReader(archive) + data, _ := ioutil.ReadAll(gzipReader) + + if string(data) != expectedFileContents { + t.Errorf("Expected:\n%s\nFound:\n%s\n", expectedFileContents, data) } }